)]}' { "commit": "22a5ae1ea8d7ef3b93d037932c3de4a14a1e41cf", "tree": "471e370b39446215e3635f088e715ed8499175cf", "parents": [ "dcf7d7edacb8dc7c41903dca94d20ff0809235b9" ], "author": { "name": "ggershinsky", "email": "ggershinsky@users.noreply.github.com", "time": "Mon Jul 06 11:01:36 2020 +0300" }, "committer": { "name": "Gabor Szadovszky", "email": "gabor@apache.org", "time": "Tue Jul 28 12:52:22 2020 +0200" }, "message": "PARQUET-1373: Encryption key tools (#615)\n\n* comments\r\n* update key tools\r\n* double wrap for minimizing KMS calls\r\n* Add information about KMS instance ID to footer metadata. Then on file reading, KMS instance ID doesn\u0027t have to be provided in properties, but can be read from the metadata.\r\nAdd RemoteKmsClient abstract class to assist implementing KMSClients for remote KMSs, that are accessed using URL.\r\nMake DoubleWrappedKeyManager inherit from WrappedKeyManager and make FileKeyManager an abstract class. Add a static factory method to FileKeyManager to initialize an appropriate KSMClient and Key manager.\r\nKMS URL should be specified in properties either directly or in a list. KMS instance ID is either default, or should be specified in properties or read from footer metadata.\r\n* major update - key rotation, crypto factory, etc\r\n* Change caches of EnvelopeKeyManager and EnvelopeKeyRetriever to be per token.\r\nKmsClient is per token and read/write KEK caches too.\r\nAdd default token value for InMemoryKMS, which has no tokens.\r\nUse concurrentHashMap for caches with computeIfAbsent.\r\nAdd expiration using to the caches - both time-based and on-demand.\r\nOn expiration delete the per-token entries from caches.\r\nAdd method for cache invalidation per token.\r\nAdd abstract methods to be implemented by RemoteKmsClients.\r\n* add in-memory KMS\r\n* Change RemoteKmsClient exceptions to IOException\r\ninstead of the higher-level ParquetCryptoRuntimeException.\r\nChange to constant names to uppercase.\r\n* Add sample VaultClient.\r\n* interface changes\r\n* Add okHttp3 dependency for VaultClient sample.\r\n* wrapping changes\r\n* Use JSON serialization for key material.\r\n* separate write and read path, update caching\r\n* improved refactoring\r\n* key rotation improvements\r\n* Add TestPropertiesDrivenEncryption\r\n* get and resfresh token for all KMS clients\r\n* minor changes\r\n* Use ConcurrentHashMap for caches\r\n* caching and store updates\r\n* Rename some encryption/decryption configurations and make the test parameterized\r\nto test combinations of isKeyMaterialExternalStorage, isDoubleWrapping, isWrapLocally.\r\nAdd RemoteKmsClient mock for remote wrapping.\r\n* add removeCacheEntriesForAllTokens\r\n* Make common method setCommonKMSProperties and extract classname strings from classes\r\n* Change TestPropertiesDrivenEncryption to accomodate latest API changes.\r\n* Remove StringUtils\r\n* address review comments\r\n* key material documentation\r\n* Boolean objects\r\n\r\nCo-authored-by: Maya Anderson \u003cmayaa@il.ibm.com\u003e\r\n", "tree_diff": [ { "type": "modify", "old_id": "939a79a51dacb7aa3dc949092283b52b358f1aff", "old_mode": 33188, "old_path": "parquet-hadoop/README.md", "new_id": "457d326d5146b0aaa7e8cb9068f00551b307eb10", "new_mode": 33188, "new_path": "parquet-hadoop/README.md" }, { "type": "modify", "old_id": "af2ac170dfa566912506b6d424530dc41bd025ae", "old_mode": 33188, "old_path": "parquet-hadoop/pom.xml", "new_id": "243aa1db888f45de44fb132e955c84ec1cface2c", "new_mode": 33188, "new_path": "parquet-hadoop/pom.xml" }, { "type": "modify", "old_id": "683155dbaac6cfcd828852562e498855ce928f23", "old_mode": 33261, "old_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/InternalFileDecryptor.java", "new_id": "ab1baa4b78cec87aacbf2c226f2077e1c43ab9d4", "new_mode": 33261, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/InternalFileDecryptor.java" }, { "type": "modify", "old_id": "c167a5eb362b84030f294cff62a5ea930534c570", "old_mode": 33261, "old_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/InternalFileEncryptor.java", "new_id": "d9619d147431fc1c1f3d2b3cde3aebb4b78c112c", "new_mode": 33261, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/InternalFileEncryptor.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "b9db9444b63b94ad99eaeda28bba65b065fc8840", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/FileKeyMaterialStore.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "60bc77f4529d08a9aae635990e250a216004b39b", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/FileKeyUnwrapper.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "488601962f2832290fd9e804db3f3030ff9d851c", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/FileKeyWrapper.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "fae700afadd0b1f9cb295c9142acf47a49d3a5a0", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/HadoopFSKeyMaterialStore.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "3d49ff4ad585a5cf89b76c4f98472714bc83b2fd", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KeyMaterial.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "f75fe7e0daec0619fbd95901f9236d6661c793b6", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KeyMetadata.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "6116babeeeeda3987f5e347c724feef9db07988c", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KeyToolkit.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "d55dd87d3620bf4414356954aa5d7981a58931c6", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KmsClient.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "8fdeca2540405d8c158d2d4fb8e8e60f0ce8876a", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/PropertiesDrivenCryptoFactory.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "c8a7435d970a699e2f82d8053ce29b0a680f22be", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/RemoteKmsClient.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "b5edbfb379a10af8386bd47eb8bf0b1ad4d79a91", "new_mode": 33188, "new_path": "parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/TwoLevelCacheWithExpiration.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "3e73d31b566337d24a3013ea54f04fb2d528edcf", "new_mode": 33188, "new_path": "parquet-hadoop/src/test/java/org/apache/parquet/crypto/SingleRow.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "a9c078b07670ea7f78e2d81f28749f83ced60900", "new_mode": 33188, "new_path": "parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestPropertiesDrivenEncryption.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "1aa1b145656555459b461315f68b2e769d306208", "new_mode": 33188, "new_path": "parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/InMemoryKMS.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "9d26cc1d1fa53ed8c866c0e165afe6b1ac3defa9", "new_mode": 33261, "new_path": "parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/samples/VaultClient.java" }, { "type": "modify", "old_id": "7f0111d2235dca4d0ade64b701e5f9037616939f", "old_mode": 33188, "old_path": "parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java", "new_id": "12f7ff5d292bb44d2dd22c1aa3933096d87cb509", "new_mode": 33188, "new_path": "parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestEncryptionOptions.java" } ] }