From 01cd8325b3b4192e896f864abb6efe559c8d6849 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:01:59 -0400 Subject: [PATCH 01/10] add it to the manifest --- sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index e0450137..ef30f72c 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -134,6 +134,7 @@ static public class KeyAccess { public String encryptedMetadata; public String kid; + public String sid; @Override public boolean equals(Object o) { From 35f0a71c594682ecea47069fcc7b9d29aafa4e70 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:46:05 -0400 Subject: [PATCH 02/10] add split logic on decrypt --- .../java/io/opentdf/platform/sdk/Config.java | 10 +++++ .../java/io/opentdf/platform/sdk/TDF.java | 45 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java index 15452d95..875bf71d 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java @@ -53,6 +53,16 @@ public AssertionConfig() { } } + public static class SplitStep { + public String kas; + public String splitID; + + public SplitStep(String kas, String sid) { + this.kas = kas; + this.splitID = sid; + } + } + public static class TDFConfig { public int defaultSegmentSize; public boolean enableEncryption; diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 1d18732f..f60d19f4 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -13,6 +13,9 @@ import com.nimbusds.jose.crypto.RSASSAVerifier; import com.nimbusds.jwt.JWTClaimsSet; import com.nimbusds.jwt.SignedJWT; + +import io.opentdf.platform.sdk.Config.SplitStep; + import com.nimbusds.jose.crypto.RSASSASigner; import com.nimbusds.jose.crypto.MACSigner; import org.apache.commons.codec.DecoderException; @@ -495,22 +498,44 @@ private void fillInPublicKeyInfo(List kasInfoList, SDK.KAS kas) } public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, - IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { + IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { TDFReader tdfReader = new TDFReader(tdf); String manifestJson = tdfReader.manifest(); Manifest manifest = gson.fromJson(manifestJson, Manifest.class); byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; + + Map knownSplits = new HashMap<>(); + Map foundSplits = new HashMap<>(); + Map skippedSplits = new HashMap<>(); + boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && !manifest.encryptionInformation.keyAccessObj.get(0).sid.isEmpty(); MessageDigest digest = MessageDigest.getInstance("SHA-256"); if (manifest.payload.isEncrypted) { for (Manifest.KeyAccess keyAccess: manifest.encryptionInformation.keyAccessObj) { - var unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); + Config.SplitStep ss = new Config.SplitStep(keyAccess.url, keyAccess.sid); + byte[] unwrappedKey; + if (!mixedSplits) { + unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); + } else { + knownSplits.put(ss.splitID, true); + if (foundSplits.get(ss.splitID)){ + continue; + } + try { + unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); + } catch (Exception e) { + skippedSplits.put(ss, e); + continue; + } + } + for (int index = 0; index < unwrappedKey.length; index++) { payloadKey[index] ^= unwrappedKey[index]; } + foundSplits.put(ss.splitID, true); if (keyAccess.encryptedMetadata != null && !keyAccess.encryptedMetadata.isEmpty()) { AesGcm aesGcm = new AesGcm(unwrappedKey); @@ -528,6 +553,22 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC unencryptedMetadata = new String(decrypted, StandardCharsets.UTF_8); } } + + if (mixedSplits && knownSplits.size() > foundSplits.size()) { + List exceptionList = new ArrayList<>(skippedSplits.size() + 1); + exceptionList.add(new Exception("splitKey.unable to reconstruct split key: " + skippedSplits)); + + for (Map.Entry entry : skippedSplits.entrySet()) { + exceptionList.add(entry.getValue()); + } + + StringBuilder combinedMessage = new StringBuilder(); + for (Exception e : exceptionList) { + combinedMessage.append(e.getMessage()).append("\n"); + } + + throw new Exception(combinedMessage.toString()); + } } // Validate root signature From ec5e91bcdeb44048045b02106356f0e631b3e576 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:49:47 -0400 Subject: [PATCH 03/10] update cli exception handling --- cmdline/src/main/java/io/opentdf/platform/Command.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 8303d22c..31324803 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -84,7 +84,7 @@ private SDK buildSDK() { void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException, TDF.FailedToCreateGMAC, - JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { + JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { try (var stdout = new BufferedOutputStream(System.out)) { @@ -95,7 +95,7 @@ void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) th } @CommandLine.Command(name = "metadata") void readMetadata(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, - TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException { + TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, Exception { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { From 484dcaa60d62ef2df68a62388cb3415ba903b020 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:56:57 -0400 Subject: [PATCH 04/10] make custom exception type --- cmdline/src/main/java/io/opentdf/platform/Command.java | 6 ++++-- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 31324803..38a832c8 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -3,6 +3,8 @@ import com.nimbusds.jose.JOSEException; import io.opentdf.platform.sdk.*; import io.opentdf.platform.sdk.TDF; +import io.opentdf.platform.sdk.TDF.SplitKeyException; + import org.apache.commons.codec.DecoderException; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -84,7 +86,7 @@ private SDK buildSDK() { void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException, TDF.FailedToCreateGMAC, - JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { + JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { try (var stdout = new BufferedOutputStream(System.out)) { @@ -95,7 +97,7 @@ void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) th } @CommandLine.Command(name = "metadata") void readMetadata(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, - TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, Exception { + TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, SplitKeyException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index f60d19f4..27dfcc65 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -71,6 +71,12 @@ public TDF() { private static final Gson gson = new Gson(); + public class SplitKeyException extends Exception { + public SplitKeyException(String errorMessage) { + super(errorMessage); + } + } + public static class DataSizeNotSupported extends RuntimeException { public DataSizeNotSupported(String errorMessage) { super(errorMessage); @@ -498,7 +504,7 @@ private void fillInPublicKeyInfo(List kasInfoList, SDK.KAS kas) } public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, - IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { + IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { TDFReader tdfReader = new TDFReader(tdf); String manifestJson = tdfReader.manifest(); @@ -567,7 +573,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC combinedMessage.append(e.getMessage()).append("\n"); } - throw new Exception(combinedMessage.toString()); + throw new SplitKeyException(combinedMessage.toString()); } } From 4552d078a5b1dd328ad72b82f769679444990b23 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 13:15:19 -0400 Subject: [PATCH 05/10] null check --- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 27dfcc65..4642b19f 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -527,7 +527,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); } else { knownSplits.put(ss.splitID, true); - if (foundSplits.get(ss.splitID)){ + if (foundSplits.get(ss.splitID) != null){ continue; } try { From 8b006dd982530e87dc7ee2a7f3d9579abfc1ba76 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 13:33:33 -0400 Subject: [PATCH 06/10] check for null pointer --- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 4642b19f..21056300 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -515,7 +515,9 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC Map knownSplits = new HashMap<>(); Map foundSplits = new HashMap<>(); Map skippedSplits = new HashMap<>(); - boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && !manifest.encryptionInformation.keyAccessObj.get(0).sid.isEmpty(); + boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && + (manifest.encryptionInformation.keyAccessObj.get(0).sid != null) && + !manifest.encryptionInformation.keyAccessObj.get(0).sid.isEmpty(); MessageDigest digest = MessageDigest.getInstance("SHA-256"); From fb889a3d88ebdab8b89468fc85b966092d84c0d2 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Wed, 14 Aug 2024 13:05:02 -0400 Subject: [PATCH 07/10] encrypt w key splits --- .github/workflows/checks.yaml | 55 ++++++++ .../java/io/opentdf/platform/Command.java | 2 +- .../java/io/opentdf/platform/sdk/Config.java | 2 + .../java/io/opentdf/platform/sdk/TDF.java | 119 ++++++++++++------ 4 files changed, 137 insertions(+), 41 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index d3f909ff..63459e6c 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -190,6 +190,61 @@ jobs: fi working-directory: cmdline + - name: Start second in background + uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 + with: + run: | + opentdf-beta.yaml yq e ' + (.server.port = 8282) + | (.mode = ["kas"]) + | (.sdk_config = {"endpoint":"http://localhost:8080","plaintext":true,"client_id":"opentdf","client_secret":"secret"}) + ' + && .github/scripts/watch.sh opentdf-beta.yaml ./opentdf --config-file ./opentdf-beta.yaml start + wait-on: | + tcp:localhost:8282 + log-output-if: true + wait-for: 90s + working-directory: platform + - name: Make sure that the second platform is up + run: | + grpcurl -plaintext localhost:8282 list && \ + grpcurl -plaintext localhost:8282 kas.AccessService/PublicKey + - name: Validate multikas through the command line interface + run: | + printf 'here is some data to encrypt' > data + + java -jar target/cmdline.jar \ + --client-id=opentdf-sdk \ + --client-secret=secret \ + --platform-endpoint=localhost:8080 \ + -i \ + encrypt --kas-url=localhost:8080,localhost:8282 -f data -m 'here is some metadata' > test.tdf + + java -jar target/cmdline.jar \ + --client-id=opentdf-sdk \ + --client-secret=secret \ + --platform-endpoint=localhost:8080 \ + -i \ + decrypt -f test.tdf > decrypted + + java -jar target/cmdline.jar \ + --client-id=opentdf-sdk \ + --client-secret=secret \ + --platform-endpoint=localhost:8080 \ + -i \ + metadata -f test.tdf > metadata + + if ! diff -q data decrypted; then + printf 'decrypted data is incorrect [%s]' "$(< decrypted)" + exit 1 + fi + + if [ "$(< metadata)" != 'here is some metadata' ]; then + printf 'metadata is incorrect [%s]\n' "$(< metadata)" + exit 1 + fi + working-directory: cmdline + platform-xtest: permissions: contents: read diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 38a832c8..f0874360 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -51,7 +51,7 @@ class Command { @CommandLine.Command(name = "encrypt") void encrypt( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, - @Option(names = {"-k", "--kas-url"}, required = true) List kas, + @Option(names = {"-k", "--kas-url"}, required = true, split = ",") List kas, @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata) throws IOException, JOSEException { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java index 875bf71d..cc276196 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java @@ -77,6 +77,7 @@ public static class TDFConfig { public List assertionList; public AssertionConfig assertionConfig; public String mimeType; + public List splitPlan; public TDFConfig() { this.defaultSegmentSize = DEFAULT_SEGMENT_SIZE; @@ -88,6 +89,7 @@ public TDFConfig() { this.kasInfoList = new ArrayList<>(); this.assertionList = new ArrayList<>(); this.mimeType = DEFAULT_MIME_TYPE; + this.splitPlan = new ArrayList<>(); } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 21056300..bef542e4 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -14,8 +14,6 @@ import com.nimbusds.jwt.JWTClaimsSet; import com.nimbusds.jwt.SignedJWT; -import io.opentdf.platform.sdk.Config.SplitStep; - import com.nimbusds.jose.crypto.RSASSASigner; import com.nimbusds.jose.crypto.MACSigner; import org.apache.commons.codec.DecoderException; @@ -173,57 +171,111 @@ PolicyObject createPolicyObject(List attributes) { } private static final Base64.Encoder encoder = Base64.getEncoder(); - private void prepareManifest(Config.TDFConfig tdfConfig) { + private void prepareManifest(Config.TDFConfig tdfConfig, SDK.KAS kas) { manifest.encryptionInformation.keyAccessType = kSplitKeyType; manifest.encryptionInformation.keyAccessObj = new ArrayList<>(); PolicyObject policyObject = createPolicyObject(tdfConfig.attributes); String base64PolicyObject = encoder.encodeToString(gson.toJson(policyObject).getBytes(StandardCharsets.UTF_8)); List symKeys = new ArrayList<>(); + Map latestKASInfo = new HashMap<>(); + if (tdfConfig.splitPlan.isEmpty()) { + // Default split plan: Split keys across all KASes + List splitPlan = new ArrayList<>(tdfConfig.kasInfoList.size()); + for (Config.KASInfo kasInfo : tdfConfig.kasInfoList) { + Config.SplitStep step = new Config.SplitStep(kasInfo.URL, ""); + if (tdfConfig.kasInfoList.size() > 1) { + step.splitID = String.format("s-%d", splitPlan.size()); + } + splitPlan.add(step); + if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isEmpty()) { + latestKASInfo.put(kasInfo.URL, kasInfo); + } + } + tdfConfig.splitPlan = splitPlan; + } + // Seed anything passed in manually for (Config.KASInfo kasInfo: tdfConfig.kasInfoList) { - if (kasInfo.PublicKey == null || kasInfo.PublicKey.isEmpty()) { - throw new KasPublicKeyMissing("Kas public key is missing in kas information list"); + if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isEmpty()) { + latestKASInfo.put(kasInfo.URL, kasInfo); + } + } + + // split plan: restructure by conjunctions + Map> conjunction = new HashMap<>(); + List splitIDs = new ArrayList<>(); + + for (Config.SplitStep splitInfo : tdfConfig.splitPlan) { + // Public key was passed in with kasInfoList + // TODO First look up in attribute information / add to split plan? + Config.KASInfo ki = latestKASInfo.get(splitInfo.kas); + if (ki == null || ki.PublicKey == null || ki.PublicKey.isBlank()) { + logger.info("no public key provided for KAS at {}, retrieving", splitInfo.kas); + var getKI = new Config.KASInfo(); + getKI.URL = splitInfo.kas; + getKI.PublicKey = kas.getPublicKey(getKI); + getKI.KID = kas.getKid(getKI); + latestKASInfo.put(splitInfo.kas, getKI); + ki = getKI; + } + if (conjunction.containsKey(splitInfo.splitID)) { + conjunction.get(splitInfo.splitID).add(ki); + } else { + List newList = new ArrayList<>(); + newList.add(ki); + conjunction.put(splitInfo.splitID, newList); + splitIDs.add(splitInfo.splitID); } + } + for (String splitID: splitIDs) { // Symmetric key byte[] symKey = new byte[GCM_KEY_SIZE]; sRandom.nextBytes(symKey); - - Manifest.KeyAccess keyAccess = new Manifest.KeyAccess(); - keyAccess.keyType = kWrapped; - keyAccess.url = kasInfo.URL; - keyAccess.kid = kasInfo.KID; - keyAccess.protocol = kKasProtocol; + symKeys.add(symKey); // Add policyBinding var hexBinding = Hex.encodeHexString(CryptoUtils.CalculateSHA256Hmac(symKey, base64PolicyObject.getBytes(StandardCharsets.UTF_8))); var policyBinding = new Manifest.PolicyBinding(); policyBinding.alg = kHmacIntegrityAlgorithm; policyBinding.hash = encoder.encodeToString(hexBinding.getBytes(StandardCharsets.UTF_8)); - keyAccess.policyBinding = policyBinding; - - // Wrap the key with kas public key - AsymEncryption asymmetricEncrypt = new AsymEncryption(kasInfo.PublicKey); - byte[] wrappedKey = asymmetricEncrypt.encrypt(symKey); - - keyAccess.wrappedKey = encoder.encodeToString(wrappedKey); + // Add meta data + var encryptedMetadata = new String(); if(tdfConfig.metaData != null && !tdfConfig.metaData.trim().isEmpty()) { AesGcm aesGcm = new AesGcm(symKey); var encrypted = aesGcm.encrypt(tdfConfig.metaData.getBytes(StandardCharsets.UTF_8)); - EncryptedMetadata encryptedMetadata = new EncryptedMetadata(); - encryptedMetadata.iv = encoder.encodeToString(encrypted.getIv()); - encryptedMetadata.ciphertext = encoder.encodeToString(encrypted.asBytes()); + EncryptedMetadata em = new EncryptedMetadata(); + em.iv = encoder.encodeToString(encrypted.getIv()); + em.ciphertext = encoder.encodeToString(encrypted.asBytes()); - var metadata = gson.toJson(encryptedMetadata); - keyAccess.encryptedMetadata = encoder.encodeToString(metadata.getBytes(StandardCharsets.UTF_8)); + var metadata = gson.toJson(em); + encryptedMetadata = encoder.encodeToString(metadata.getBytes(StandardCharsets.UTF_8)); } - symKeys.add(symKey); - manifest.encryptionInformation.keyAccessObj.add(keyAccess); + for (Config.KASInfo kasInfo: conjunction.get(splitID)){ + if (kasInfo.PublicKey == null || kasInfo.PublicKey.isEmpty()) { + throw new KasPublicKeyMissing("Kas public key is missing in kas information list"); + } + + // Wrap the key with kas public key + AsymEncryption asymmetricEncrypt = new AsymEncryption(kasInfo.PublicKey); + byte[] wrappedKey = asymmetricEncrypt.encrypt(symKey); + + Manifest.KeyAccess keyAccess = new Manifest.KeyAccess(); + keyAccess.keyType = kWrapped; + keyAccess.url = kasInfo.URL; + keyAccess.kid = kasInfo.KID; + keyAccess.protocol = kKasProtocol; + keyAccess.policyBinding = policyBinding; + keyAccess.wrappedKey = encoder.encodeToString(wrappedKey); + keyAccess.encryptedMetadata = encryptedMetadata; + + manifest.encryptionInformation.keyAccessObj.add(keyAccess); + } } manifest.encryptionInformation.policy = base64PolicyObject; @@ -328,10 +380,8 @@ public TDFObject createTDF(InputStream payload, throw new KasInfoMissing("kas information is missing"); } - fillInPublicKeyInfo(tdfConfig.kasInfoList, kas); - TDFObject tdfObject = new TDFObject(); - tdfObject.prepareManifest(tdfConfig); + tdfObject.prepareManifest(tdfConfig, kas); long encryptedSegmentSize = tdfConfig.defaultSegmentSize + kGcmIvSize + kAesBlockSize; TDFWriter tdfWriter = new TDFWriter(outputStream); @@ -492,17 +542,6 @@ public TDFObject createTDF(InputStream payload, return tdfObject; } - private void fillInPublicKeyInfo(List kasInfoList, SDK.KAS kas) { - for (var kasInfo: kasInfoList) { - if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isBlank()) { - continue; - } - logger.info("no public key provided for KAS at {}, retrieving", kasInfo.URL); - kasInfo.PublicKey = kas.getPublicKey(kasInfo); - kasInfo.KID = kas.getKid(kasInfo); - } - } - public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { @@ -566,7 +605,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC List exceptionList = new ArrayList<>(skippedSplits.size() + 1); exceptionList.add(new Exception("splitKey.unable to reconstruct split key: " + skippedSplits)); - for (Map.Entry entry : skippedSplits.entrySet()) { + for (Map.Entry entry : skippedSplits.entrySet()) { exceptionList.add(entry.getValue()); } From 03fae78dfe34125a8b49096604cca33b27461a4f Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Wed, 14 Aug 2024 13:26:14 -0400 Subject: [PATCH 08/10] try to fix test, exception fix, sets --- .github/workflows/checks.yaml | 6 +++--- .../src/main/java/io/opentdf/platform/Command.java | 5 ++--- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 14 +++++++------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 63459e6c..cf8f237f 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -190,10 +190,10 @@ jobs: fi working-directory: cmdline - - name: Start second in background - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 + - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 + name: start another KAS server in background with: - run: | + run: > opentdf-beta.yaml yq e ' (.server.port = 8282) | (.mode = ["kas"]) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index f0874360..ddb9d61f 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -3,7 +3,6 @@ import com.nimbusds.jose.JOSEException; import io.opentdf.platform.sdk.*; import io.opentdf.platform.sdk.TDF; -import io.opentdf.platform.sdk.TDF.SplitKeyException; import org.apache.commons.codec.DecoderException; import picocli.CommandLine; @@ -86,7 +85,7 @@ private SDK buildSDK() { void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException, TDF.FailedToCreateGMAC, - JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { + JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { try (var stdout = new BufferedOutputStream(System.out)) { @@ -97,7 +96,7 @@ void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) th } @CommandLine.Command(name = "metadata") void readMetadata(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, - TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, SplitKeyException { + TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index bef542e4..1beb902f 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -69,7 +69,7 @@ public TDF() { private static final Gson gson = new Gson(); - public class SplitKeyException extends Exception { + public class SplitKeyException extends IOException { public SplitKeyException(String errorMessage) { super(errorMessage); } @@ -543,7 +543,7 @@ public TDFObject createTDF(InputStream payload, } public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, - IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { + IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { TDFReader tdfReader = new TDFReader(tdf); String manifestJson = tdfReader.manifest(); @@ -551,8 +551,8 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; - Map knownSplits = new HashMap<>(); - Map foundSplits = new HashMap<>(); + Set knownSplits = new HashSet(); + Set foundSplits = new HashSet();; Map skippedSplits = new HashMap<>(); boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && (manifest.encryptionInformation.keyAccessObj.get(0).sid != null) && @@ -567,8 +567,8 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC if (!mixedSplits) { unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); } else { - knownSplits.put(ss.splitID, true); - if (foundSplits.get(ss.splitID) != null){ + knownSplits.add(unencryptedMetadata); + if (foundSplits.contains(ss.splitID)){ continue; } try { @@ -582,7 +582,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC for (int index = 0; index < unwrappedKey.length; index++) { payloadKey[index] ^= unwrappedKey[index]; } - foundSplits.put(ss.splitID, true); + foundSplits.add(ss.splitID); if (keyAccess.encryptedMetadata != null && !keyAccess.encryptedMetadata.isEmpty()) { AesGcm aesGcm = new AesGcm(unwrappedKey); From ee193e3abfcb4ac863a3eeee210b5214aec3c54f Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Wed, 14 Aug 2024 13:35:19 -0400 Subject: [PATCH 09/10] fix test --- .github/workflows/checks.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index cf8f237f..baf65236 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -199,7 +199,7 @@ jobs: | (.mode = ["kas"]) | (.sdk_config = {"endpoint":"http://localhost:8080","plaintext":true,"client_id":"opentdf","client_secret":"secret"}) ' - && .github/scripts/watch.sh opentdf-beta.yaml ./opentdf --config-file ./opentdf-beta.yaml start + && go run ./service --config-file ./opentdf-beta.yaml start wait-on: | tcp:localhost:8282 log-output-if: true @@ -207,7 +207,6 @@ jobs: working-directory: platform - name: Make sure that the second platform is up run: | - grpcurl -plaintext localhost:8282 list && \ grpcurl -plaintext localhost:8282 kas.AccessService/PublicKey - name: Validate multikas through the command line interface run: | From bc4afc152bf64b63b0d77537baaa1354940e722d Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Mon, 19 Aug 2024 11:11:30 -0400 Subject: [PATCH 10/10] suggested change --- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 1beb902f..96d11112 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -182,10 +182,11 @@ private void prepareManifest(Config.TDFConfig tdfConfig, SDK.KAS kas) { if (tdfConfig.splitPlan.isEmpty()) { // Default split plan: Split keys across all KASes List splitPlan = new ArrayList<>(tdfConfig.kasInfoList.size()); + int i = 0; for (Config.KASInfo kasInfo : tdfConfig.kasInfoList) { Config.SplitStep step = new Config.SplitStep(kasInfo.URL, ""); if (tdfConfig.kasInfoList.size() > 1) { - step.splitID = String.format("s-%d", splitPlan.size()); + step.splitID = String.format("s-%d", i++); } splitPlan.add(step); if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isEmpty()) {