From 01cd8325b3b4192e896f864abb6efe559c8d6849 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:01:59 -0400 Subject: [PATCH 01/16] add it to the manifest --- sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index e0450137..ef30f72c 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -134,6 +134,7 @@ static public class KeyAccess { public String encryptedMetadata; public String kid; + public String sid; @Override public boolean equals(Object o) { From 35f0a71c594682ecea47069fcc7b9d29aafa4e70 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:46:05 -0400 Subject: [PATCH 02/16] add split logic on decrypt --- .../java/io/opentdf/platform/sdk/Config.java | 10 +++++ .../java/io/opentdf/platform/sdk/TDF.java | 45 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java index 15452d95..875bf71d 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java @@ -53,6 +53,16 @@ public AssertionConfig() { } } + public static class SplitStep { + public String kas; + public String splitID; + + public SplitStep(String kas, String sid) { + this.kas = kas; + this.splitID = sid; + } + } + public static class TDFConfig { public int defaultSegmentSize; public boolean enableEncryption; diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 1d18732f..f60d19f4 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -13,6 +13,9 @@ import com.nimbusds.jose.crypto.RSASSAVerifier; import com.nimbusds.jwt.JWTClaimsSet; import com.nimbusds.jwt.SignedJWT; + +import io.opentdf.platform.sdk.Config.SplitStep; + import com.nimbusds.jose.crypto.RSASSASigner; import com.nimbusds.jose.crypto.MACSigner; import org.apache.commons.codec.DecoderException; @@ -495,22 +498,44 @@ private void fillInPublicKeyInfo(List kasInfoList, SDK.KAS kas) } public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, - IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { + IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { TDFReader tdfReader = new TDFReader(tdf); String manifestJson = tdfReader.manifest(); Manifest manifest = gson.fromJson(manifestJson, Manifest.class); byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; + + Map knownSplits = new HashMap<>(); + Map foundSplits = new HashMap<>(); + Map skippedSplits = new HashMap<>(); + boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && !manifest.encryptionInformation.keyAccessObj.get(0).sid.isEmpty(); MessageDigest digest = MessageDigest.getInstance("SHA-256"); if (manifest.payload.isEncrypted) { for (Manifest.KeyAccess keyAccess: manifest.encryptionInformation.keyAccessObj) { - var unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); + Config.SplitStep ss = new Config.SplitStep(keyAccess.url, keyAccess.sid); + byte[] unwrappedKey; + if (!mixedSplits) { + unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); + } else { + knownSplits.put(ss.splitID, true); + if (foundSplits.get(ss.splitID)){ + continue; + } + try { + unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); + } catch (Exception e) { + skippedSplits.put(ss, e); + continue; + } + } + for (int index = 0; index < unwrappedKey.length; index++) { payloadKey[index] ^= unwrappedKey[index]; } + foundSplits.put(ss.splitID, true); if (keyAccess.encryptedMetadata != null && !keyAccess.encryptedMetadata.isEmpty()) { AesGcm aesGcm = new AesGcm(unwrappedKey); @@ -528,6 +553,22 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC unencryptedMetadata = new String(decrypted, StandardCharsets.UTF_8); } } + + if (mixedSplits && knownSplits.size() > foundSplits.size()) { + List exceptionList = new ArrayList<>(skippedSplits.size() + 1); + exceptionList.add(new Exception("splitKey.unable to reconstruct split key: " + skippedSplits)); + + for (Map.Entry entry : skippedSplits.entrySet()) { + exceptionList.add(entry.getValue()); + } + + StringBuilder combinedMessage = new StringBuilder(); + for (Exception e : exceptionList) { + combinedMessage.append(e.getMessage()).append("\n"); + } + + throw new Exception(combinedMessage.toString()); + } } // Validate root signature From ec5e91bcdeb44048045b02106356f0e631b3e576 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:49:47 -0400 Subject: [PATCH 03/16] update cli exception handling --- cmdline/src/main/java/io/opentdf/platform/Command.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 8303d22c..31324803 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -84,7 +84,7 @@ private SDK buildSDK() { void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException, TDF.FailedToCreateGMAC, - JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { + JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { try (var stdout = new BufferedOutputStream(System.out)) { @@ -95,7 +95,7 @@ void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) th } @CommandLine.Command(name = "metadata") void readMetadata(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, - TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException { + TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, Exception { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { From 484dcaa60d62ef2df68a62388cb3415ba903b020 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 11:56:57 -0400 Subject: [PATCH 04/16] make custom exception type --- cmdline/src/main/java/io/opentdf/platform/Command.java | 6 ++++-- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 31324803..38a832c8 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -3,6 +3,8 @@ import com.nimbusds.jose.JOSEException; import io.opentdf.platform.sdk.*; import io.opentdf.platform.sdk.TDF; +import io.opentdf.platform.sdk.TDF.SplitKeyException; + import org.apache.commons.codec.DecoderException; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -84,7 +86,7 @@ private SDK buildSDK() { void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException, TDF.FailedToCreateGMAC, - JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { + JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { try (var stdout = new BufferedOutputStream(System.out)) { @@ -95,7 +97,7 @@ void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) th } @CommandLine.Command(name = "metadata") void readMetadata(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, - TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, Exception { + TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, SplitKeyException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index f60d19f4..27dfcc65 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -71,6 +71,12 @@ public TDF() { private static final Gson gson = new Gson(); + public class SplitKeyException extends Exception { + public SplitKeyException(String errorMessage) { + super(errorMessage); + } + } + public static class DataSizeNotSupported extends RuntimeException { public DataSizeNotSupported(String errorMessage) { super(errorMessage); @@ -498,7 +504,7 @@ private void fillInPublicKeyInfo(List kasInfoList, SDK.KAS kas) } public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, - IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, Exception { + IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { TDFReader tdfReader = new TDFReader(tdf); String manifestJson = tdfReader.manifest(); @@ -567,7 +573,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC combinedMessage.append(e.getMessage()).append("\n"); } - throw new Exception(combinedMessage.toString()); + throw new SplitKeyException(combinedMessage.toString()); } } From 4552d078a5b1dd328ad72b82f769679444990b23 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 13:15:19 -0400 Subject: [PATCH 05/16] null check --- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 27dfcc65..4642b19f 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -527,7 +527,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); } else { knownSplits.put(ss.splitID, true); - if (foundSplits.get(ss.splitID)){ + if (foundSplits.get(ss.splitID) != null){ continue; } try { From 8b006dd982530e87dc7ee2a7f3d9579abfc1ba76 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Tue, 13 Aug 2024 13:33:33 -0400 Subject: [PATCH 06/16] check for null pointer --- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 4642b19f..21056300 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -515,7 +515,9 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC Map knownSplits = new HashMap<>(); Map foundSplits = new HashMap<>(); Map skippedSplits = new HashMap<>(); - boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && !manifest.encryptionInformation.keyAccessObj.get(0).sid.isEmpty(); + boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && + (manifest.encryptionInformation.keyAccessObj.get(0).sid != null) && + !manifest.encryptionInformation.keyAccessObj.get(0).sid.isEmpty(); MessageDigest digest = MessageDigest.getInstance("SHA-256"); From fb889a3d88ebdab8b89468fc85b966092d84c0d2 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Wed, 14 Aug 2024 13:05:02 -0400 Subject: [PATCH 07/16] encrypt w key splits --- .github/workflows/checks.yaml | 55 ++++++++ .../java/io/opentdf/platform/Command.java | 2 +- .../java/io/opentdf/platform/sdk/Config.java | 2 + .../java/io/opentdf/platform/sdk/TDF.java | 119 ++++++++++++------ 4 files changed, 137 insertions(+), 41 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index d3f909ff..63459e6c 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -190,6 +190,61 @@ jobs: fi working-directory: cmdline + - name: Start second in background + uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 + with: + run: | + opentdf-beta.yaml yq e ' + (.server.port = 8282) + | (.mode = ["kas"]) + | (.sdk_config = {"endpoint":"http://localhost:8080","plaintext":true,"client_id":"opentdf","client_secret":"secret"}) + ' + && .github/scripts/watch.sh opentdf-beta.yaml ./opentdf --config-file ./opentdf-beta.yaml start + wait-on: | + tcp:localhost:8282 + log-output-if: true + wait-for: 90s + working-directory: platform + - name: Make sure that the second platform is up + run: | + grpcurl -plaintext localhost:8282 list && \ + grpcurl -plaintext localhost:8282 kas.AccessService/PublicKey + - name: Validate multikas through the command line interface + run: | + printf 'here is some data to encrypt' > data + + java -jar target/cmdline.jar \ + --client-id=opentdf-sdk \ + --client-secret=secret \ + --platform-endpoint=localhost:8080 \ + -i \ + encrypt --kas-url=localhost:8080,localhost:8282 -f data -m 'here is some metadata' > test.tdf + + java -jar target/cmdline.jar \ + --client-id=opentdf-sdk \ + --client-secret=secret \ + --platform-endpoint=localhost:8080 \ + -i \ + decrypt -f test.tdf > decrypted + + java -jar target/cmdline.jar \ + --client-id=opentdf-sdk \ + --client-secret=secret \ + --platform-endpoint=localhost:8080 \ + -i \ + metadata -f test.tdf > metadata + + if ! diff -q data decrypted; then + printf 'decrypted data is incorrect [%s]' "$(< decrypted)" + exit 1 + fi + + if [ "$(< metadata)" != 'here is some metadata' ]; then + printf 'metadata is incorrect [%s]\n' "$(< metadata)" + exit 1 + fi + working-directory: cmdline + platform-xtest: permissions: contents: read diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 38a832c8..f0874360 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -51,7 +51,7 @@ class Command { @CommandLine.Command(name = "encrypt") void encrypt( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, - @Option(names = {"-k", "--kas-url"}, required = true) List kas, + @Option(names = {"-k", "--kas-url"}, required = true, split = ",") List kas, @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata) throws IOException, JOSEException { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java index 875bf71d..cc276196 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Config.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Config.java @@ -77,6 +77,7 @@ public static class TDFConfig { public List assertionList; public AssertionConfig assertionConfig; public String mimeType; + public List splitPlan; public TDFConfig() { this.defaultSegmentSize = DEFAULT_SEGMENT_SIZE; @@ -88,6 +89,7 @@ public TDFConfig() { this.kasInfoList = new ArrayList<>(); this.assertionList = new ArrayList<>(); this.mimeType = DEFAULT_MIME_TYPE; + this.splitPlan = new ArrayList<>(); } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 21056300..bef542e4 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -14,8 +14,6 @@ import com.nimbusds.jwt.JWTClaimsSet; import com.nimbusds.jwt.SignedJWT; -import io.opentdf.platform.sdk.Config.SplitStep; - import com.nimbusds.jose.crypto.RSASSASigner; import com.nimbusds.jose.crypto.MACSigner; import org.apache.commons.codec.DecoderException; @@ -173,57 +171,111 @@ PolicyObject createPolicyObject(List attributes) { } private static final Base64.Encoder encoder = Base64.getEncoder(); - private void prepareManifest(Config.TDFConfig tdfConfig) { + private void prepareManifest(Config.TDFConfig tdfConfig, SDK.KAS kas) { manifest.encryptionInformation.keyAccessType = kSplitKeyType; manifest.encryptionInformation.keyAccessObj = new ArrayList<>(); PolicyObject policyObject = createPolicyObject(tdfConfig.attributes); String base64PolicyObject = encoder.encodeToString(gson.toJson(policyObject).getBytes(StandardCharsets.UTF_8)); List symKeys = new ArrayList<>(); + Map latestKASInfo = new HashMap<>(); + if (tdfConfig.splitPlan.isEmpty()) { + // Default split plan: Split keys across all KASes + List splitPlan = new ArrayList<>(tdfConfig.kasInfoList.size()); + for (Config.KASInfo kasInfo : tdfConfig.kasInfoList) { + Config.SplitStep step = new Config.SplitStep(kasInfo.URL, ""); + if (tdfConfig.kasInfoList.size() > 1) { + step.splitID = String.format("s-%d", splitPlan.size()); + } + splitPlan.add(step); + if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isEmpty()) { + latestKASInfo.put(kasInfo.URL, kasInfo); + } + } + tdfConfig.splitPlan = splitPlan; + } + // Seed anything passed in manually for (Config.KASInfo kasInfo: tdfConfig.kasInfoList) { - if (kasInfo.PublicKey == null || kasInfo.PublicKey.isEmpty()) { - throw new KasPublicKeyMissing("Kas public key is missing in kas information list"); + if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isEmpty()) { + latestKASInfo.put(kasInfo.URL, kasInfo); + } + } + + // split plan: restructure by conjunctions + Map> conjunction = new HashMap<>(); + List splitIDs = new ArrayList<>(); + + for (Config.SplitStep splitInfo : tdfConfig.splitPlan) { + // Public key was passed in with kasInfoList + // TODO First look up in attribute information / add to split plan? + Config.KASInfo ki = latestKASInfo.get(splitInfo.kas); + if (ki == null || ki.PublicKey == null || ki.PublicKey.isBlank()) { + logger.info("no public key provided for KAS at {}, retrieving", splitInfo.kas); + var getKI = new Config.KASInfo(); + getKI.URL = splitInfo.kas; + getKI.PublicKey = kas.getPublicKey(getKI); + getKI.KID = kas.getKid(getKI); + latestKASInfo.put(splitInfo.kas, getKI); + ki = getKI; + } + if (conjunction.containsKey(splitInfo.splitID)) { + conjunction.get(splitInfo.splitID).add(ki); + } else { + List newList = new ArrayList<>(); + newList.add(ki); + conjunction.put(splitInfo.splitID, newList); + splitIDs.add(splitInfo.splitID); } + } + for (String splitID: splitIDs) { // Symmetric key byte[] symKey = new byte[GCM_KEY_SIZE]; sRandom.nextBytes(symKey); - - Manifest.KeyAccess keyAccess = new Manifest.KeyAccess(); - keyAccess.keyType = kWrapped; - keyAccess.url = kasInfo.URL; - keyAccess.kid = kasInfo.KID; - keyAccess.protocol = kKasProtocol; + symKeys.add(symKey); // Add policyBinding var hexBinding = Hex.encodeHexString(CryptoUtils.CalculateSHA256Hmac(symKey, base64PolicyObject.getBytes(StandardCharsets.UTF_8))); var policyBinding = new Manifest.PolicyBinding(); policyBinding.alg = kHmacIntegrityAlgorithm; policyBinding.hash = encoder.encodeToString(hexBinding.getBytes(StandardCharsets.UTF_8)); - keyAccess.policyBinding = policyBinding; - - // Wrap the key with kas public key - AsymEncryption asymmetricEncrypt = new AsymEncryption(kasInfo.PublicKey); - byte[] wrappedKey = asymmetricEncrypt.encrypt(symKey); - - keyAccess.wrappedKey = encoder.encodeToString(wrappedKey); + // Add meta data + var encryptedMetadata = new String(); if(tdfConfig.metaData != null && !tdfConfig.metaData.trim().isEmpty()) { AesGcm aesGcm = new AesGcm(symKey); var encrypted = aesGcm.encrypt(tdfConfig.metaData.getBytes(StandardCharsets.UTF_8)); - EncryptedMetadata encryptedMetadata = new EncryptedMetadata(); - encryptedMetadata.iv = encoder.encodeToString(encrypted.getIv()); - encryptedMetadata.ciphertext = encoder.encodeToString(encrypted.asBytes()); + EncryptedMetadata em = new EncryptedMetadata(); + em.iv = encoder.encodeToString(encrypted.getIv()); + em.ciphertext = encoder.encodeToString(encrypted.asBytes()); - var metadata = gson.toJson(encryptedMetadata); - keyAccess.encryptedMetadata = encoder.encodeToString(metadata.getBytes(StandardCharsets.UTF_8)); + var metadata = gson.toJson(em); + encryptedMetadata = encoder.encodeToString(metadata.getBytes(StandardCharsets.UTF_8)); } - symKeys.add(symKey); - manifest.encryptionInformation.keyAccessObj.add(keyAccess); + for (Config.KASInfo kasInfo: conjunction.get(splitID)){ + if (kasInfo.PublicKey == null || kasInfo.PublicKey.isEmpty()) { + throw new KasPublicKeyMissing("Kas public key is missing in kas information list"); + } + + // Wrap the key with kas public key + AsymEncryption asymmetricEncrypt = new AsymEncryption(kasInfo.PublicKey); + byte[] wrappedKey = asymmetricEncrypt.encrypt(symKey); + + Manifest.KeyAccess keyAccess = new Manifest.KeyAccess(); + keyAccess.keyType = kWrapped; + keyAccess.url = kasInfo.URL; + keyAccess.kid = kasInfo.KID; + keyAccess.protocol = kKasProtocol; + keyAccess.policyBinding = policyBinding; + keyAccess.wrappedKey = encoder.encodeToString(wrappedKey); + keyAccess.encryptedMetadata = encryptedMetadata; + + manifest.encryptionInformation.keyAccessObj.add(keyAccess); + } } manifest.encryptionInformation.policy = base64PolicyObject; @@ -328,10 +380,8 @@ public TDFObject createTDF(InputStream payload, throw new KasInfoMissing("kas information is missing"); } - fillInPublicKeyInfo(tdfConfig.kasInfoList, kas); - TDFObject tdfObject = new TDFObject(); - tdfObject.prepareManifest(tdfConfig); + tdfObject.prepareManifest(tdfConfig, kas); long encryptedSegmentSize = tdfConfig.defaultSegmentSize + kGcmIvSize + kAesBlockSize; TDFWriter tdfWriter = new TDFWriter(outputStream); @@ -492,17 +542,6 @@ public TDFObject createTDF(InputStream payload, return tdfObject; } - private void fillInPublicKeyInfo(List kasInfoList, SDK.KAS kas) { - for (var kasInfo: kasInfoList) { - if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isBlank()) { - continue; - } - logger.info("no public key provided for KAS at {}, retrieving", kasInfo.URL); - kasInfo.PublicKey = kas.getPublicKey(kasInfo); - kasInfo.KID = kas.getKid(kasInfo); - } - } - public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { @@ -566,7 +605,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC List exceptionList = new ArrayList<>(skippedSplits.size() + 1); exceptionList.add(new Exception("splitKey.unable to reconstruct split key: " + skippedSplits)); - for (Map.Entry entry : skippedSplits.entrySet()) { + for (Map.Entry entry : skippedSplits.entrySet()) { exceptionList.add(entry.getValue()); } From 03fae78dfe34125a8b49096604cca33b27461a4f Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Wed, 14 Aug 2024 13:26:14 -0400 Subject: [PATCH 08/16] try to fix test, exception fix, sets --- .github/workflows/checks.yaml | 6 +++--- .../src/main/java/io/opentdf/platform/Command.java | 5 ++--- sdk/src/main/java/io/opentdf/platform/sdk/TDF.java | 14 +++++++------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 63459e6c..cf8f237f 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -190,10 +190,10 @@ jobs: fi working-directory: cmdline - - name: Start second in background - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 + - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 + name: start another KAS server in background with: - run: | + run: > opentdf-beta.yaml yq e ' (.server.port = 8282) | (.mode = ["kas"]) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index f0874360..ddb9d61f 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -3,7 +3,6 @@ import com.nimbusds.jose.JOSEException; import io.opentdf.platform.sdk.*; import io.opentdf.platform.sdk.TDF; -import io.opentdf.platform.sdk.TDF.SplitKeyException; import org.apache.commons.codec.DecoderException; import picocli.CommandLine; @@ -86,7 +85,7 @@ private SDK buildSDK() { void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException, TDF.FailedToCreateGMAC, - JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { + JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { try (var stdout = new BufferedOutputStream(System.out)) { @@ -97,7 +96,7 @@ void decrypt(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) th } @CommandLine.Command(name = "metadata") void readMetadata(@Option(names = {"-f", "--file"}, required = true) Path tdfPath) throws IOException, - TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException, SplitKeyException { + TDF.FailedToCreateGMAC, JOSEException, NoSuchAlgorithmException, ParseException, DecoderException { var sdk = buildSDK(); try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index bef542e4..1beb902f 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -69,7 +69,7 @@ public TDF() { private static final Gson gson = new Gson(); - public class SplitKeyException extends Exception { + public class SplitKeyException extends IOException { public SplitKeyException(String errorMessage) { super(errorMessage); } @@ -543,7 +543,7 @@ public TDFObject createTDF(InputStream payload, } public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch, - IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException, SplitKeyException { + IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException { TDFReader tdfReader = new TDFReader(tdf); String manifestJson = tdfReader.manifest(); @@ -551,8 +551,8 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; - Map knownSplits = new HashMap<>(); - Map foundSplits = new HashMap<>(); + Set knownSplits = new HashSet(); + Set foundSplits = new HashSet();; Map skippedSplits = new HashMap<>(); boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 && (manifest.encryptionInformation.keyAccessObj.get(0).sid != null) && @@ -567,8 +567,8 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC if (!mixedSplits) { unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy); } else { - knownSplits.put(ss.splitID, true); - if (foundSplits.get(ss.splitID) != null){ + knownSplits.add(unencryptedMetadata); + if (foundSplits.contains(ss.splitID)){ continue; } try { @@ -582,7 +582,7 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC for (int index = 0; index < unwrappedKey.length; index++) { payloadKey[index] ^= unwrappedKey[index]; } - foundSplits.put(ss.splitID, true); + foundSplits.add(ss.splitID); if (keyAccess.encryptedMetadata != null && !keyAccess.encryptedMetadata.isEmpty()) { AesGcm aesGcm = new AesGcm(unwrappedKey); From ee193e3abfcb4ac863a3eeee210b5214aec3c54f Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Wed, 14 Aug 2024 13:35:19 -0400 Subject: [PATCH 09/16] fix test --- .github/workflows/checks.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index cf8f237f..baf65236 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -199,7 +199,7 @@ jobs: | (.mode = ["kas"]) | (.sdk_config = {"endpoint":"http://localhost:8080","plaintext":true,"client_id":"opentdf","client_secret":"secret"}) ' - && .github/scripts/watch.sh opentdf-beta.yaml ./opentdf --config-file ./opentdf-beta.yaml start + && go run ./service --config-file ./opentdf-beta.yaml start wait-on: | tcp:localhost:8282 log-output-if: true @@ -207,7 +207,6 @@ jobs: working-directory: platform - name: Make sure that the second platform is up run: | - grpcurl -plaintext localhost:8282 list && \ grpcurl -plaintext localhost:8282 kas.AccessService/PublicKey - name: Validate multikas through the command line interface run: | From 44a8d963543863bcc7156d351221a0ee7802c8fc Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Fri, 16 Aug 2024 01:12:33 -0400 Subject: [PATCH 10/16] set up attributes client --- .../platform/sdk/AttributesClient.java | 90 +++++++++++++++++++ .../java/io/opentdf/platform/sdk/SDK.java | 16 +++- .../io/opentdf/platform/sdk/SDKBuilder.java | 5 +- 3 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java b/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java new file mode 100644 index 00000000..15a5d33b --- /dev/null +++ b/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java @@ -0,0 +1,90 @@ +package io.opentdf.platform.sdk; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.function.Function; + +import io.grpc.ManagedChannel; +import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsRequest; +import io.opentdf.platform.policy.attributes.AttributesServiceGrpc; +import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsResponse; + +import static java.lang.String.format; + + +public class AttributesClient implements SDK.AttributesService { + + private final ManagedChannel channel; + + /*** + * A client that communicates with KAS + * @param channelFactory A function that produces channels that can be used to communicate + * @param dpopKey + */ + public AttributesClient(ManagedChannel channel) { + this.channel = channel; + } + + + @Override + public synchronized void close() { + var entries = new ArrayList<>(stubs.values()); + stubs.clear(); + for (var entry: entries) { + entry.channel.shutdownNow(); + } + } + + private String normalizeAddress(String urlString) { + URL url; + try { + url = new URL(urlString); + } catch (MalformedURLException e) { + // if there is no protocol then they either gave us + // a correct address or one we don't know how to fix + return urlString; + } + + // otherwise we take the specified port or default + // based on whether the URL uses a scheme that + // implies TLS + int port; + if (url.getPort() == -1) { + if ("http".equals(url.getProtocol())) { + port = 80; + } else { + port = 443; + } + } else { + port = url.getPort(); + } + + return format("%s:%d", url.getHost(), port); + } + + + private final HashMap stubs = new HashMap<>(); + private static class CacheEntry { + final ManagedChannel channel; + final AttributesServiceGrpc.AttributesServiceBlockingStub stub; + private CacheEntry(ManagedChannel channel, AttributesServiceGrpc.AttributesServiceBlockingStub stub) { + this.channel = channel; + this.stub = stub; + } + } + + // make this protected so we can test the address normalization logic + synchronized AttributesServiceGrpc.AttributesServiceBlockingStub getStub() { + return AttributesServiceGrpc.newBlockingStub(channel); + } + + + @Override + public GetAttributeValuesByFqnsResponse getAttributeValuesByFqn(GetAttributeValuesByFqnsRequest request) { + return getStub().getAttributeValuesByFqns(request); + } + +} diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index df94dc59..4e32b1fb 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -5,6 +5,7 @@ import io.opentdf.platform.authorization.AuthorizationServiceGrpc; import io.opentdf.platform.authorization.AuthorizationServiceGrpc.AuthorizationServiceFutureStub; import io.opentdf.platform.policy.attributes.AttributesServiceGrpc; +import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsRequest; import io.opentdf.platform.policy.attributes.AttributesServiceGrpc.AttributesServiceFutureStub; import io.opentdf.platform.policy.namespaces.NamespaceServiceGrpc; import io.opentdf.platform.policy.namespaces.NamespaceServiceGrpc.NamespaceServiceFutureStub; @@ -13,8 +14,11 @@ import io.opentdf.platform.policy.subjectmapping.SubjectMappingServiceGrpc; import io.opentdf.platform.policy.subjectmapping.SubjectMappingServiceGrpc.SubjectMappingServiceFutureStub; import io.opentdf.platform.sdk.nanotdf.NanoTDFType; +import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsResponse; import javax.net.ssl.TrustManager; + +import java.util.List; import java.util.Optional; /** @@ -39,17 +43,20 @@ public interface KAS extends AutoCloseable { byte[] unwrapNanoTDF(NanoTDFType.ECCurve curve, String header, String kasURL); } + public interface AttributesService extends AutoCloseable { + GetAttributeValuesByFqnsResponse getAttributeValuesByFqn(GetAttributeValuesByFqnsRequest request); + } + // TODO: add KAS public interface Services extends AutoCloseable { AuthorizationServiceFutureStub authorization(); - AttributesServiceFutureStub attributes(); + AttributesService attributes(); NamespaceServiceFutureStub namespaces(); SubjectMappingServiceFutureStub subjectMappings(); ResourceMappingServiceFutureStub resourceMappings(); KAS kas(); - static Services newServices(ManagedChannel channel, KAS kas) { - var attributeService = AttributesServiceGrpc.newFutureStub(channel); + static Services newServices(ManagedChannel channel, KAS kas, AttributesService attributeService) { var namespaceService = NamespaceServiceGrpc.newFutureStub(channel); var subjectMappingService = SubjectMappingServiceGrpc.newFutureStub(channel); var resourceMappingService = ResourceMappingServiceGrpc.newFutureStub(channel); @@ -59,11 +66,12 @@ static Services newServices(ManagedChannel channel, KAS kas) { @Override public void close() throws Exception { channel.shutdownNow(); + attributeService.close(); kas.close(); } @Override - public AttributesServiceFutureStub attributes() { + public AttributesService attributes() { return attributeService; } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java index a29a0040..d7c1fde5 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java @@ -202,11 +202,12 @@ ServicesAndInternals buildServices() { channel = getManagedChannelBuilder(platformEndpoint).intercept(authInterceptor).build(); managedChannelFactory = (String endpoint) -> getManagedChannelBuilder(endpoint).intercept(authInterceptor).build(); } - var client = new KASClient(managedChannelFactory, dpopKey); + var kasclient = new KASClient(managedChannelFactory, dpopKey); + var attrclient = new AttributesClient(channel); return new ServicesAndInternals( authInterceptor, sslFactory == null ? null : sslFactory.getTrustManager().orElse(null), - SDK.Services.newServices(channel, client) + SDK.Services.newServices(channel, kasclient, attrclient) ); } From f5deb9bcaac8154fc91e60b68f9275d1dc43ddbb Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Mon, 19 Aug 2024 12:32:30 -0400 Subject: [PATCH 11/16] add test for attribute client --- .../platform/sdk/AttributesClient.java | 1 + .../io/opentdf/platform/sdk/SDKBuilder.java | 5 +- .../platform/sdk/AttributeClientTest.java | 84 +++++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java b/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java index 15a5d33b..91b7a0ae 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java @@ -36,6 +36,7 @@ public synchronized void close() { for (var entry: entries) { entry.channel.shutdownNow(); } + this.channel.shutdownNow(); } private String normalizeAddress(String urlString) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java index 0dde61fa..5caf0daa 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java @@ -193,17 +193,20 @@ ServicesAndInternals buildServices() { var authInterceptor = getGrpcAuthInterceptor(dpopKey); ManagedChannel channel; + ManagedChannel attributesChannel; Function managedChannelFactory; if (authInterceptor == null) { channel = getManagedChannelBuilder(platformEndpoint).build(); + attributesChannel = getManagedChannelBuilder(platformEndpoint).build(); managedChannelFactory = (String endpoint) -> getManagedChannelBuilder(endpoint).build(); } else { channel = getManagedChannelBuilder(platformEndpoint).intercept(authInterceptor).build(); + attributesChannel = getManagedChannelBuilder(platformEndpoint).intercept(authInterceptor).build(); managedChannelFactory = (String endpoint) -> getManagedChannelBuilder(endpoint).intercept(authInterceptor).build(); } var kasclient = new KASClient(managedChannelFactory, dpopKey); - var attrclient = new AttributesClient(channel); + var attrclient = new AttributesClient(attributesChannel); return new ServicesAndInternals( authInterceptor, sslFactory == null ? null : sslFactory.getTrustManager().orElse(null), diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java new file mode 100644 index 00000000..4083e19a --- /dev/null +++ b/sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java @@ -0,0 +1,84 @@ +package io.opentdf.platform.sdk; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.opentdf.platform.policy.attributes.AttributesServiceGrpc; +import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsRequest; +import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsResponse; +import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsResponse.AttributeAndValue; +import io.opentdf.platform.policy.Attribute; +import io.opentdf.platform.policy.Namespace; +import io.opentdf.platform.policy.Value; +import io.opentdf.platform.policy.AttributeRuleTypeEnum; + +import static io.opentdf.platform.sdk.SDKBuilderTest.getRandomPort; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + + +public class AttributeClientTest { + @Test + void testGettingAttributeByFqn() throws IOException { + AttributesServiceGrpc.AttributesServiceImplBase attributesService = new AttributesServiceGrpc.AttributesServiceImplBase() { + @Override + public void getAttributeValuesByFqns(GetAttributeValuesByFqnsRequest request, + io.grpc.stub.StreamObserver responseObserver) { + Attribute attribute1 = Attribute.newBuilder().setId("CLS").setNamespace( + Namespace.newBuilder().setId("v").setName("virtru.com").setFqn("https://virtru.com").build()) + .setName("Classification").setRule(AttributeRuleTypeEnum.ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY).setFqn("https://virtru.com/attr/classification").build(); + + Value attributeValue1 = Value.newBuilder() + .setValue("value1") + .build(); + + // Create a sample AttributeValues object + AttributeAndValue attributeAndValues = AttributeAndValue.newBuilder().setAttribute(attribute1) + .setValue(attributeValue1) + .build(); + GetAttributeValuesByFqnsResponse response = GetAttributeValuesByFqnsResponse.newBuilder() + .putFqnAttributeValues("https://virtru.com/attr/classification/value/value1",attributeAndValues) + .build(); + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + }; + + Server attrServer = null; + try { + attrServer = startServer(attributesService); + String attrServerUrl = "localhost:" + attrServer.getPort(); + ManagedChannel channel = ManagedChannelBuilder + .forTarget(attrServerUrl) + .usePlaintext() + .build(); + try (var attr = new AttributesClient(channel)) { + GetAttributeValuesByFqnsResponse resp = attr.getAttributeValuesByFqn(GetAttributeValuesByFqnsRequest.newBuilder().build()); + Set fqnSet = new HashSet<>(Arrays.asList("https://virtru.com/attr/classification/value/value1")); + assertThat(resp.getFqnAttributeValuesMap().keySet()).isEqualTo(fqnSet); + assertThat(resp.getFqnAttributeValuesCount()).isEqualTo(1); + } + } finally { + if (attrServer != null) { + attrServer.shutdownNow(); + } + } + } + private static Server startServer(AttributesServiceGrpc.AttributesServiceImplBase attrService) throws IOException { + return ServerBuilder + .forPort(getRandomPort()) + .directExecutor() + .addService(attrService) + .build() + .start(); + } + +} From 5b222b26548a15869a5e90c846a5ecb3f36ed20b Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Mon, 19 Aug 2024 12:51:35 -0400 Subject: [PATCH 12/16] suggested changes --- .../platform/sdk/AttributesClient.java | 51 ------------------- .../platform/sdk/AttributeClientTest.java | 16 +++--- 2 files changed, 8 insertions(+), 59 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java b/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java index 91b7a0ae..85be7b60 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/AttributesClient.java @@ -1,19 +1,10 @@ package io.opentdf.platform.sdk; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.function.Function; - import io.grpc.ManagedChannel; import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsRequest; import io.opentdf.platform.policy.attributes.AttributesServiceGrpc; import io.opentdf.platform.policy.attributes.GetAttributeValuesByFqnsResponse; -import static java.lang.String.format; - public class AttributesClient implements SDK.AttributesService { @@ -31,51 +22,9 @@ public AttributesClient(ManagedChannel channel) { @Override public synchronized void close() { - var entries = new ArrayList<>(stubs.values()); - stubs.clear(); - for (var entry: entries) { - entry.channel.shutdownNow(); - } this.channel.shutdownNow(); } - private String normalizeAddress(String urlString) { - URL url; - try { - url = new URL(urlString); - } catch (MalformedURLException e) { - // if there is no protocol then they either gave us - // a correct address or one we don't know how to fix - return urlString; - } - - // otherwise we take the specified port or default - // based on whether the URL uses a scheme that - // implies TLS - int port; - if (url.getPort() == -1) { - if ("http".equals(url.getProtocol())) { - port = 80; - } else { - port = 443; - } - } else { - port = url.getPort(); - } - - return format("%s:%d", url.getHost(), port); - } - - - private final HashMap stubs = new HashMap<>(); - private static class CacheEntry { - final ManagedChannel channel; - final AttributesServiceGrpc.AttributesServiceBlockingStub stub; - private CacheEntry(ManagedChannel channel, AttributesServiceGrpc.AttributesServiceBlockingStub stub) { - this.channel = channel; - this.stub = stub; - } - } // make this protected so we can test the address normalization logic synchronized AttributesServiceGrpc.AttributesServiceBlockingStub getStub() { diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java index 4083e19a..8dd04c50 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/AttributeClientTest.java @@ -1,12 +1,5 @@ package io.opentdf.platform.sdk; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import org.junit.jupiter.api.Test; - import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.Server; @@ -20,8 +13,15 @@ import io.opentdf.platform.policy.Value; import io.opentdf.platform.policy.AttributeRuleTypeEnum; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.junit.jupiter.api.Test; + import static io.opentdf.platform.sdk.SDKBuilderTest.getRandomPort; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.Assertions.assertThat; public class AttributeClientTest { From 64e8dd7b3c286c5cd33f72d934c18c053214f7aa Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Mon, 19 Aug 2024 13:21:37 -0400 Subject: [PATCH 13/16] add attributes to cli --- .github/workflows/checks.yaml | 4 ++-- .../main/java/io/opentdf/platform/Command.java | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index b24db9e5..d985651e 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -139,7 +139,7 @@ jobs: --client-secret=secret \ --platform-endpoint=localhost:8080 \ -i \ - encrypt --kas-url=localhost:8080 --mime-type=text/plain -f data -m 'here is some metadata' > test.tdf + encrypt --kas-url=localhost:8080 --mime-type=text/plain --attr https://example.com/attr/attr1/value/value1 -f data -m 'here is some metadata' > test.tdf java -jar target/cmdline.jar \ --client-id=opentdf-sdk \ @@ -175,7 +175,7 @@ jobs: --client-secret=secret \ --platform-endpoint=localhost:8080 \ -i \ - encryptnano --kas-url=http://localhost:8080 -f data -m 'here is some metadata' > nano.ntdf + encryptnano --kas-url=http://localhost:8080 --attr https://example.com/attr/attr1/value/value1 -f data -m 'here is some metadata' > nano.ntdf java -jar target/cmdline.jar \ --client-id=opentdf-sdk \ diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 387a72e5..e438d14d 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Optional; import java.util.function.Consumer; +import java.util.stream.Stream; @CommandLine.Command(name = "tdf") class Command { @@ -52,6 +53,7 @@ void encrypt( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, @Option(names = {"-k", "--kas-url"}, required = true, split = ",") List kas, @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata, + @Option(names = {"-a", "--attr"}, split = ",") Optional> attributes, @Option(names = {"--mime-type"}, defaultValue = Option.NULL_VALUE) Optional mimeType) throws IOException, JOSEException { @@ -66,6 +68,11 @@ void encrypt( configs.add(Config.withKasInformation(kasInfos)); metadata.map(Config::withMetaData).ifPresent(configs::add); mimeType.map(Config::withMimeType).ifPresent(configs::add); + attributes.ifPresent(attrList -> { + // Convert List to String[] + String[] attrArray = attrList.toArray(new String[0]); + configs.add(Config.withDataAttributes(attrArray)); + }); var tdfConfig = Config.newTDFConfig(configs.toArray(Consumer[]::new)); try (var in = file.isEmpty() ? new BufferedInputStream(System.in) : new FileInputStream(file.get())) { @@ -113,7 +120,8 @@ void readMetadata(@Option(names = {"-f", "--file"}, required = true) Path tdfPat void createNanoTDF( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, @Option(names = {"-k", "--kas-url"}, required = true) List kas, - @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata) throws Exception { + @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata, + @Option(names = {"-a", "--attr"}, split = ",") Optional> attributes) throws Exception { var sdk = buildSDK(); var kasInfos = kas.stream().map(k -> { @@ -124,6 +132,11 @@ void createNanoTDF( List> configs = new ArrayList<>(); configs.add(Config.withNanoKasInformation(kasInfos)); + attributes.ifPresent(attrList -> { + // Convert List to String[] + String[] attrArray = attrList.toArray(new String[0]); + configs.add(Config.witDataAttributes(attrArray)); + }); var nanoTDFConfig = Config.newNanoTDFConfig(configs.toArray(Consumer[]::new)); try (var in = file.isEmpty() ? new BufferedInputStream(System.in) : new FileInputStream(file.get())) { From 7d8a34795769355e5ce94916e4c362c937505e91 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Mon, 19 Aug 2024 13:34:51 -0400 Subject: [PATCH 14/16] fix optional attr cli --- cmdline/src/main/java/io/opentdf/platform/Command.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index e438d14d..6f347cf3 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -53,7 +53,7 @@ void encrypt( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, @Option(names = {"-k", "--kas-url"}, required = true, split = ",") List kas, @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata, - @Option(names = {"-a", "--attr"}, split = ",") Optional> attributes, + @Option(names = {"-a", "--attr"}, split = ",", defaultValue = Option.NULL_VALUE) Optional> attributes, @Option(names = {"--mime-type"}, defaultValue = Option.NULL_VALUE) Optional mimeType) throws IOException, JOSEException { @@ -121,7 +121,7 @@ void createNanoTDF( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, @Option(names = {"-k", "--kas-url"}, required = true) List kas, @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata, - @Option(names = {"-a", "--attr"}, split = ",") Optional> attributes) throws Exception { + @Option(names = {"-a", "--attr"}, split = ",", defaultValue = Option.NULL_VALUE) Optional> attributes) throws Exception { var sdk = buildSDK(); var kasInfos = kas.stream().map(k -> { From 19ebb47948417e24d356b62538c4f2f34a3c6993 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Mon, 19 Aug 2024 13:51:57 -0400 Subject: [PATCH 15/16] fix attr splitting cmd --- .../main/java/io/opentdf/platform/Command.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index 6f347cf3..f3d9eb3c 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -53,7 +53,8 @@ void encrypt( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, @Option(names = {"-k", "--kas-url"}, required = true, split = ",") List kas, @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata, - @Option(names = {"-a", "--attr"}, split = ",", defaultValue = Option.NULL_VALUE) Optional> attributes, + // cant split on optional parameters + @Option(names = {"-a", "--attr"}, defaultValue = Option.NULL_VALUE) Optional attributes, @Option(names = {"--mime-type"}, defaultValue = Option.NULL_VALUE) Optional mimeType) throws IOException, JOSEException { @@ -68,9 +69,8 @@ void encrypt( configs.add(Config.withKasInformation(kasInfos)); metadata.map(Config::withMetaData).ifPresent(configs::add); mimeType.map(Config::withMimeType).ifPresent(configs::add); - attributes.ifPresent(attrList -> { - // Convert List to String[] - String[] attrArray = attrList.toArray(new String[0]); + attributes.ifPresent(attr -> { + String[] attrArray = attr.split(","); configs.add(Config.withDataAttributes(attrArray)); }); @@ -121,7 +121,7 @@ void createNanoTDF( @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional file, @Option(names = {"-k", "--kas-url"}, required = true) List kas, @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional metadata, - @Option(names = {"-a", "--attr"}, split = ",", defaultValue = Option.NULL_VALUE) Optional> attributes) throws Exception { + @Option(names = {"-a", "--attr"}, defaultValue = Option.NULL_VALUE) Optional attributes) throws Exception { var sdk = buildSDK(); var kasInfos = kas.stream().map(k -> { @@ -132,9 +132,8 @@ void createNanoTDF( List> configs = new ArrayList<>(); configs.add(Config.withNanoKasInformation(kasInfos)); - attributes.ifPresent(attrList -> { - // Convert List to String[] - String[] attrArray = attrList.toArray(new String[0]); + attributes.ifPresent(attr -> { + String[] attrArray = attr.split(","); configs.add(Config.witDataAttributes(attrArray)); }); From d4b41274b14142765b029725c3bb79cf64748f04 Mon Sep 17 00:00:00 2001 From: Elizabeth Healy Date: Mon, 19 Aug 2024 14:04:02 -0400 Subject: [PATCH 16/16] remove extra lines --- cmdline/src/main/java/io/opentdf/platform/Command.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index f3d9eb3c..817c2095 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -70,8 +70,7 @@ void encrypt( metadata.map(Config::withMetaData).ifPresent(configs::add); mimeType.map(Config::withMimeType).ifPresent(configs::add); attributes.ifPresent(attr -> { - String[] attrArray = attr.split(","); - configs.add(Config.withDataAttributes(attrArray)); + configs.add(Config.withDataAttributes(attr.split(","))); }); var tdfConfig = Config.newTDFConfig(configs.toArray(Consumer[]::new)); @@ -133,8 +132,7 @@ void createNanoTDF( List> configs = new ArrayList<>(); configs.add(Config.withNanoKasInformation(kasInfos)); attributes.ifPresent(attr -> { - String[] attrArray = attr.split(","); - configs.add(Config.witDataAttributes(attrArray)); + configs.add(Config.witDataAttributes(attr.split(","))); }); var nanoTDFConfig = Config.newNanoTDFConfig(configs.toArray(Consumer[]::new));