Added update example to repository and added support for change streams

java-sample/README.md

@@ -37,3 +37,67 @@ The project will expose a REST api located @ ``` /customer ``` by which you can
    - This could be a great example of where you can replace the $or with a $search.  Ultimately thats why this exists with an aggregation instead of a traditional fine 
 
 
+# Queryable Encryption
+
+This has been extended to add in support for Queryable Encryption
+
+This happens by creating a new model ```Patient``` that uses Queryable Encryption to protect a couple fields. 
+Internally we do this by having the MongoDBConnection class create 2 different mongoClients beans to be injected
+
+```mongoClient``` is a standard Java connection with the default PojoCodec configured
+
+```mongoSecureClient``` is a mongo client with Queryable Encryption configured. All work to configure encryption is done in the creation for this bean.
+
+To use the queryable encryption portion, you need to do a couple things. Most of these are borrowed from our QuickStart here https://www.mongodb.com/docs/manual/core/queryable-encryption/quick-start/#std-label-qe-quick-start
+
+
+- [Create a local master key](https://www.mongodb.com/docs/manual/core/queryable-encryption/quick-start/#create-your-encrypted-collection)
+- Update the application.properties file with appropriate values
+  - Note that the ```Patient``` model annotation will need to be updated to match the value of the collectionName field if changed from the default
+- [Download the encryption library](https://www.mongodb.com/docs/manual/core/queryable-encryption/reference/shared-library/)
+- Ensure your user has permissions to drop and create databases
+  -  For simplicity sake the app drops and re-builds the Db and the keys on each run. I might update it to check for those before hand in the future, but thats for another time
+
+Once the app runs, it will create 2 collections
+
+```patientKeys``` - This is where the encryption keys are stored
+```patient``` - this is where patient data is stored (unless you changed the collection names)
+
+
+There are 2 key API endpoints that also are available. 
+
+### POST /patient
+
+Send a body with payload like 
+```angular2html
+{
+    "name":"Josh Smith",
+    "ssn": "987654321",
+    "medicalRecords": [
+        {
+        "weight": 190,
+        "bloodPressure":"120/80"
+        },
+        {
+        "weight": 185,
+        "bloodPressure":"130/90"
+        }
+    ]
+}
+```
+This will create a patient records with encryption on the medicalRecords and SSN fields. SSN will be a queryable field and medicalRecords will not
+
+
+### GET /patient?ssn=xxxxxx
+
+A GET call to this endpoint, using the same SSN provided in the POST will show that you can search for a record by SSN even though it's encrypted in the DB. 
+
+
+
+
+
+
+
+
+
+

java-sample/pom.xml

@@ -30,19 +30,27 @@
         <dependency>
             <groupId>org.mongodb</groupId>
             <artifactId>mongodb-driver-sync</artifactId>
-            <version>4.9.0</version>
-            <!-- This shouldn't be needed but my local kept trying to use the wrong version of the core library -->
-            <exclusions>
-                <exclusion>
-                    <groupId>org.mongodb</groupId>
-                    <artifactId>mongodb-driver-core</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>5.0.0</version>
         </dependency>
         <dependency>
             <groupId>org.mongodb</groupId>
             <artifactId>mongodb-driver-core</artifactId>
-            <version>4.9.0</version>
+            <version>5.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>mongodb-crypt</artifactId>
+            <version>1.8.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>bson</artifactId>
+            <version>5.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>bson-record-codec</artifactId>
+            <version>5.0.0</version>
         </dependency>
     </dependencies>
 

java-sample/src/main/java/com/mongodb/ms0/example/javasample/config/MongoDBConnection.java

@@ -1,44 +1,221 @@
 package com.mongodb.ms0.example.javasample.config;
 
+import com.mongodb.AutoEncryptionSettings;
+import com.mongodb.ClientEncryptionSettings;
 import com.mongodb.ConnectionString;
 import com.mongodb.MongoClientSettings;
 import com.mongodb.client.MongoClient;
 import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.CreateCollectionOptions;
+import com.mongodb.client.model.CreateEncryptedCollectionParams;
+import com.mongodb.client.model.vault.DataKeyOptions;
+import com.mongodb.client.vault.ClientEncryption;
+import com.mongodb.client.vault.ClientEncryptions;
+import com.mongodb.ms0.example.javasample.models.Customer;
+import com.mongodb.ms0.example.javasample.models.Patient;
+import org.bson.*;
 import org.bson.codecs.configuration.CodecProvider;
 import org.bson.codecs.configuration.CodecRegistry;
+import org.bson.codecs.pojo.ClassModel;
 import org.bson.codecs.pojo.PojoCodecProvider;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.PropertySource;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Service;
 
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.ObjectInputFilter;
+import java.util.*;
+
 import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry;
 import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
 import static org.bson.codecs.configuration.CodecRegistries.fromRegistries;
 
 @Configuration
 @Service
+@PropertySource("classpath:application.properties")
 public class MongoDBConnection {
 
-    private MongoClient client;
 
+    public Map<String, Object> keyMap = new HashMap<>();
+    public Map<String, Map<String, Object>> kmsProviders = new HashMap<>();
+
+    @Value("${encryption.keyCollectionName}")
+    private String keyCollectionName;
+
+    public String base64DEK;
+    private Map<String, BsonDocument> schemaMap = new HashMap<>();
+
+    @Value("${mongo.uri}")
+    private String uri;
+
+    @Value("${encryption.cryptLibSharedPath}")
+    private String cryptLibSharedPath;
+
+    @Value("${encryption.localKeyFile}")
+    private String localKeyFile;
+
+    @Value("${encryption.databaseName}")
+    private String databaseName;
+
+    @Value("${encryption.collectionName}")
+    private String collectionName;
 
-    @Bean
+
+
+
+
+
+
+    @Bean(name="mongoClient")
     @Scope(value= ConfigurableBeanFactory.SCOPE_SINGLETON)
     public MongoClient mongoClient() {
-        String uri = "mongodb://localhost:27017";
+
+        ConnectionString connectionString = new ConnectionString(uri);
+        CodecProvider pojoCodecProvider = PojoCodecProvider.builder().automatic(true).build();
+        CodecRegistry codecRegistry = fromRegistries(getDefaultCodecRegistry(), fromProviders(pojoCodecProvider));
+
+        MongoClientSettings clientSettings = MongoClientSettings.builder()
+                .applyConnectionString(connectionString)
+                .codecRegistry(codecRegistry)
+                .build();
+        // Replace the uri string with your MongoDB deployment's connection string
+        return MongoClients.create(clientSettings);
+    }
+
+
+
+    @Bean(name="mongoSecureClient")
+    @Scope(value= ConfigurableBeanFactory.SCOPE_SINGLETON)
+    public MongoClient mongoSecureClient(){
+
+        String keyVaultNamespace = this.databaseName + "." + this.keyCollectionName;
+        String kmsProvider = "local";
+        String path = this.localKeyFile;
+        byte[] localMasterKeyRead = new byte[96];
+
+
+
+        try (FileInputStream fs = new FileInputStream(path)){
+            if (fs.read(localMasterKeyRead) < 96){
+                throw new RuntimeException("Did not read the expected number of bytes");
+            }
+            keyMap.put("key", localMasterKeyRead);
+            kmsProviders.put("local", keyMap);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        ClientEncryptionSettings clientEncryptionSettings = ClientEncryptionSettings.builder()
+                .keyVaultMongoClientSettings(MongoClientSettings.builder()
+                        .applyConnectionString(new ConnectionString(uri))
+                        .build())
+                .keyVaultNamespace(keyVaultNamespace)
+                .kmsProviders(kmsProviders)
+                .build();
+
+
+
+        /*
+        //Use this block for CSFLE without query support
+
+        ClientEncryption clientEncryption = ClientEncryptions.create(clientEncryptionSettings);
+        BsonBinary dataKeyId = clientEncryption.createDataKey("local", new DataKeyOptions());
+        base64DEK = Base64.getEncoder().encodeToString(dataKeyId.getData());
+
+
+        Document jsonSchema = new Document().append("bsonType", "object").append("encryptMetadata",
+                        new Document().append("keyId", new ArrayList<>((Arrays.asList(new Document().append("$binary", new Document()
+                                .append("base64", base64DEK)
+                                .append("subType", "04")))))))
+                .append("properties", new Document()
+                        .append("ssn", new Document().append("encrypt", new Document()
+                                .append("bsonType", "string")
+                                .append("algorithm", "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic")))
+                        .append("bloodType", new Document().append("encrypt", new Document()
+                                .append("bsonType", "string")
+                                .append("algorithm", "AEAD_AES_256_CBC_HMAC_SHA_512-Random")))
+                        .append("medicalRecords", new Document().append("encrypt", new Document()
+                                .append("bsonType", "array")
+                                .append("algorithm", "AEAD_AES_256_CBC_HMAC_SHA_512-Random"))));
+
+        schemaMap.put("csfle.patients", BsonDocument.parse(jsonSchema.toJson()));
+        */
+
+
+        /*
+
+        Use this block to do queryable encryption
+         */
+
+        BsonDocument queryableEncryptionSchema = new BsonDocument().append("fields",
+                new BsonArray(Arrays.asList(
+                    new BsonDocument()
+                        .append("path", new BsonString("ssn"))
+                        .append("bsonType", new BsonString("string"))
+                        .append("keyId", new BsonNull())
+                        .append("queries", new BsonDocument().append("queryType", new BsonString("equality"))),
+                    new BsonDocument()
+                        .append("path", new BsonString("bloodType"))
+                        .append("keyId", new BsonNull())
+                        .append("bsonType",new BsonString( "string")),
+                    new BsonDocument()
+                        .append("path",new BsonString( "medicalRecords"))
+                        .append("keyId", new BsonNull())
+                )));
+
+
+
+        HashMap<String, BsonDocument> queryableMap = new HashMap<>();
+        queryableMap.put(this.databaseName + "." + this.collectionName, queryableEncryptionSchema);
+
+        Map<String, Object> extraOptions = new HashMap<String, Object>();
+        extraOptions.put("cryptSharedLibPath", this.cryptLibSharedPath);
+
         ConnectionString connectionString = new ConnectionString(uri);
         CodecRegistry pojoCodecRegistry = fromProviders(PojoCodecProvider.builder().automatic(true).build());
         CodecRegistry codecRegistry = fromRegistries(MongoClientSettings.getDefaultCodecRegistry(), pojoCodecRegistry);
         MongoClientSettings clientSettings = MongoClientSettings.builder()
                 .applyConnectionString(connectionString)
                 .codecRegistry(codecRegistry)
+                .autoEncryptionSettings(AutoEncryptionSettings.builder()
+                        .keyVaultNamespace(keyVaultNamespace)
+                        .kmsProviders(kmsProviders)
+                        //.schemaMap(schemaMap)
+
+                        /*
+                        Removing the .encryptedFieldsMap from the config seems to be what actually allows it to work.
+                        But that goes against the documentation.
+                        So we have this field in the autoEncrypt work, but if we use it, it breaks things?
+                         */
+                        //.encryptedFieldsMap(queryableMap)
+
+                        .extraOptions(extraOptions)
+                        .build())
                 .build();
 
-        // Replace the uri string with your MongoDB deployment's connection string
-        MongoClient mongoClient = MongoClients.create(clientSettings);
-        return mongoClient;
+        // This is a bit clunky to do this here, but you have to make sure this is done on the collection first so the DB can create the key indexes.
+        MongoClient client = MongoClients.create(clientSettings);
+
+        MongoDatabase csfleDB = client.getDatabase(this.databaseName);
+        csfleDB.drop();
+        ClientEncryption clientEncryption = ClientEncryptions.create(clientEncryptionSettings);
+
+        CreateEncryptedCollectionParams encryptedCollectionParams = new CreateEncryptedCollectionParams("local");
+        encryptedCollectionParams.masterKey(new BsonDocument());
+        CreateCollectionOptions options = new CreateCollectionOptions();
+        options.encryptedFields(queryableEncryptionSchema);
+        clientEncryption.createEncryptedCollection(client.getDatabase(this.databaseName), this.collectionName, options, encryptedCollectionParams);
+        return client;
+
     }
+
+
+
 }

java-sample/src/main/java/com/mongodb/ms0/example/javasample/controllers/CustomerController.java

@@ -22,6 +22,12 @@ public Customer getCustomerById(@PathVariable("id") String id) {
         return service.getCustomerById(id);
     }
 
+
+    @GetMapping()
+    public List<Customer> getAllCustomers() {
+        return service.getAllCustomers();
+    }
+
     @PostMapping
     public Customer createCustomer(@RequestBody Customer customer) {
         return service.createCustomer(customer);

java-sample/src/main/java/com/mongodb/ms0/example/javasample/controllers/PatientController.java

@@ -0,0 +1,42 @@
+package com.mongodb.ms0.example.javasample.controllers;
+
+import com.mongodb.ms0.example.javasample.models.Patient;
+import com.mongodb.ms0.example.javasample.service.PatientService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping(value="patient", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+public class PatientController {
+
+    @Autowired
+    public PatientService service;
+
+
+    @GetMapping(value="{id}")
+    public Patient getPatientById(@PathVariable("id") String id) {
+        return service.getPatientById(id);
+    }
+
+    @GetMapping()
+    public Patient getPatientBySSN(@RequestParam String ssn) {
+        return service.getPatientBySSN(ssn);
+    }
+
+    @PostMapping
+    public Patient createPatient(@RequestBody Patient Patient) {
+        return service.createPatient(Patient);
+    }
+
+
+    @PostMapping(value="search")
+    public List<Patient> PatientSearch(@RequestBody Map<String, String> values) {
+        return service.PatientSearch(values.get("name"));
+    }
+
+
+}