Encrypt HybridFileBackedSqlStorage with Android Keystore

app/src/org/commcare/activities/FormEntryActivity.java

@@ -1068,7 +1068,7 @@ private void loadForm() {
                 return;
             }
 
-            mFormLoaderTask = new FormLoaderTask<FormEntryActivity>(symetricKey, instanceIsReadOnly,
+            mFormLoaderTask = new FormLoaderTask<FormEntryActivity>(symetricKey, isKeyFromKeystore, instanceIsReadOnly,
                     formEntryRestoreSession.isRecording(), FormEntryInstanceState.mFormRecordPath, this, savedFormSession) {
                 @Override
                 protected void deliverResult(FormEntryActivity receiver, FECWrapper wrapperResult) {

app/src/org/commcare/models/database/HybridFileBackedSqlStorage.java

@@ -8,6 +8,7 @@
 import org.apache.commons.io.FilenameUtils;
 import org.commcare.CommCareApplication;
 import org.commcare.google.services.analytics.CCPerfMonitoring;
+import org.commcare.services.CommCareKeyManager;
 import org.commcare.interfaces.AppFilePathBuilder;
 import org.commcare.models.encryption.EncryptionIO;
 import org.commcare.modern.database.DatabaseHelper;
@@ -165,8 +166,13 @@ private IDatabase getDbOrThrow() {
     }
 
     protected InputStream getInputStreamFromFile(String filename, byte[] aesKeyBytes) throws FileNotFoundException {
-        SecretKeySpec aesKey = new SecretKeySpec(aesKeyBytes, "AES");
-        return EncryptionIO.getFileInputStream(filename, aesKey, null, false);
+        if (usesKeystoreEncryption(aesKeyBytes)) {
+            return EncryptionIO.getFileInputStreamWithKeystore(
+                    filename, CommCareKeyManager.retrieveSessionKeyAndTransformation());
+        } else {
+            SecretKeySpec aesKey = new SecretKeySpec(aesKeyBytes, "AES");
+            return EncryptionIO.getFileInputStream(filename, aesKey, null, false);
+        }
     }
 
     @Override
@@ -325,18 +331,22 @@ protected boolean blobFitsInDb(ByteArrayOutputStream blobStream) {
     }
 
     protected byte[] generateKeyAndAdd(ContentValues contentValues) {
-        byte[] key = CommCareApplication.instance().createNewSymmetricKey().getEncoded();
+        byte[] key = CommCareKeyManager.generateLegacyKeyOrEmpty();
         contentValues.put(DatabaseHelper.AES_COL, key);
         return key;
     }
 
+    protected static boolean usesKeystoreEncryption(byte[] aesKeyBytes) {
+        return aesKeyBytes == null || aesKeyBytes.length == 0;
+    }
+
     private void writeStreamToFile(ByteArrayOutputStream bos, String filename,
-                                   byte[] key) throws IOException {
+                                   byte[] aesKeyBytes) throws IOException {
         Trace trace = CCPerfMonitoring.INSTANCE.startTracing(CCPerfMonitoring.TRACE_FILE_ENCRYPTION_TIME);
 
         DataOutputStream fileOutputStream = null;
         try {
-            fileOutputStream = getOutputFileStream(filename, key);
+            fileOutputStream = getOutputFileStream(filename, aesKeyBytes);
             bos.writeTo(fileOutputStream);
         } finally {
             if (fileOutputStream != null) {
@@ -350,15 +360,20 @@ private void writeStreamToFile(ByteArrayOutputStream bos, String filename,
                     trace,
                     bos.size(),
                     FilenameUtils.getExtension(filename),
-                    false
+                    usesKeystoreEncryption(aesKeyBytes)
             );
         }
     }
 
     protected DataOutputStream getOutputFileStream(String filename,
                                                    byte[] aesKeyBytes) throws IOException {
-        SecretKeySpec aesKey = new SecretKeySpec(aesKeyBytes, "AES");
-        return new DataOutputStream(EncryptionIO.createFileOutputStream(filename, aesKey));
+        if (usesKeystoreEncryption(aesKeyBytes)) {
+            return new DataOutputStream(EncryptionIO.createFileOutputStreamWithKeystore(
+                    filename, CommCareKeyManager.retrieveSessionKeyAndTransformation()));
+        } else {
+            SecretKeySpec aesKey = new SecretKeySpec(aesKeyBytes, "AES");
+            return new DataOutputStream(EncryptionIO.createFileOutputStream(filename, aesKey));
+        }
     }
 
     @Override

app/src/org/commcare/models/encryption/EncryptionIO.java

@@ -29,7 +29,6 @@
 import javax.crypto.CipherOutputStream;
 import javax.crypto.NoSuchPaddingException;
 import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
 
 /**
  * Methods for dealing with encrypted input/output.

app/src/org/commcare/services/CommCareKeyManager.kt

@@ -1,5 +1,6 @@
 package org.commcare.services
 
+import androidx.annotation.VisibleForTesting
 import org.commcare.CommCareApplication
 import org.commcare.android.security.AesKeyStoreHandler
 import org.commcare.android.security.AndroidKeyStore
@@ -17,7 +18,8 @@ class CommCareKeyManager {
         }
 
         @JvmStatic
-        fun retrieveSessionKeyAndTransformation(): EncryptionKeyAndTransform = sessionKeyAndTransformation
+        fun retrieveSessionKeyAndTransformation(): EncryptionKeyAndTransform =
+            testKeyAndTransformation ?: sessionKeyAndTransformation
 
         /**
          * An empty array indicates that the Android Keystore is supported and the key should be retrieved
@@ -30,5 +32,26 @@ class CommCareKeyManager {
             } else {
                 CommCareApplication.instance().createNewSymmetricKey().encoded
             }
+
+        /**
+         * For testing purposes only
+         */
+        @VisibleForTesting
+        private var testKeyAndTransformation: EncryptionKeyAndTransform? = null
+
+        /**
+         * Set a test override for the session key. When set, retrieveSessionKeyAndTransformation()
+         * returns this value instead of going through AesKeyStoreHandler.
+         */
+        @JvmStatic
+        fun setTestKeyAndTransformation(encryptionKeyAndTransform: EncryptionKeyAndTransform?) {
+            testKeyAndTransformation = encryptionKeyAndTransform
+        }
+
+        @JvmStatic
+        fun clearTestKeyAndTransformation() {
+            testKeyAndTransformation = null
+        }
+
     }
 }

app/src/org/commcare/tasks/FormLoaderTask.java

@@ -19,6 +19,7 @@
 import org.commcare.models.database.InterruptedFormState;
 import org.commcare.models.encryption.EncryptionIO;
 import org.commcare.preferences.DeveloperPreferences;
+import org.commcare.services.CommCareKeyManager;
 import org.commcare.tasks.templates.CommCareTask;
 import org.commcare.util.LogTypes;
 import org.commcare.utils.FileUtil;
@@ -67,6 +68,7 @@ public abstract class FormLoaderTask<R> extends CommCareTask<Integer, String, Fo
     public static InstanceInitializationFactory iif;
 
     private final SecretKeySpec mSymetricKey;
+    private final boolean isKeyFromKeystore;
     private final boolean mReadOnly;
     private final boolean recordEntrySession;
     private final InterruptedFormState savedFormSession;
@@ -81,9 +83,10 @@ public abstract class FormLoaderTask<R> extends CommCareTask<Integer, String, Fo
 
     public static final int FORM_LOADER_TASK_ID = 16;
 
-    public FormLoaderTask(SecretKeySpec symetricKey, boolean readOnly,
+    public FormLoaderTask(SecretKeySpec symetricKey, boolean isKeyFromKeystore, boolean readOnly,
                           boolean recordEntrySession, String formRecordPath, R activity, InterruptedFormState savedFormSession) {
         this.mSymetricKey = symetricKey;
+        this.isKeyFromKeystore = isKeyFromKeystore;
         this.mReadOnly = readOnly;
         this.activity = activity;
         this.taskId = FORM_LOADER_TASK_ID;
@@ -260,7 +263,12 @@ private boolean importData(String filePath, FormEntryController fec) {
         // convert files into a byte array
         InputStream is;
         try {
-            is = EncryptionIO.getFileInputStream(filePath, mSymetricKey, null, false);
+            if (isKeyFromKeystore) {
+                is = EncryptionIO.getFileInputStreamWithKeystore(filePath,
+                        CommCareKeyManager.retrieveSessionKeyAndTransformation());
+            } else {
+                is = EncryptionIO.getFileInputStream(filePath, mSymetricKey, null, false);
+            }
         } catch (FileNotFoundException e) {
             e.printStackTrace();
             throw new RuntimeException("Unable to open encrypted form instance file: " + filePath);

app/src/org/commcare/utils/FormUploadUtil.java

@@ -69,29 +69,6 @@ public class FormUploadUtil {
                     ".m4v", ".mpg", ".mpeg", ".qcp", ".ogg", ".pdf", 
                     ".html", ".rtf", ".txt", ".docx", ".xlsx", ".msg"};
 
-    public static Cipher getDecryptCipher(Key key) {
-        return getDecryptCipher(key, null, null);
-    }
-
-    public static Cipher getDecryptCipher(Key key, String transformation, byte[] iv) {
-        Cipher cipher;
-        try {
-            cipher = Cipher.getInstance(Objects.requireNonNullElse(transformation, "AES"));
-            if (iv != null) {
-                cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
-            } else {
-                cipher.init(Cipher.DECRYPT_MODE, key);
-            }
-
-            return cipher;
-            //TODO: Something smart here;
-        } catch (NoSuchAlgorithmException |
-                NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) {
-            Logger.exception("Failed to initialize decryption cipher ", e);
-        }
-        return null;
-    }
-
     /**
      * Send unencrypted data to the server without user facing progress
      * reporting.

app/unit-tests/src/org/commcare/models/database/HybridFileBackedSqlStorageKeystoreTest.java

@@ -0,0 +1,132 @@
+package org.commcare.models.database;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+
+import org.commcare.CommCareApplication;
+import org.commcare.CommCareTestApplication;
+import org.commcare.services.CommCareKeyManager;
+import org.commcare.utils.MockAndroidKeyStoreProvider;
+import org.commcare.utils.MockEncryptionKeyProvider;
+import org.javarosa.core.model.instance.FormInstance;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
+
+import java.io.File;
+
+/**
+ * Tests for HybridFileBackedSqlStorage using Android Keystore encryption.
+ * Registers a mock Keystore provider so that generateLegacyKeyOrEmpty()
+ * returns an empty key, triggering the Keystore encryption path.
+ */
+@Config(application = CommCareTestApplication.class)
+@RunWith(AndroidJUnit4.class)
+public class HybridFileBackedSqlStorageKeystoreTest {
+
+    @Before
+    public void setup() {
+        MockAndroidKeyStoreProvider.registerProvider();
+        MockEncryptionKeyProvider provider = new MockEncryptionKeyProvider(CommCareApplication.instance());
+        CommCareKeyManager.setTestKeyAndTransformation(provider.getKeyForEncryption());
+        UnencryptedHybridFileBackedSqlStorageMock.alwaysPutInFilesystem();
+        HybridFileBackedSqlStorageMock.alwaysPutInFilesystem();
+
+        StoreFixturesOnFilesystemTests.installAppWithFixtureData(this.getClass(), "odk_level_ipm_restore.xml");
+    }
+
+    @After
+    public void tearDown() {
+        MockAndroidKeyStoreProvider.deregisterProvider();
+        CommCareKeyManager.clearTestKeyAndTransformation();
+    }
+
+    @Test
+    public void testKeystoreEncryptedWriteAndRead() {
+        HybridFileBackedSqlStorage<FormInstance> userFixtureStorage =
+                CommCareApplication.instance().getFileBackedUserStorage(
+                        HybridFileBackedSqlStorage.FIXTURE_STORAGE_TABLE_NAME, FormInstance.class);
+
+        FormInstance form = userFixtureStorage.getRecordForValues(
+                new String[]{FormInstance.META_ID},
+                new String[]{"commtrack:programs"});
+
+        String updatedName = "keystore_encrypted_fixture";
+        form.setName(updatedName);
+        userFixtureStorage.update(form.getID(), form);
+
+        FormInstance readBack = userFixtureStorage.getRecordForValues(
+                new String[]{FormInstance.META_ID},
+                new String[]{"commtrack:programs"});
+        Assert.assertEquals("Fixture should be readable after Keystore-encrypted write",
+                updatedName, readBack.getName());
+    }
+
+    @Test
+    public void testKeystoreEncryptedNewWrite() {
+        HybridFileBackedSqlStorage<FormInstance> userFixtureStorage =
+                CommCareApplication.instance().getFileBackedUserStorage(
+                        HybridFileBackedSqlStorage.FIXTURE_STORAGE_TABLE_NAME, FormInstance.class);
+
+        FormInstance form = userFixtureStorage.getRecordForValues(
+                new String[]{FormInstance.META_ID},
+                new String[]{"commtrack:programs"});
+        form.setID(-1);
+        form.initialize(null, "keystore-new-fixture");
+        userFixtureStorage.write(form);
+
+        FormInstance readBack = userFixtureStorage.getRecordForValues(
+                new String[]{FormInstance.META_ID},
+                new String[]{"keystore-new-fixture"});
+        Assert.assertNotNull("New Keystore-encrypted fixture should be readable", readBack);
+    }
+
+    @Test
+    public void testMoveKeystoreEncryptedFixtureFromFsToDbAndBack() {
+        HybridFileBackedSqlStorageMock.alwaysPutInDatabase();
+
+        HybridFileBackedSqlStorage<FormInstance> userFixtureStorage =
+                CommCareApplication.instance().getFileBackedUserStorage(
+                        HybridFileBackedSqlStorage.FIXTURE_STORAGE_TABLE_NAME, FormInstance.class);
+        FormInstance form = userFixtureStorage.getRecordForValues(
+                new String[]{FormInstance.META_ID},
+                new String[]{"commtrack:programs"});
+
+        File dbDir = userFixtureStorage.getDbDirForTesting();
+        int fileCountBefore = dbDir.listFiles().length;
+
+        // move fixture from filesystem to database
+        String newName = "keystore_fixture_in_db";
+        form.setName(newName);
+        userFixtureStorage.update(form.getID(), form);
+
+        // ensure the data can still be read
+        form = userFixtureStorage.getRecordForValues(
+                new String[]{FormInstance.META_ID},
+                new String[]{"commtrack:programs"});
+        Assert.assertEquals(newName, form.getName());
+
+        // ensure the old file was removed
+        HybridFileBackedSqlHelpers.removeOrphanedFiles(
+                CommCareApplication.instance().getUserDbHandle());
+        int fileCountAfter = dbDir.listFiles().length;
+        Assert.assertEquals(fileCountBefore - 1, fileCountAfter);
+
+        // move fixture back into filesystem (now with Keystore encryption)
+        HybridFileBackedSqlStorageMock.alwaysPutInFilesystem();
+        newName = "keystore_fixture_back_in_fs";
+        form.setName(newName);
+        userFixtureStorage.update(form.getID(), form);
+
+        // ensure the data can still be read
+        form = userFixtureStorage.getRecordForValues(
+                new String[]{FormInstance.META_ID},
+                new String[]{"commtrack:programs"});
+        Assert.assertEquals(newName, form.getName());
+
+        fileCountAfter = dbDir.listFiles().length;
+        Assert.assertEquals(fileCountBefore, fileCountAfter);
+    }
+}