From fd186c1f32e93975a3341cbe8e59df4487ca0134 Mon Sep 17 00:00:00 2001 From: valere Date: Sat, 6 May 2023 09:48:01 +0200 Subject: [PATCH] create rust db as a realm migration --- .../database/CryptoSanityMigrationTest.kt | 30 ++- ...cElementAndroidToElementRMigrationTest.kt} | 62 +++-- .../store/db/RealmCryptoStoreMigration.kt | 0 .../session/MigrateEAtoEROperation.kt | 31 --- .../sdk/internal/session/SessionModule.kt | 15 +- .../store/db/RealmCryptoStoreMigration.kt | 95 +++++++ .../store/db/migration/MigrateCryptoTo022.kt | 48 ++++ .../rust/ExtractMigrationDataUseCase.kt | 91 ++----- .../store/db/migration/rust/ExtractUtils.kt | 244 ++++++++++++++++++ .../session/MigrateEAtoEROperation.kt | 26 +- 10 files changed, 491 insertions(+), 151 deletions(-) rename matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/{ElementAndroidToElementRMigrationTest.kt => DynamicElementAndroidToElementRMigrationTest.kt} (63%) rename matrix-sdk-android/src/{main => kotlinCrypto}/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt (100%) delete mode 100644 matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt create mode 100644 matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt create mode 100644 matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt create mode 100644 matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt diff --git a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt b/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt index 2643bf643a..762129a3f7 100644 --- a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt +++ b/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt @@ -18,14 +18,20 @@ package org.matrix.android.sdk.internal.database import android.content.Context import androidx.test.platform.app.InstrumentationRegistry +import io.mockk.spyk import io.realm.Realm import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test +import org.matrix.android.sdk.TestBuildVersionSdkIntProvider +import org.matrix.android.sdk.api.securestorage.SecretStoringUtils +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule import org.matrix.android.sdk.internal.util.time.Clock +import java.io.File +import java.security.KeyStore class CryptoSanityMigrationTest { @get:Rule val configurationFactory = TestRealmConfigurationFactory() @@ -43,14 +49,28 @@ class CryptoSanityMigrationTest { realm?.close() } + private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) } + @Test fun cryptoDatabaseShouldMigrateGracefully() { val realmName = "crypto_store_20.realm" - val migration = RealmCryptoStoreMigration(object : Clock { - override fun epochMillis(): Long { - return 0L - } - }) + + val migration = RealmCryptoStoreMigration( + object : Clock { + override fun epochMillis(): Long { + return 0L + } + }, + RustEncryptionConfiguration( + "foo", + RealmKeysUtils( + context, + SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) + ) + ), + File("test_rust") + ) + val realmConfiguration = configurationFactory.createConfiguration( realmName, "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca", diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt similarity index 63% rename from matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt rename to matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt index a0847a7ad3..e6f027d408 100644 --- a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt @@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration import android.content.Context import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry +import io.mockk.spyk import io.realm.Realm import io.realm.kotlin.where import org.amshove.kluent.internal.assertEquals @@ -31,32 +32,33 @@ import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith import org.matrix.android.sdk.InstrumentedTest +import org.matrix.android.sdk.TestBuildVersionSdkIntProvider +import org.matrix.android.sdk.api.securestorage.SecretStoringUtils +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields +import org.matrix.android.sdk.internal.database.RealmKeysUtils import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory -import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation import org.matrix.android.sdk.internal.util.time.Clock import org.matrix.olm.OlmAccount import org.matrix.olm.OlmManager import org.matrix.rustcomponents.sdk.crypto.OlmMachine import java.io.File +import java.security.KeyStore @RunWith(AndroidJUnit4::class) -class ElementAndroidToElementRMigrationTest : InstrumentedTest { +class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest { @get:Rule val configurationFactory = TestRealmConfigurationFactory() - lateinit var context: Context + var context: Context = InstrumentationRegistry.getInstrumentation().context var realm: Realm? = null @Before fun setUp() { // Ensure Olm is initialized OlmManager() - context = InstrumentationRegistry.getInstrumentation().context } @After @@ -64,7 +66,22 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { realm?.close() } + private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) } + + private val rustEncryptionConfiguration = RustEncryptionConfiguration( + "foo", + RealmKeysUtils( + context, + SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) + ) + ) + + private val fakeClock = object : Clock { + override fun epochMillis() = 0L + } + @Test +<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() { testMigrate(false) } @@ -76,10 +93,13 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { } private fun testMigrate(migrateGroupSessions: Boolean) { +======= + fun dynamic_migration_to_rust() { + val targetFile = File(configurationFactory.root, "rust-sdk") + +>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt val realmName = "crypto_store_migration_16.realm" - val migration = RealmCryptoStoreMigration(object : Clock { - override fun epochMillis() = 0L - }) + val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile) val realmConfiguration = configurationFactory.createConfiguration( realmName, @@ -91,12 +111,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { configurationFactory.copyRealmFromAssets(context, realmName, realmName) realm = Realm.getInstance(realmConfiguration) - val metaData = realm!!.where().findFirst()!! val userId = metaData.userId!! val deviceId = metaData.deviceId!! val olmAccount = metaData.getOlmAccount()!! +<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt val extractor = MigrateEAtoEROperation(migrateGroupSessions) val targetFile = File(configurationFactory.root, "rust-sdk") @@ -104,6 +124,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { extractor.execute(realmConfiguration, targetFile, null) val machine = OlmMachine(userId, deviceId, targetFile.path, null) +======= + val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase()) +>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"]) assertNotNull(machine.getBackupKeys()) @@ -112,6 +135,7 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { assertTrue(crossSigningStatus.hasSelfSigning) assertTrue(crossSigningStatus.hasUserSigning) +<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt if (migrateGroupSessions) { val inboundGroupSessionEntities = realm!!.where().findAll() assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt()) @@ -122,19 +146,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { .findAll() assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt()) } +======= + // How to check that olm sessions have been migrated? + // Can see it from logs +>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt } - -// @Test -// fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() { -// val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false) -// Realm.getInstance(realmConfiguration).use { -// assertTrue(it.isEmpty) -// } -// val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null) -// assertNull(machine.getBackupKeys()) -// val crossSigningStatus = machine.crossSigningStatus() -// assertFalse(crossSigningStatus.hasMaster) -// assertFalse(crossSigningStatus.hasSelfSigning) -// assertFalse(crossSigningStatus.hasUserSigning) -// } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt b/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt similarity index 100% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt rename to matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt diff --git a/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt b/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt deleted file mode 100644 index 3fd6d1ecf1..0000000000 --- a/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2021 The Matrix.org Foundation C.I.C. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.matrix.android.sdk.internal.session - -import io.realm.RealmConfiguration -import timber.log.Timber -import java.io.File - -class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) { - - fun execute(cryptoRealm: RealmConfiguration, sessionFilesDir: File, passphrase: String?): File { - // to remove unused warning - Timber.v("Not used in kotlin crypto $cryptoRealm ${"*".repeat(passphrase?.length ?: 0)} lazy:$migrateGroupSessions") - // no op in kotlinCrypto - return sessionFilesDir - } -} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt index f0aaf8e59e..4e778e04cf 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt @@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService import org.matrix.android.sdk.api.session.typing.TypingUsersTracker import org.matrix.android.sdk.api.util.md5 -import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask @@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory import org.matrix.android.sdk.internal.di.Authenticated import org.matrix.android.sdk.internal.di.CacheDirectory -import org.matrix.android.sdk.internal.di.CryptoDatabase import org.matrix.android.sdk.internal.di.DeviceId import org.matrix.android.sdk.internal.di.SessionDatabase import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory @@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter import retrofit2.Retrofit -import timber.log.Timber import java.io.File import javax.inject.Provider import javax.inject.Qualifier -import kotlin.system.measureTimeMillis @Qualifier @Retention(AnnotationRetention.RUNTIME) @@ -189,17 +185,8 @@ internal abstract class SessionModule { @SessionScope fun providesRustCryptoFilesDir( @SessionFilesDirectory parent: File, - @CryptoDatabase realmConfiguration: RealmConfiguration, - rustEncryptionConfiguration: RustEncryptionConfiguration, ): File { - val target = File(parent, "rustFlavor") - val file: File - measureTimeMillis { - file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase()) - }.let { duration -> - Timber.v("Migrating to ER in $duration ms") - } - return file + return File(parent, "rustFlavor") } @JvmStatic diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt new file mode 100644 index 0000000000..76054a5257 --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db + +import io.realm.DynamicRealm +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo004 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo005 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo006 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo007 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo008 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo009 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo010 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo011 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo012 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo017 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo018 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo019 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022 +import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory +import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration +import org.matrix.android.sdk.internal.util.time.Clock +import java.io.File +import javax.inject.Inject + +/** + * Schema version history: + * 0, 1, 2: legacy Riot-Android; + * 3: migrate to RiotX schema; + * 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6). + */ +internal class RealmCryptoStoreMigration @Inject constructor( + private val clock: Clock, + private val rustEncryptionConfiguration: RustEncryptionConfiguration, + @SessionRustFilesDirectory + private val rustDirectory: File, +) : MatrixRealmMigration( + dbName = "Crypto", + schemaVersion = 22L, +) { + /** + * Forces all RealmCryptoStoreMigration instances to be equal. + * Avoids Realm throwing when multiple instances of the migration are set. + */ + override fun equals(other: Any?) = other is RealmCryptoStoreMigration + override fun hashCode() = 5000 + + override fun doMigrate(realm: DynamicRealm, oldVersion: Long) { + if (oldVersion < 1) MigrateCryptoTo001Legacy(realm).perform() + if (oldVersion < 2) MigrateCryptoTo002Legacy(realm).perform() + if (oldVersion < 3) MigrateCryptoTo003RiotX(realm).perform() + if (oldVersion < 4) MigrateCryptoTo004(realm).perform() + if (oldVersion < 5) MigrateCryptoTo005(realm).perform() + if (oldVersion < 6) MigrateCryptoTo006(realm).perform() + if (oldVersion < 7) MigrateCryptoTo007(realm).perform() + if (oldVersion < 8) MigrateCryptoTo008(realm, clock).perform() + if (oldVersion < 9) MigrateCryptoTo009(realm).perform() + if (oldVersion < 10) MigrateCryptoTo010(realm).perform() + if (oldVersion < 11) MigrateCryptoTo011(realm).perform() + if (oldVersion < 12) MigrateCryptoTo012(realm).perform() + if (oldVersion < 13) MigrateCryptoTo013(realm).perform() + if (oldVersion < 14) MigrateCryptoTo014(realm).perform() + if (oldVersion < 15) MigrateCryptoTo015(realm).perform() + if (oldVersion < 16) MigrateCryptoTo016(realm).perform() + if (oldVersion < 17) MigrateCryptoTo017(realm).perform() + if (oldVersion < 18) MigrateCryptoTo018(realm).perform() + if (oldVersion < 19) MigrateCryptoTo019(realm).perform() + if (oldVersion < 20) MigrateCryptoTo020(realm).perform() + if (oldVersion < 21) MigrateCryptoTo021(realm).perform() + if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform() + } +} diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt new file mode 100644 index 0000000000..d26fe01741 --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db.migration + +import io.realm.DynamicRealm +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation +import org.matrix.android.sdk.internal.util.database.RealmMigrator +import java.io.File + +/** + * This migration creates the rust database and migrates from legacy crypto + */ +internal class MigrateCryptoTo022( + realm: DynamicRealm, + private val rustDirectory: File, + private val rustEncryptionConfiguration: RustEncryptionConfiguration +) : RealmMigrator( + realm, + 22 +) { + override fun doMigrate(realm: DynamicRealm) { + // Migrate to rust! + val migrateOperation = MigrateEAtoEROperation() + migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase()) + + // wa can't delete all for now, but we can do some cleaning + realm.schema.get("OlmSessionEntity")?.transform { + it.deleteFromRealm() + } + + // a future migration will clean the rest + } +} diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt index da125b8e16..5431f17dae 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt @@ -24,12 +24,9 @@ import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity import org.matrix.olm.OlmSession import org.matrix.olm.OlmUtility -import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport import org.matrix.rustcomponents.sdk.crypto.MigrationData -import org.matrix.rustcomponents.sdk.crypto.PickledAccount import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession import org.matrix.rustcomponents.sdk.crypto.PickledSession import timber.log.Timber @@ -40,7 +37,7 @@ private val charset = Charset.forName("UTF-8") internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) { - fun extractData(realm: Realm, importPartial: ((MigrationData) -> Unit)) { + fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) { return try { extract(realm, importPartial) } catch (failure: Throwable) { @@ -57,89 +54,33 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f } } - private fun extract(realm: Realm, importPartial: ((MigrationData) -> Unit)) { - val metadataEntity = realm.where().findFirst() - ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity") - + private fun extract(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) { val pickleKey = OlmUtility.getRandomKey() - val masterKey = metadataEntity.xSignMasterPrivateKey - val userKey = metadataEntity.xSignUserPrivateKey - val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey - - val userId = metadataEntity.userId - ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") - val deviceId = metadataEntity.deviceId - ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") - - val backupVersion = metadataEntity.backupVersion - val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey - - val isOlmAccountShared = metadataEntity.deviceKeysSentToServer - - val olmAccount = metadataEntity.getOlmAccount() - ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") - val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() - olmAccount.oneTimeKeys() - val pickledAccount = PickledAccount( - userId = userId, - deviceId = deviceId, - pickle = pickledOlmAccount, - shared = isOlmAccountShared, - uploadedSignedKeyCount = 50 - ) - - val baseExtract = MigrationData( - account = pickledAccount, - pickleKey = pickleKey.map { it.toUByte() }, - crossSigning = CrossSigningKeyExport( - masterKey = masterKey, - selfSigningKey = selfSignedKey, - userSigningKey = userKey - ), - sessions = emptyList(), - backupRecoveryKey = backupRecoveryKey, - trackedUsers = emptyList(), - inboundGroupSessions = emptyList(), - backupVersion = backupVersion, - // TODO import room settings from legacy DB - roomSettings = emptyMap() - ) + val baseExtract = realm.getPickledAccount(pickleKey) // import the account asap importPartial(baseExtract) val chunkSize = 500 - realm.where() - .findAll() - .chunked(chunkSize) { chunk -> - val trackedUserIds = chunk.mapNotNull { it.userId } - importPartial( - baseExtract.copy(trackedUsers = trackedUserIds) - ) - } + realm.trackedUsersChunk(500) { + importPartial( + baseExtract.copy(trackedUsers = it) + ) + } var migratedOlmSessionCount = 0 - var readTime = 0L var writeTime = 0L measureTimeMillis { - realm.where().findAll() - .chunked(chunkSize) { chunk -> - migratedOlmSessionCount += chunk.size - val export: List - measureTimeMillis { - export = chunk.map { it.toPickledSession(pickleKey) } - }.also { - readTime += it - } - measureTimeMillis { - importPartial( - baseExtract.copy(sessions = export) - ) - }.also { writeTime += it } - } + realm.pickledOlmSessions(pickleKey, chunkSize) { pickledSessions -> + migratedOlmSessionCount += pickledSessions.size + measureTimeMillis { + importPartial( + baseExtract.copy(sessions = pickledSessions) + ) + }.also { writeTime += it } + } }.also { Timber.i("Migration: took $it ms to migrate $migratedOlmSessionCount olm sessions") - Timber.i("Migration: extract time $readTime") Timber.i("Migration: rust import time $writeTime") } diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt new file mode 100644 index 0000000000..343960b92b --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db.migration.rust + +import io.realm.kotlin.where +import okhttp3.internal.toImmutableList +import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm +import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields +import org.matrix.olm.OlmAccount +import org.matrix.olm.OlmSession +import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport +import org.matrix.rustcomponents.sdk.crypto.MigrationData +import org.matrix.rustcomponents.sdk.crypto.PickledAccount +import org.matrix.rustcomponents.sdk.crypto.PickledSession +import java.nio.charset.Charset + +sealed class RealmToMigrate { + data class DynamicRealm(val realm: io.realm.DynamicRealm) : RealmToMigrate() + data class ClassicRealm(val realm: io.realm.Realm) : RealmToMigrate() +} + +fun RealmToMigrate.hasExistingData(): Boolean { + return when (this) { + is RealmToMigrate.ClassicRealm -> { + !this.realm.isEmpty && + // Check if there is a MetaData object + this.realm.where().count() > 0 && + this.realm.where().findFirst()?.olmAccountData != null + } + is RealmToMigrate.DynamicRealm -> { + return true + } + } +} + +@Throws +fun RealmToMigrate.getPickledAccount(pickleKey: ByteArray): MigrationData { + return when (this) { + is RealmToMigrate.ClassicRealm -> { + val metadataEntity = realm.where().findFirst() + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity") + + val masterKey = metadataEntity.xSignMasterPrivateKey + val userKey = metadataEntity.xSignUserPrivateKey + val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey + + val userId = metadataEntity.userId + ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") + val deviceId = metadataEntity.deviceId + ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") + + val backupVersion = metadataEntity.backupVersion + val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey + + val isOlmAccountShared = metadataEntity.deviceKeysSentToServer + + val olmAccount = metadataEntity.getOlmAccount() + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") + val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() + + val pickledAccount = PickledAccount( + userId = userId, + deviceId = deviceId, + pickle = pickledOlmAccount, + shared = isOlmAccountShared, + uploadedSignedKeyCount = 50 + ) + MigrationData( + account = pickledAccount, + pickleKey = pickleKey.map { it.toUByte() }, + crossSigning = CrossSigningKeyExport( + masterKey = masterKey, + selfSigningKey = selfSignedKey, + userSigningKey = userKey + ), + sessions = emptyList(), + backupRecoveryKey = backupRecoveryKey, + trackedUsers = emptyList(), + inboundGroupSessions = emptyList(), + backupVersion = backupVersion, + // TODO import room settings from legacy DB + roomSettings = emptyMap() + ) + } + is RealmToMigrate.DynamicRealm -> { + val cryptoMetadataEntitySchema = realm.schema.get("CryptoMetadataEntity") + ?: throw java.lang.IllegalStateException("Missing Metadata entity") + + var migrationData: MigrationData? = null + cryptoMetadataEntitySchema.transform { dynMetaData -> + + val serializedOlmAccount = dynMetaData.getString(CryptoMetadataEntityFields.OLM_ACCOUNT_DATA) + + val masterKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_MASTER_PRIVATE_KEY) + val userKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_USER_PRIVATE_KEY) + val selfSignedKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_SELF_SIGNED_PRIVATE_KEY) + + val userId = dynMetaData.getString(CryptoMetadataEntityFields.USER_ID) + ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") + val deviceId = dynMetaData.getString(CryptoMetadataEntityFields.DEVICE_ID) + ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") + + val backupVersion = dynMetaData.getString(CryptoMetadataEntityFields.BACKUP_VERSION) + val backupRecoveryKey = dynMetaData.getString(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY) + + val isOlmAccountShared = dynMetaData.getBoolean(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER) + + val olmAccount = deserializeFromRealm(serializedOlmAccount) + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") + + val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() + + val pickledAccount = PickledAccount( + userId = userId, + deviceId = deviceId, + pickle = pickledOlmAccount, + shared = isOlmAccountShared, + uploadedSignedKeyCount = 50 + ) + + migrationData = MigrationData( + account = pickledAccount, + pickleKey = pickleKey.map { it.toUByte() }, + crossSigning = CrossSigningKeyExport( + masterKey = masterKey, + selfSigningKey = selfSignedKey, + userSigningKey = userKey + ), + sessions = emptyList(), + backupRecoveryKey = backupRecoveryKey, + trackedUsers = emptyList(), + inboundGroupSessions = emptyList(), + backupVersion = backupVersion, + // TODO import room settings from legacy DB + roomSettings = emptyMap() + ) + } + migrationData!! + } + } +} + +fun RealmToMigrate.trackedUsersChunk(chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where() + .findAll() + .chunked(chunkSize) + .onEach { + onChunk(it.mapNotNull { it.userId }) + } + } + is RealmToMigrate.DynamicRealm -> { + val userList = mutableListOf() + realm.schema.get("UserEntity")?.transform { + val userId = it.getString(UserEntityFields.USER_ID) + // should we check the tracking status? + userList.add(userId) + if (userList.size > chunkSize) { + onChunk(userList.toImmutableList()) + userList.clear() + } + } + if (userList.isNotEmpty()) { + onChunk(userList) + } + } + } +} + +fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where().findAll() + .chunked(chunkSize) { chunk -> + val export = chunk.map { it.toPickledSession(pickleKey) } + onChunk(export) + } + } + is RealmToMigrate.DynamicRealm -> { + val pickledSessions = mutableListOf() + realm.schema.get("OlmSessionEntity")?.transform { + val sessionData = it.getString(OlmSessionEntityFields.OLM_SESSION_DATA) + val deviceKey = it.getString(OlmSessionEntityFields.DEVICE_KEY) + val lastReceivedMessageTs = it.getLong(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS) + val olmSession = deserializeFromRealm(sessionData)!! + val pickle = olmSession.pickle(pickleKey, StringBuffer()).asString() + val pickledSession = PickledSession( + pickle = pickle, + senderKey = deviceKey, + createdUsingFallbackKey = false, + creationTime = lastReceivedMessageTs.toString(), + lastUseTime = lastReceivedMessageTs.toString() + ) + // should we check the tracking status? + pickledSessions.add(pickledSession) + if (pickledSessions.size > chunkSize) { + onChunk(pickledSessions.toImmutableList()) + pickledSessions.clear() + } + } + if (pickledSessions.isNotEmpty()) { + onChunk(pickledSessions) + } + } + } +} + +private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession { + val deviceKey = this.deviceKey ?: "" + val lastReceivedMessageTs = this.lastReceivedMessageTs + val olmSessionStr = this.olmSessionData + val olmSession = deserializeFromRealm(olmSessionStr)!! + val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString() + return PickledSession( + pickle = pickledOlmSession, + senderKey = deviceKey, + createdUsingFallbackKey = false, + creationTime = lastReceivedMessageTs.toString(), + lastUseTime = lastReceivedMessageTs.toString() + ) +} + +private val charset = Charset.forName("UTF-8") +private fun ByteArray.asString() = String(this, charset) diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt index 741b5a4c8f..5bfafb44e8 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt @@ -16,9 +16,11 @@ package org.matrix.android.sdk.internal.session +import io.realm.DynamicRealm import io.realm.Realm import io.realm.RealmConfiguration import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase +import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.RealmToMigrate import org.matrix.rustcomponents.sdk.crypto.ProgressListener import timber.log.Timber import java.io.File @@ -40,9 +42,8 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) Timber.v("OnProgress: $progress/$total") } } - Realm.getInstance(cryptoRealm).use { realm -> - extractMigrationData.extractData(realm) { + extractMigrationData.extractData(RealmToMigrate.ClassicRealm(realm)) { org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener) } } @@ -53,4 +54,25 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) } return rustFilesDir } + + fun dynamicExecute(dynamicRealm: DynamicRealm, rustFilesDir: File, passphrase: String?) { + if (!rustFilesDir.exists()) { + rustFilesDir.mkdir() + } + val extractMigrationData = ExtractMigrationDataUseCase() + + try { + val progressListener = object : ProgressListener { + override fun onProgress(progress: Int, total: Int) { + Timber.v("OnProgress: $progress/$total") + } + } + extractMigrationData.extractData(RealmToMigrate.DynamicRealm(dynamicRealm)) { + org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener) + } + } catch (failure: Throwable) { + Timber.e(failure, "Failure while calling rust migration method") + throw failure + } + } }