create rust db as a realm migration

This commit is contained in:
valere 2023-05-06 09:48:01 +02:00 committed by Benoit Marty
parent 62ec1eb505
commit fd186c1f32
10 changed files with 491 additions and 151 deletions

View File

@ -18,14 +18,20 @@ package org.matrix.android.sdk.internal.database
import android.content.Context
import androidx.test.platform.app.InstrumentationRegistry
import io.mockk.spyk
import io.realm.Realm
import org.junit.After
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.util.time.Clock
import java.io.File
import java.security.KeyStore
class CryptoSanityMigrationTest {
@get:Rule val configurationFactory = TestRealmConfigurationFactory()
@ -43,14 +49,28 @@ class CryptoSanityMigrationTest {
realm?.close()
}
private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
@Test
fun cryptoDatabaseShouldMigrateGracefully() {
val realmName = "crypto_store_20.realm"
val migration = RealmCryptoStoreMigration(object : Clock {
val migration = RealmCryptoStoreMigration(
object : Clock {
override fun epochMillis(): Long {
return 0L
}
})
},
RustEncryptionConfiguration(
"foo",
RealmKeysUtils(
context,
SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
)
),
File("test_rust")
)
val realmConfiguration = configurationFactory.createConfiguration(
realmName,
"7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca",

View File

@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration
import android.content.Context
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import io.mockk.spyk
import io.realm.Realm
import io.realm.kotlin.where
import org.amshove.kluent.internal.assertEquals
@ -31,32 +32,33 @@ import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.matrix.android.sdk.InstrumentedTest
import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields
import org.matrix.android.sdk.internal.database.RealmKeysUtils
import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory
import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
import org.matrix.android.sdk.internal.util.time.Clock
import org.matrix.olm.OlmAccount
import org.matrix.olm.OlmManager
import org.matrix.rustcomponents.sdk.crypto.OlmMachine
import java.io.File
import java.security.KeyStore
@RunWith(AndroidJUnit4::class)
class ElementAndroidToElementRMigrationTest : InstrumentedTest {
class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
@get:Rule val configurationFactory = TestRealmConfigurationFactory()
lateinit var context: Context
var context: Context = InstrumentationRegistry.getInstrumentation().context
var realm: Realm? = null
@Before
fun setUp() {
// Ensure Olm is initialized
OlmManager()
context = InstrumentationRegistry.getInstrumentation().context
}
@After
@ -64,7 +66,22 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
realm?.close()
}
private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
private val rustEncryptionConfiguration = RustEncryptionConfiguration(
"foo",
RealmKeysUtils(
context,
SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
)
)
private val fakeClock = object : Clock {
override fun epochMillis() = 0L
}
@Test
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() {
testMigrate(false)
}
@ -76,10 +93,13 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
}
private fun testMigrate(migrateGroupSessions: Boolean) {
=======
fun dynamic_migration_to_rust() {
val targetFile = File(configurationFactory.root, "rust-sdk")
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
val realmName = "crypto_store_migration_16.realm"
val migration = RealmCryptoStoreMigration(object : Clock {
override fun epochMillis() = 0L
})
val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile)
val realmConfiguration = configurationFactory.createConfiguration(
realmName,
@ -91,12 +111,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
configurationFactory.copyRealmFromAssets(context, realmName, realmName)
realm = Realm.getInstance(realmConfiguration)
val metaData = realm!!.where<CryptoMetadataEntity>().findFirst()!!
val userId = metaData.userId!!
val deviceId = metaData.deviceId!!
val olmAccount = metaData.getOlmAccount()!!
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
val extractor = MigrateEAtoEROperation(migrateGroupSessions)
val targetFile = File(configurationFactory.root, "rust-sdk")
@ -104,6 +124,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
extractor.execute(realmConfiguration, targetFile, null)
val machine = OlmMachine(userId, deviceId, targetFile.path, null)
=======
val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase())
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"])
assertNotNull(machine.getBackupKeys())
@ -112,6 +135,7 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
assertTrue(crossSigningStatus.hasSelfSigning)
assertTrue(crossSigningStatus.hasUserSigning)
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
if (migrateGroupSessions) {
val inboundGroupSessionEntities = realm!!.where<OlmInboundGroupSessionEntity>().findAll()
assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt())
@ -122,19 +146,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
.findAll()
assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt())
}
=======
// How to check that olm sessions have been migrated?
// Can see it from logs
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
}
// @Test
// fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() {
// val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false)
// Realm.getInstance(realmConfiguration).use {
// assertTrue(it.isEmpty)
// }
// val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null)
// assertNull(machine.getBackupKeys())
// val crossSigningStatus = machine.crossSigningStatus()
// assertFalse(crossSigningStatus.hasMaster)
// assertFalse(crossSigningStatus.hasSelfSigning)
// assertFalse(crossSigningStatus.hasUserSigning)
// }
}

View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2021 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.session
import io.realm.RealmConfiguration
import timber.log.Timber
import java.io.File
class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) {
fun execute(cryptoRealm: RealmConfiguration, sessionFilesDir: File, passphrase: String?): File {
// to remove unused warning
Timber.v("Not used in kotlin crypto $cryptoRealm ${"*".repeat(passphrase?.length ?: 0)} lazy:$migrateGroupSessions")
// no op in kotlinCrypto
return sessionFilesDir
}
}

View File

@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService
import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService
import org.matrix.android.sdk.api.session.typing.TypingUsersTracker
import org.matrix.android.sdk.api.util.md5
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService
import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask
import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask
@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider
import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory
import org.matrix.android.sdk.internal.di.Authenticated
import org.matrix.android.sdk.internal.di.CacheDirectory
import org.matrix.android.sdk.internal.di.CryptoDatabase
import org.matrix.android.sdk.internal.di.DeviceId
import org.matrix.android.sdk.internal.di.SessionDatabase
import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory
@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker
import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService
import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter
import retrofit2.Retrofit
import timber.log.Timber
import java.io.File
import javax.inject.Provider
import javax.inject.Qualifier
import kotlin.system.measureTimeMillis
@Qualifier
@Retention(AnnotationRetention.RUNTIME)
@ -189,17 +185,8 @@ internal abstract class SessionModule {
@SessionScope
fun providesRustCryptoFilesDir(
@SessionFilesDirectory parent: File,
@CryptoDatabase realmConfiguration: RealmConfiguration,
rustEncryptionConfiguration: RustEncryptionConfiguration,
): File {
val target = File(parent, "rustFlavor")
val file: File
measureTimeMillis {
file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase())
}.let { duration ->
Timber.v("Migrating to ER in $duration ms")
}
return file
return File(parent, "rustFlavor")
}
@JvmStatic

View File

@ -0,0 +1,95 @@
/*
* Copyright 2020 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db
import io.realm.DynamicRealm
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo004
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo005
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo006
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo007
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo008
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo009
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo010
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo011
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo012
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo017
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo018
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo019
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022
import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory
import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration
import org.matrix.android.sdk.internal.util.time.Clock
import java.io.File
import javax.inject.Inject
/**
* Schema version history:
* 0, 1, 2: legacy Riot-Android;
* 3: migrate to RiotX schema;
* 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6).
*/
internal class RealmCryptoStoreMigration @Inject constructor(
private val clock: Clock,
private val rustEncryptionConfiguration: RustEncryptionConfiguration,
@SessionRustFilesDirectory
private val rustDirectory: File,
) : MatrixRealmMigration(
dbName = "Crypto",
schemaVersion = 22L,
) {
/**
* Forces all RealmCryptoStoreMigration instances to be equal.
* Avoids Realm throwing when multiple instances of the migration are set.
*/
override fun equals(other: Any?) = other is RealmCryptoStoreMigration
override fun hashCode() = 5000
override fun doMigrate(realm: DynamicRealm, oldVersion: Long) {
if (oldVersion < 1) MigrateCryptoTo001Legacy(realm).perform()
if (oldVersion < 2) MigrateCryptoTo002Legacy(realm).perform()
if (oldVersion < 3) MigrateCryptoTo003RiotX(realm).perform()
if (oldVersion < 4) MigrateCryptoTo004(realm).perform()
if (oldVersion < 5) MigrateCryptoTo005(realm).perform()
if (oldVersion < 6) MigrateCryptoTo006(realm).perform()
if (oldVersion < 7) MigrateCryptoTo007(realm).perform()
if (oldVersion < 8) MigrateCryptoTo008(realm, clock).perform()
if (oldVersion < 9) MigrateCryptoTo009(realm).perform()
if (oldVersion < 10) MigrateCryptoTo010(realm).perform()
if (oldVersion < 11) MigrateCryptoTo011(realm).perform()
if (oldVersion < 12) MigrateCryptoTo012(realm).perform()
if (oldVersion < 13) MigrateCryptoTo013(realm).perform()
if (oldVersion < 14) MigrateCryptoTo014(realm).perform()
if (oldVersion < 15) MigrateCryptoTo015(realm).perform()
if (oldVersion < 16) MigrateCryptoTo016(realm).perform()
if (oldVersion < 17) MigrateCryptoTo017(realm).perform()
if (oldVersion < 18) MigrateCryptoTo018(realm).perform()
if (oldVersion < 19) MigrateCryptoTo019(realm).perform()
if (oldVersion < 20) MigrateCryptoTo020(realm).perform()
if (oldVersion < 21) MigrateCryptoTo021(realm).perform()
if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform()
}
}

View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2023 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db.migration
import io.realm.DynamicRealm
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
import org.matrix.android.sdk.internal.util.database.RealmMigrator
import java.io.File
/**
* This migration creates the rust database and migrates from legacy crypto
*/
internal class MigrateCryptoTo022(
realm: DynamicRealm,
private val rustDirectory: File,
private val rustEncryptionConfiguration: RustEncryptionConfiguration
) : RealmMigrator(
realm,
22
) {
override fun doMigrate(realm: DynamicRealm) {
// Migrate to rust!
val migrateOperation = MigrateEAtoEROperation()
migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase())
// wa can't delete all for now, but we can do some cleaning
realm.schema.get("OlmSessionEntity")?.transform {
it.deleteFromRealm()
}
// a future migration will clean the rest
}
}

View File

@ -24,12 +24,9 @@ import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
import org.matrix.olm.OlmSession
import org.matrix.olm.OlmUtility
import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
import org.matrix.rustcomponents.sdk.crypto.MigrationData
import org.matrix.rustcomponents.sdk.crypto.PickledAccount
import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession
import org.matrix.rustcomponents.sdk.crypto.PickledSession
import timber.log.Timber
@ -40,7 +37,7 @@ private val charset = Charset.forName("UTF-8")
internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) {
fun extractData(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
return try {
extract(realm, importPartial)
} catch (failure: Throwable) {
@ -57,89 +54,33 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f
}
}
private fun extract(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
private fun extract(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
val pickleKey = OlmUtility.getRandomKey()
val masterKey = metadataEntity.xSignMasterPrivateKey
val userKey = metadataEntity.xSignUserPrivateKey
val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
val userId = metadataEntity.userId
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
val deviceId = metadataEntity.deviceId
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
val backupVersion = metadataEntity.backupVersion
val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
val olmAccount = metadataEntity.getOlmAccount()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
olmAccount.oneTimeKeys()
val pickledAccount = PickledAccount(
userId = userId,
deviceId = deviceId,
pickle = pickledOlmAccount,
shared = isOlmAccountShared,
uploadedSignedKeyCount = 50
)
val baseExtract = MigrationData(
account = pickledAccount,
pickleKey = pickleKey.map { it.toUByte() },
crossSigning = CrossSigningKeyExport(
masterKey = masterKey,
selfSigningKey = selfSignedKey,
userSigningKey = userKey
),
sessions = emptyList(),
backupRecoveryKey = backupRecoveryKey,
trackedUsers = emptyList(),
inboundGroupSessions = emptyList(),
backupVersion = backupVersion,
// TODO import room settings from legacy DB
roomSettings = emptyMap()
)
val baseExtract = realm.getPickledAccount(pickleKey)
// import the account asap
importPartial(baseExtract)
val chunkSize = 500
realm.where<UserEntity>()
.findAll()
.chunked(chunkSize) { chunk ->
val trackedUserIds = chunk.mapNotNull { it.userId }
realm.trackedUsersChunk(500) {
importPartial(
baseExtract.copy(trackedUsers = trackedUserIds)
baseExtract.copy(trackedUsers = it)
)
}
var migratedOlmSessionCount = 0
var readTime = 0L
var writeTime = 0L
measureTimeMillis {
realm.where<OlmSessionEntity>().findAll()
.chunked(chunkSize) { chunk ->
migratedOlmSessionCount += chunk.size
val export: List<PickledSession>
measureTimeMillis {
export = chunk.map { it.toPickledSession(pickleKey) }
}.also {
readTime += it
}
realm.pickledOlmSessions(pickleKey, chunkSize) { pickledSessions ->
migratedOlmSessionCount += pickledSessions.size
measureTimeMillis {
importPartial(
baseExtract.copy(sessions = export)
baseExtract.copy(sessions = pickledSessions)
)
}.also { writeTime += it }
}
}.also {
Timber.i("Migration: took $it ms to migrate $migratedOlmSessionCount olm sessions")
Timber.i("Migration: extract time $readTime")
Timber.i("Migration: rust import time $writeTime")
}

View File

@ -0,0 +1,244 @@
/*
* Copyright (c) 2023 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
import io.realm.kotlin.where
import okhttp3.internal.toImmutableList
import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields
import org.matrix.olm.OlmAccount
import org.matrix.olm.OlmSession
import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
import org.matrix.rustcomponents.sdk.crypto.MigrationData
import org.matrix.rustcomponents.sdk.crypto.PickledAccount
import org.matrix.rustcomponents.sdk.crypto.PickledSession
import java.nio.charset.Charset
sealed class RealmToMigrate {
data class DynamicRealm(val realm: io.realm.DynamicRealm) : RealmToMigrate()
data class ClassicRealm(val realm: io.realm.Realm) : RealmToMigrate()
}
fun RealmToMigrate.hasExistingData(): Boolean {
return when (this) {
is RealmToMigrate.ClassicRealm -> {
!this.realm.isEmpty &&
// Check if there is a MetaData object
this.realm.where<CryptoMetadataEntity>().count() > 0 &&
this.realm.where<CryptoMetadataEntity>().findFirst()?.olmAccountData != null
}
is RealmToMigrate.DynamicRealm -> {
return true
}
}
}
@Throws
fun RealmToMigrate.getPickledAccount(pickleKey: ByteArray): MigrationData {
return when (this) {
is RealmToMigrate.ClassicRealm -> {
val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
val masterKey = metadataEntity.xSignMasterPrivateKey
val userKey = metadataEntity.xSignUserPrivateKey
val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
val userId = metadataEntity.userId
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
val deviceId = metadataEntity.deviceId
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
val backupVersion = metadataEntity.backupVersion
val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
val olmAccount = metadataEntity.getOlmAccount()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
val pickledAccount = PickledAccount(
userId = userId,
deviceId = deviceId,
pickle = pickledOlmAccount,
shared = isOlmAccountShared,
uploadedSignedKeyCount = 50
)
MigrationData(
account = pickledAccount,
pickleKey = pickleKey.map { it.toUByte() },
crossSigning = CrossSigningKeyExport(
masterKey = masterKey,
selfSigningKey = selfSignedKey,
userSigningKey = userKey
),
sessions = emptyList(),
backupRecoveryKey = backupRecoveryKey,
trackedUsers = emptyList(),
inboundGroupSessions = emptyList(),
backupVersion = backupVersion,
// TODO import room settings from legacy DB
roomSettings = emptyMap()
)
}
is RealmToMigrate.DynamicRealm -> {
val cryptoMetadataEntitySchema = realm.schema.get("CryptoMetadataEntity")
?: throw java.lang.IllegalStateException("Missing Metadata entity")
var migrationData: MigrationData? = null
cryptoMetadataEntitySchema.transform { dynMetaData ->
val serializedOlmAccount = dynMetaData.getString(CryptoMetadataEntityFields.OLM_ACCOUNT_DATA)
val masterKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_MASTER_PRIVATE_KEY)
val userKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_USER_PRIVATE_KEY)
val selfSignedKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_SELF_SIGNED_PRIVATE_KEY)
val userId = dynMetaData.getString(CryptoMetadataEntityFields.USER_ID)
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
val deviceId = dynMetaData.getString(CryptoMetadataEntityFields.DEVICE_ID)
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
val backupVersion = dynMetaData.getString(CryptoMetadataEntityFields.BACKUP_VERSION)
val backupRecoveryKey = dynMetaData.getString(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY)
val isOlmAccountShared = dynMetaData.getBoolean(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER)
val olmAccount = deserializeFromRealm<OlmAccount>(serializedOlmAccount)
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
val pickledAccount = PickledAccount(
userId = userId,
deviceId = deviceId,
pickle = pickledOlmAccount,
shared = isOlmAccountShared,
uploadedSignedKeyCount = 50
)
migrationData = MigrationData(
account = pickledAccount,
pickleKey = pickleKey.map { it.toUByte() },
crossSigning = CrossSigningKeyExport(
masterKey = masterKey,
selfSigningKey = selfSignedKey,
userSigningKey = userKey
),
sessions = emptyList(),
backupRecoveryKey = backupRecoveryKey,
trackedUsers = emptyList(),
inboundGroupSessions = emptyList(),
backupVersion = backupVersion,
// TODO import room settings from legacy DB
roomSettings = emptyMap()
)
}
migrationData!!
}
}
}
fun RealmToMigrate.trackedUsersChunk(chunkSize: Int, onChunk: ((List<String>) -> Unit)) {
when (this) {
is RealmToMigrate.ClassicRealm -> {
realm.where<UserEntity>()
.findAll()
.chunked(chunkSize)
.onEach {
onChunk(it.mapNotNull { it.userId })
}
}
is RealmToMigrate.DynamicRealm -> {
val userList = mutableListOf<String>()
realm.schema.get("UserEntity")?.transform {
val userId = it.getString(UserEntityFields.USER_ID)
// should we check the tracking status?
userList.add(userId)
if (userList.size > chunkSize) {
onChunk(userList.toImmutableList())
userList.clear()
}
}
if (userList.isNotEmpty()) {
onChunk(userList)
}
}
}
}
fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List<PickledSession>) -> Unit)) {
when (this) {
is RealmToMigrate.ClassicRealm -> {
realm.where<OlmSessionEntity>().findAll()
.chunked(chunkSize) { chunk ->
val export = chunk.map { it.toPickledSession(pickleKey) }
onChunk(export)
}
}
is RealmToMigrate.DynamicRealm -> {
val pickledSessions = mutableListOf<PickledSession>()
realm.schema.get("OlmSessionEntity")?.transform {
val sessionData = it.getString(OlmSessionEntityFields.OLM_SESSION_DATA)
val deviceKey = it.getString(OlmSessionEntityFields.DEVICE_KEY)
val lastReceivedMessageTs = it.getLong(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS)
val olmSession = deserializeFromRealm<OlmSession>(sessionData)!!
val pickle = olmSession.pickle(pickleKey, StringBuffer()).asString()
val pickledSession = PickledSession(
pickle = pickle,
senderKey = deviceKey,
createdUsingFallbackKey = false,
creationTime = lastReceivedMessageTs.toString(),
lastUseTime = lastReceivedMessageTs.toString()
)
// should we check the tracking status?
pickledSessions.add(pickledSession)
if (pickledSessions.size > chunkSize) {
onChunk(pickledSessions.toImmutableList())
pickledSessions.clear()
}
}
if (pickledSessions.isNotEmpty()) {
onChunk(pickledSessions)
}
}
}
}
private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession {
val deviceKey = this.deviceKey ?: ""
val lastReceivedMessageTs = this.lastReceivedMessageTs
val olmSessionStr = this.olmSessionData
val olmSession = deserializeFromRealm<OlmSession>(olmSessionStr)!!
val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString()
return PickledSession(
pickle = pickledOlmSession,
senderKey = deviceKey,
createdUsingFallbackKey = false,
creationTime = lastReceivedMessageTs.toString(),
lastUseTime = lastReceivedMessageTs.toString()
)
}
private val charset = Charset.forName("UTF-8")
private fun ByteArray.asString() = String(this, charset)

View File

@ -16,9 +16,11 @@
package org.matrix.android.sdk.internal.session
import io.realm.DynamicRealm
import io.realm.Realm
import io.realm.RealmConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase
import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.RealmToMigrate
import org.matrix.rustcomponents.sdk.crypto.ProgressListener
import timber.log.Timber
import java.io.File
@ -40,9 +42,8 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
Timber.v("OnProgress: $progress/$total")
}
}
Realm.getInstance(cryptoRealm).use { realm ->
extractMigrationData.extractData(realm) {
extractMigrationData.extractData(RealmToMigrate.ClassicRealm(realm)) {
org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
}
}
@ -53,4 +54,25 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
}
return rustFilesDir
}
fun dynamicExecute(dynamicRealm: DynamicRealm, rustFilesDir: File, passphrase: String?) {
if (!rustFilesDir.exists()) {
rustFilesDir.mkdir()
}
val extractMigrationData = ExtractMigrationDataUseCase()
try {
val progressListener = object : ProgressListener {
override fun onProgress(progress: Int, total: Int) {
Timber.v("OnProgress: $progress/$total")
}
}
extractMigrationData.extractData(RealmToMigrate.DynamicRealm(dynamicRealm)) {
org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
}
} catch (failure: Throwable) {
Timber.e(failure, "Failure while calling rust migration method")
throw failure
}
}
}