diff --git a/changelog.d/4446.bugfix b/changelog.d/4446.bugfix
new file mode 100644
index 0000000000..140e18a3ba
--- /dev/null
+++ b/changelog.d/4446.bugfix
@@ -0,0 +1 @@
+Unable to establish Olm outbound session from fallback key
\ No newline at end of file
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/model/MXKey.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/model/MXKey.kt
index f71c5079b3..9f425eee7f 100755
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/model/MXKey.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/model/MXKey.kt
@@ -38,14 +38,22 @@ data class MXKey(
         /**
          * signature user Id to [deviceid][signature]
          */
-        private val signatures: Map<String, Map<String, String>>
+        private val signatures: Map<String, Map<String, String>>,
+
+        /**
+         * We have to store the original json because it can contain other fields
+         * that we don't support yet but they would be needed to check signatures
+         */
+        private val rawMap: JsonDict
 ) {
 
     /**
      * @return the signed data map
      */
     fun signalableJSONDictionary(): Map<String, Any> {
-        return mapOf("key" to value)
+        return rawMap.filter {
+            it.key != "signatures" && it.key != "unsigned"
+        }
     }
 
     /**
@@ -82,6 +90,7 @@ data class MXKey(
          * <pre>
          *   "signed_curve25519:AAAAFw": {
          *     "key": "IjwIcskng7YjYcn0tS8TUOT2OHHtBSfMpcfIczCgXj4",
+         *     "fallback" : true|false
          *     "signatures": {
          *       "@userId:matrix.org": {
          *         "ed25519:GMJRREOASV": "EUjp6pXzK9u3SDFR\/qLbzpOi3bEREeI6qMnKzXu992HsfuDDZftfJfiUXv9b\/Hqq1og4qM\/vCQJGTHAWMmgkCg"
@@ -107,7 +116,8 @@ data class MXKey(
                                     type = components[0],
                                     keyId = components[1],
                                     value = params["key"] as String,
-                                    signatures = params["signatures"] as Map<String, Map<String, String>>
+                                    signatures = params["signatures"] as Map<String, Map<String, String>>,
+                                    rawMap = params
                             )
                         }
                     }