Merge branch 'develop' into patch-1

.github/ISSUE_TEMPLATE/matrix-sdk.md

@@ -0,0 +1,10 @@
+---
+name: Matrix SDK
+about: Report issue or ask for a feature regarding the Android Matrix SDK
+title: "[SDK] "
+labels: matrix-sdk
+assignees: ''
+
+---
+
+<!-- This issue template should be used by third party application maintainers, to report a bug or to request a feature on the SDK module of the application Element Android-->

CHANGES.md

@@ -1,4 +1,81 @@
-Changes in Element 1.0.6 (2020-XX-XX)
+Changes in Element 1.0.9 (2020-XX-XX)
+===================================================
+
+Features ✨:
+ -
+
+Improvements 🙌:
+ - PIN code: request PIN code if phone has been locked
+ - Small optimisation of scrolling experience in timeline (#2114)
+
+Bugfix 🐛:
+ - Improve support for image selection with intent changes (#1376)
+ - Fix Splash layout on small screens
+
+Translations 🗣:
+ -
+
+SDK API changes ⚠️:
+ - 
+
+Build 🧱:
+ -
+
+Other changes:
+ - Added registration/verification automated UI tests
+ - Create a script to help getting public information form any homeserver
+
+Changes in Element 1.0.8 (2020-09-25)
+===================================================
+
+Improvements 🙌:
+ - Add "show password" in import Megolm keys dialog
+ - Visually disable call buttons in menu and prohibit calling when permissions are insufficient (#2112)
+ - Better management of requested permissions (#2048)
+ - Add a setting to show timestamp for all messages (#2123)
+ - Use cache for user color
+ - Allow using an outdated homeserver, at user's risk (#1972)
+ - Restore small logo on login screens and fix scrolling issue on those screens
+ - PIN Code Improvements: Add more settings: biometrics, grace period, notification content (#1985)
+
+Bugfix 🐛:
+ - Long message cannot be sent/takes infinite time & blocks other messages (#1397)
+ - Fix crash when wellknown are malformed, or redirect to some HTML content (reported by rageshakes)
+ - User Verification in DM not working
+ - Manual import of Megolm keys does back up the imported keys
+ - Auto scrolling to the latest message when sending (#2094)
+ - Fix incorrect permission check when creating widgets (#2137)
+ - Pin code: user has to enter pin code twice (#2005)
+
+SDK API changes ⚠️:
+ - Rename `tryThis` to `tryOrNull`
+
+Other changes:
+ - Add an advanced action to reset an account data entry
+
+Changes in Element 1.0.7 (2020-09-17)
+===================================================
+
+Improvements 🙌:
+ - Handle date formatting properly (show time am/pm if needed, display year when needed)
+ - Improve F-Droid Notification (#2055)
+
+Bugfix 🐛:
+ - Clear the notification when the event is read elsewhere (#1822)
+ - Speakerphone is not used for ringback tone (#1644, #1645)
+ - Back camera preview is not mirrored anymore (#1776)
+ - Various report of people that cannot play video (#2107)
+ - Rooms incorrectly marked as unread (#588)
+ - Allow users to show/hide room member state events (#1231) 
+ - Fix stuck on loader when launching home
+
+SDK API changes ⚠️:
+ - Create a new RawService to get plain data from the server.
+
+Other changes:
+ - Performance: share Realm instance used on UI thread and improve SharedPreferences reading time.
+
+Changes in Element 1.0.6 (2020-09-08)
 ===================================================
 
 Features ✨:
@@ -7,8 +84,9 @@ Features ✨:
 Improvements 🙌:
  - You can now join room through permalink and within room directory search
  - Add long click gesture to copy userId, user display name, room name, room topic and room alias (#1774)
- - Fix several issues when uploading bug files (#1889)
+ - Fix several issues when uploading big files (#1889)
  - Do not propose to verify session if there is only one session and 4S is not configured (#1901)
+ - Call screen does not use proximity sensor (#1735)
 
 Bugfix 🐛:
  - Display name not shown under Settings/General (#1926)
@@ -21,15 +99,19 @@ Bugfix 🐛:
  - Loudspeaker is always used (#1685)
  - Fix uploads still don't work with room v6 (#1879)
  - Can't handle ongoing call events in background (#1992)
+ - Handle room, user and group links by the Element app (#1795)
+ - Update associated site domain (#1833)
  - Crash / Attachment viewer: Cannot draw a recycled Bitmap #2034
  - Login with Matrix-Id | Autodiscovery fails if identity server is invalid and Homeserver ok (#2027)
- - Improve support for image selection with intent changes (#1376)
+ - Support for image compression on Android 10
+ - Verification popup won't show
+ - Android 6: App crash when read Contact permission is granted (#2064)
+ - JSON for verification events leaks in to the room list (#1246)
+ - Replies to poll appears in timeline as unsupported events during sending (#1004)
 
 Translations 🗣:
- -
+ - The SDK is now using SAS string translations from [Weblate Matrix-doc project](https://translate.riot.im/projects/matrix-doc/) (#1909)
-
+ - New translation to kabyle
-SDK API changes ⚠️:
- - 
 
 Build 🧱:
  - Some dependencies have been upgraded (coroutine, recyclerView, appCompat, core-ktx, firebase-messaging)
@@ -37,10 +119,10 @@ Build 🧱:
     New pipeline location: https://github.com/matrix-org/pipelines/blob/master/element-android/pipeline.yml
     New build location: https://buildkite.com/matrix-dot-org/element-android
 
-
 Other changes:
  - Use File extension functions to make code more concise (#1996)
  - Create a script to import SAS strings (#1909)
+ - Support `data-mx-[bg-]color` attributes on `<font>` tags.
 
 Changes in Element 1.0.5 (2020-08-21)
 ===================================================

docs/add_threePids.md

@@ -261,11 +261,11 @@ This is not an ideal, but the client will display a hint to check the entered co
 
 200
 
-````json
+```json
 {
   "success": true
 }
-````
+```
 
 Then the app call `https://homeserver.org/_matrix/client/r0/account/3pid/add` as per adding an email and follow the same UIS flow
 

docs/signin.md

@@ -8,7 +8,9 @@ This document describes the flow of signin to a homeserver, and also the flow wh
 
 Client request the sign-in flows, once the homeserver is chosen by the user and its url is known (in the example it's `https://matrix.org`)
 
-> curl -X GET 'https://matrix.org/_matrix/client/r0/login'
+```shell script
+curl -X GET 'https://matrix.org/_matrix/client/r0/login'
+```
 
 200
 
@@ -26,7 +28,9 @@ Client request the sign-in flows, once the homeserver is chosen by the user and
 
 The user is able to connect using `m.login.password`
 
-> curl -X POST --data $'{"identifier":{"type":"m.id.user","user":"alice"},"password":"weak_password","type":"m.login.password","initial_device_display_name":"Portable"}' 'https://matrix.org/_matrix/client/r0/login'
+```shell script
+curl -X POST --data $'{"identifier":{"type":"m.id.user","user":"alice"},"password":"weak_password","type":"m.login.password","initial_device_display_name":"Portable"}' 'https://matrix.org/_matrix/client/r0/login'
+```
 
 ```json
 {
@@ -73,14 +77,16 @@ We get credential (200)
 
 If the user has associated an email with its account, he can signin using the email.
 
-> curl -X POST --data $'{"identifier":{"type":"m.id.thirdparty","medium":"email","address":"alice@yopmail.com"},"password":"weak_password","type":"m.login.password","initial_device_display_name":"Portable"}' 'https://matrix.org/_matrix/client/r0/login'
+```shell script
+curl -X POST --data $'{"identifier":{"type":"m.id.thirdparty","medium":"email","address":"alice@email-provider.org"},"password":"weak_password","type":"m.login.password","initial_device_display_name":"Portable"}' 'https://matrix.org/_matrix/client/r0/login'
+```
 
 ```json
 {
   "identifier": {
     "type": "m.id.thirdparty",
     "medium": "email",
-    "address": "alice@yopmail.com"
+    "address": "alice@email-provider.org"
   },
   "password": "weak_password",
   "type": "m.login.password",
@@ -136,7 +142,9 @@ Not supported yet in Element
 
 ### Login with SSO
 
-> curl -X GET 'https://homeserver.with.sso/_matrix/client/r0/login'
+```shell script
+curl -X GET 'https://homeserver.with.sso/_matrix/client/r0/login'
+```
 
 200
 
@@ -171,7 +179,9 @@ Once the process is finished, the web page will call the `redirectUrl` with an e
 
 This navigation is intercepted by Element by the `LoginActivity`, which will then ask the homeserver to convert this `loginToken` to an access token
 
-> curl -X POST --data $'{"type":"m.login.token","token":"MDAxOWxvY2F0aW9uIG1vemlsbGEub3JnCjAwMTNpZGVudGlmaWVy"}' 'https://homeserver.with.sso/_matrix/client/r0/login'
+```shell script
+curl -X POST --data $'{"type":"m.login.token","token":"MDAxOWxvY2F0aW9uIG1vemlsbGEub3JnCjAwMTNpZGVudGlmaWVy"}' 'https://homeserver.with.sso/_matrix/client/r0/login'
+```
 
 ```json
 {
@@ -214,7 +224,9 @@ We display a warning regarding e2e.
 
 At the first step, we do not send the password, only the email and a client secret, generated by the application
 
-> curl -X POST --data $'{"client_secret":"6c57f284-85e2-421b-8270-fb1795a120a7","send_attempt":0,"email":"user@domain.com"}' 'https://matrix.org/_matrix/client/r0/account/password/email/requestToken'
+```shell script
+curl -X POST --data $'{"client_secret":"6c57f284-85e2-421b-8270-fb1795a120a7","send_attempt":0,"email":"user@domain.com"}' 'https://matrix.org/_matrix/client/r0/account/password/email/requestToken'
+```
 
 ```json
 {
@@ -251,7 +263,9 @@ During this step, the new password is sent to the homeserver.
 
 If the user confirms before the link is clicked, we get an error:
 
-> curl -X POST --data $'{"auth":{"type":"m.login.email.identity","threepid_creds":{"client_secret":"6c57f284-85e2-421b-8270-fb1795a120a7","sid":"tQNbrREDACTEDldA"}},"new_password":"weak_password"}' 'https://matrix.org/_matrix/client/r0/account/password'
+```shell script
+curl -X POST --data $'{"auth":{"type":"m.login.email.identity","threepid_creds":{"client_secret":"6c57f284-85e2-421b-8270-fb1795a120a7","sid":"tQNbrREDACTEDldA"}},"new_password":"weak_password"}' 'https://matrix.org/_matrix/client/r0/account/password'
+```
 
 ```json
 {
@@ -285,7 +299,9 @@ It contains the client secret, a token and the sid
 
 When the user click the link, if validate his ownership and the new password can now be ent by the application (on user demand):
 
-> curl -X POST --data $'{"auth":{"type":"m.login.email.identity","threepid_creds":{"client_secret":"6c57f284-85e2-421b-8270-fb1795a120a7","sid":"tQNbrREDACTEDldA"}},"new_password":"weak_password"}' 'https://matrix.org/_matrix/client/r0/account/password'
+```shell script
+curl -X POST --data $'{"auth":{"type":"m.login.email.identity","threepid_creds":{"client_secret":"6c57f284-85e2-421b-8270-fb1795a120a7","sid":"tQNbrREDACTEDldA"}},"new_password":"weak_password"}' 'https://matrix.org/_matrix/client/r0/account/password'
+```
 
 ```json
 {

docs/signup.md

@@ -10,7 +10,9 @@ This document describes the flow of registration to a homeserver. Examples come
 
 Client request the sign-up flows, once the homeserver is chosen by the user and its url is known (in the example it's `https://matrix.org`)
 
-> curl -X POST --data $'{}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -70,7 +72,9 @@ If the registration is not possible, we get a 403
 
 The app is displaying a form to enter username and password.
 
-> curl -X POST --data $'{"initial_device_display_name":"Mobile device","username":"alice","password": "weak_password"}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{"initial_device_display_name":"Mobile device","username":"alice","password": "weak_password"}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -133,9 +137,11 @@ We get a 400:
 
 ### Step 2: entering email
 
-User is proposed to enter an email. We skip this step.
+User is proposed to enter an email. User skips this step.
 
-> curl -X POST --data $'{"auth":{"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.dummy"}}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{"auth":{"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.dummy"}}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -189,16 +195,18 @@ User is proposed to enter an email. We skip this step.
 }
 ```
 
-### Step 2 bis: we enter an email
+### Step 2 bis: user enters an email
 
 We request a token to the homeserver. The `client_secret` is generated by the application
 
-> curl -X POST --data $'{"client_secret":"53e679ea-oRED-ACTED-92b8-3012c49c6cfa","email":"alice@yopmail.com","send_attempt":0}' 'https://matrix.org/_matrix/client/r0/register/email/requestToken'
+```shell script
+curl -X POST --data $'{"client_secret":"53e679ea-oRED-ACTED-92b8-3012c49c6cfa","email":"alice@email-provider.org","send_attempt":0}' 'https://matrix.org/_matrix/client/r0/register/email/requestToken'
+```
 
 ```json
 {
   "client_secret": "53e679ea-oRED-ACTED-92b8-3012c49c6cfa",
-  "email": "alice@yopmail.com",
+  "email": "alice@email-provider.org",
   "send_attempt": 0
 }
 ```
@@ -213,7 +221,9 @@ We request a token to the homeserver. The `client_secret` is generated by the ap
 
 And
 
-> curl -X POST --data $'{"auth":{"threepid_creds":{"client_secret":"53e679ea-oRED-ACTED-92b8-3012c49c6cfa","sid":"qlBCREDACTEDEtgxD"},"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.email.identity"}}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{"auth":{"threepid_creds":{"client_secret":"53e679ea-oRED-ACTED-92b8-3012c49c6cfa","sid":"qlBCREDACTEDEtgxD"},"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.email.identity"}}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -239,7 +249,9 @@ We get 401 since the email is not validated yet:
 
 The app is now polling on 
 
-> curl -X POST --data $'{"auth":{"threepid_creds":{"client_secret":"53e679ea-oRED-ACTED-92b8-3012c49c6cfa","sid":"qlBCREDACTEDEtgxD"},"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.email.identity"}}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{"auth":{"threepid_creds":{"client_secret":"53e679ea-oRED-ACTED-92b8-3012c49c6cfa","sid":"qlBCREDACTEDEtgxD"},"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.email.identity"}}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -254,7 +266,7 @@ The app is now polling on
 }
 ```
 
-We click on the link received by email `https://matrix.org/_matrix/client/unstable/registration/email/submit_token?token=vtQjQIZfwdoREDACTEDozrmKYSWlCXsJ&client_secret=53e679ea-oRED-ACTED-92b8-3012c49c6cfa&sid=qlBCREDACTEDEtgxD` which contains:
+User clicks on the link received by email `https://matrix.org/_matrix/client/unstable/registration/email/submit_token?token=vtQjQIZfwdoREDACTEDozrmKYSWlCXsJ&client_secret=53e679ea-oRED-ACTED-92b8-3012c49c6cfa&sid=qlBCREDACTEDEtgxD` which contains:
 - A `token` vtQjQIZfwdoREDACTEDozrmKYSWlCXsJ
 - The `client_secret`: 53e679ea-oRED-ACTED-92b8-3012c49c6cfa
 - A `sid`: qlBCREDACTEDEtgxD
@@ -306,7 +318,9 @@ Once the link is clicked, the registration request (polling) returns a 401 with
 
 User is proposed to accept T&C and he accepts them
 
-> curl -X POST --data $'{"auth":{"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.terms"}}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{"auth":{"session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.terms"}}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -365,7 +379,9 @@ User is proposed to accept T&C and he accepts them
 
 User is proposed to prove he is not a robot and he does it:
 
-> curl -X POST --data $'{"auth":{"response":"03AOLTBLSiGS9GhFDpAMblJ2nlXOmHXqAYJ5OvHCPUjiVLBef3k9snOYI_BDC32-t4D2jv-tpvkaiEI_uloobFd9RUTPpJ7con2hMddbKjSCYqXqcUQFhzhbcX6kw8uBnh2sbwBe80_ihrHGXEoACXQkL0ki1Q0uEtOeW20YBRjbNABsZPpLNZhGIWC0QVXnQ4FouAtZrl3gOAiyM-oG3cgP6M9pcANIAC_7T2P2amAHbtsTlSR9CsazNyS-rtDR9b5MywdtnWN9Aw8fTJb8cXQk_j7nvugMxzofPjSOrPKcr8h5OqPlpUCyxxnFtag6cuaPSUwh43D2L0E-ZX7djzaY2Yh_U2n6HegFNPOQ22CJmfrKwDlodmAfMPvAXyq77n3HpoREDACTEDo3830RHF4BfkGXUaZjctgg-A1mvC17hmQmQpkG7IhDqyw0onU-0vF_-ehCjq_CcQEDpS_O3uiHJaG5xGf-0rhLm57v_wA3deugbsZuO4uTuxZZycN_mKxZ97jlDVBetl9hc_5REPbhcT1w3uzTCSx7Q","session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.recaptcha"}}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{"auth":{"response":"03AOLTBLSiGS9GhFDpAMblJ2nlXOmHXqAYJ5OvHCPUjiVLBef3k9snOYI_BDC32-t4D2jv-tpvkaiEI_uloobFd9RUTPpJ7con2hMddbKjSCYqXqcUQFhzhbcX6kw8uBnh2sbwBe80_ihrHGXEoACXQkL0ki1Q0uEtOeW20YBRjbNABsZPpLNZhGIWC0QVXnQ4FouAtZrl3gOAiyM-oG3cgP6M9pcANIAC_7T2P2amAHbtsTlSR9CsazNyS-rtDR9b5MywdtnWN9Aw8fTJb8cXQk_j7nvugMxzofPjSOrPKcr8h5OqPlpUCyxxnFtag6cuaPSUwh43D2L0E-ZX7djzaY2Yh_U2n6HegFNPOQ22CJmfrKwDlodmAfMPvAXyq77n3HpoREDACTEDo3830RHF4BfkGXUaZjctgg-A1mvC17hmQmQpkG7IhDqyw0onU-0vF_-ehCjq_CcQEDpS_O3uiHJaG5xGf-0rhLm57v_wA3deugbsZuO4uTuxZZycN_mKxZ97jlDVBetl9hc_5REPbhcT1w3uzTCSx7Q","session":"xptUYoREDACTEDogOWAGVnbJQ","type":"m.login.recaptcha"}}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -396,9 +412,11 @@ Some homeservers may require the user to enter MSISDN.
 
 On matrix.org, it's not required, and not even optional, but it's still possible for the app to add a MSISDN during the registration.
 
-The user enter a phone number and select a country, the `client_secret` is generated by the application
+The user enters a phone number and selects a country, the `client_secret` is generated by the application
 
-> curl -X POST --data $'{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","send_attempt":1,"country":"FR","phone_number":"+33611223344"}'  'https://matrix.org/_matrix/client/r0/register/msisdn/requestToken'
+```shell script
+curl -X POST --data $'{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","send_attempt":1,"country":"FR","phone_number":"+33611223344"}'  'https://matrix.org/_matrix/client/r0/register/msisdn/requestToken'
+```
 
 ```json
 {
@@ -430,10 +448,11 @@ If it is not the case, the homeserver send the SMS and returns some data, especi
 }
 ```
 
-When you execute the register request, with the received `sid`, you get an error since the MSISDN is not validated yet:
+When we execute the register request, with the received `sid`, we get an error since the MSISDN is not validated yet:
-
-> curl -X POST --data $'{"auth":{"type":"m.login.msisdn","session":"xptUYoREDACTEDogOWAGVnbJQ","threepid_creds":{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","sid":"1678881798"}}}' 'https://matrix.org/_matrix/client/r0/register'
 
+```shell script
+curl -X POST --data $'{"auth":{"type":"m.login.msisdn","session":"xptUYoREDACTEDogOWAGVnbJQ","threepid_creds":{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","sid":"1678881798"}}}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
   "auth": {
@@ -492,7 +511,9 @@ There is an issue on Synapse, which return a 401, it sends too much data along w
 
 The user receive the SMS, he can enter the SMS code in the app, which is sent using the "submit_url" received ie the response of the `requestToken` request:
 
-> curl -X POST --data $'{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","sid":"1678881798","token":"123456"}' 'https://matrix.org/_matrix/client/unstable/add_threepid/msisdn/submit_token'
+```shell script
+curl -X POST --data $'{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","sid":"1678881798","token":"123456"}' 'https://matrix.org/_matrix/client/unstable/add_threepid/msisdn/submit_token'
+```
 
 ```json
 {
@@ -520,7 +541,9 @@ And if the code is correct we get a 200 with:
 
 We can now execute the registration request, to the homeserver
 
-> curl -X POST --data $'{"auth":{"type":"m.login.msisdn","session":"xptUYoREDACTEDogOWAGVnbJQ","threepid_creds":{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","sid":"1678881798"}}}' 'https://matrix.org/_matrix/client/r0/register'
+```shell script
+curl -X POST --data $'{"auth":{"type":"m.login.msisdn","session":"xptUYoREDACTEDogOWAGVnbJQ","threepid_creds":{"client_secret":"d3e285f6-972a-496c-9a22-7915a2db57c7","sid":"1678881798"}}}' 'https://matrix.org/_matrix/client/r0/register'
+```
 
 ```json
 {
@@ -535,7 +558,7 @@ We can now execute the registration request, to the homeserver
 }
 ```
 
-Now the homeserver consider that the `m.login.msisdn` step is completed (401):
+Now the homeserver considers that the `m.login.msisdn` step is completed (401):
 
 ```json
 {

docs/ui-tests.md

@@ -0,0 +1,107 @@
+# Automate user interface tests
+
+Element Android ensures that some fundamental flows are properly working by running automated user interface tests.
+Ui tests are using the android [Espresso](https://developer.android.com/training/testing/espresso) library.
+
+Tests can be run on a real device, or on a virtual device (such as the emulator in Android Studio).
+
+Currently the test are covering a small set of application flows:
+	- Registration
+	- Self verification via emoji
+	- Self verification via passphrase
+
+## Prerequisites:
+
+Out of the box, the tests use one of the homeservers (located at http://localhost:8080) of the "Demo Federation of Homeservers" (https://github.com/matrix-org/synapse#running-a-demo-federation-of-synapses).
+
+You first need to follow instructions to set up Synapse in development mode at https://github.com/matrix-org/synapse#synapse-development. If you have already installed all dependencies, the steps are:
+
+```shell script
+$ git clone https://github.com/matrix-org/synapse.git
+$ cd synapse
+$ virtualenv -p python3 env
+$ source env/bin/activate
+(env) $ python -m pip install --no-use-pep517 -e .
+```
+
+Every time you want to launch these test homeservers, type:
+
+```shell script
+$ virtualenv -p python3 env
+$ source env/bin/activate
+(env) $ demo/start.sh --no-rate-limit
+```
+
+**Emulator/Device set up**
+
+When running the test via android studio on a device, you have to disable system animations in order for the test to work properly.
+
+First, ensure developer mode is enabled:
+
+- To enable developer options, tap the **Build Number** option 7 times. You can find this option in one of the following locations, depending on your Android version:
+
+	-   Android 9 (API level 28) and higher: **Settings > About Phone > Build Number**
+	-   Android 8.0.0 (API level 26) and Android 8.1.0 (API level 26): **Settings > System > About Phone > Build Number**
+	-   Android 7.1 (API level 25) and lower: **Settings > About Phone > Build Number**
+
+On your device, under **Settings > Developer options**, disable the following 3 settings:
+
+-   Window animation scale
+-   Transition animation scale
+-   Animator duration scale
+
+## Run the tests
+
+Once Synapse is running, and an emulator is running, you can run the UI tests.
+
+### From the source code
+
+Click on the green arrow in front of each test. Clicking on the arrow in front of the test class, or from the package directory does not always work (Tests not found issue).
+
+### From command line
+
+````shell script
+./gradlew vector:connectedGplayDebugAndroidTest
+````
+
+To run all the tests from the `vector` module.
+
+In case of trouble, you can try to uninstall the previous installed test APK first with this command:
+
+```shell script
+adb uninstall im.vector.app.debug.test
+```
+## Recipes
+
+We added some specific Espresso IdlingResources, and other utilities for matrix related tests
+
+### Wait for initial sync
+
+```kotlin
+// Wait for initial sync and check room list is there
+withIdlingResource(initialSyncIdlingResource(uiSession)) {
+  onView(withId(R.id.roomListContainer))
+            .check(matches(isDisplayed()))
+}
+```
+
+### Accessing current activity
+
+```kotlin
+    val activity = EspressoHelper.getCurrentActivity()!!
+    val uiSession = (activity as HomeActivity).activeSessionHolder.getActiveSession()
+```
+
+### Interact with other session
+
+It's possible to create a session via the SDK, and then use this session to interact with the one that the emulator is using (to check verifications for example)
+
+```kotlin
+@Before
+fun initAccount() {
+  val context = InstrumentationRegistry.getInstrumentation().targetContext
+  val matrix = Matrix.getInstance(context)
+  val userName = "foobar_${System.currentTimeMillis()}"
+  existingSession = createAccountAndSync(matrix, userName, password, true)
+}
+```

matrix-sdk-android/build.gradle

@@ -144,7 +144,6 @@ dependencies {
 
     // Image
     implementation 'androidx.exifinterface:exifinterface:1.3.0-alpha01'
-    implementation 'id.zelory:compressor:3.0.0'
 
     // Database
     implementation 'com.github.Zhuinden:realm-monarchy:0.5.1'

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/api/Matrix.kt

@@ -24,6 +24,7 @@ import com.zhuinden.monarchy.Monarchy
 import org.matrix.android.sdk.BuildConfig
 import org.matrix.android.sdk.api.auth.AuthenticationService
 import org.matrix.android.sdk.api.legacy.LegacySessionImporter
+import org.matrix.android.sdk.api.raw.RawService
 import org.matrix.android.sdk.common.DaggerTestMatrixComponent
 import org.matrix.android.sdk.internal.SessionManager
 import org.matrix.android.sdk.internal.network.UserAgentHolder
@@ -41,6 +42,7 @@ class Matrix private constructor(context: Context, matrixConfiguration: MatrixCo
 
     @Inject internal lateinit var legacySessionImporter: LegacySessionImporter
     @Inject internal lateinit var authenticationService: AuthenticationService
+    @Inject internal lateinit var rawService: RawService
     @Inject internal lateinit var userAgentHolder: UserAgentHolder
     @Inject internal lateinit var backgroundDetectionObserver: BackgroundDetectionObserver
     @Inject internal lateinit var olmManager: OlmManager
@@ -61,6 +63,8 @@ class Matrix private constructor(context: Context, matrixConfiguration: MatrixCo
         return authenticationService
     }
 
+    fun rawService() = rawService
+
     fun legacySessionImporter(): LegacySessionImporter {
         return legacySessionImporter
     }

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/common/CommonTestHelper.kt

@@ -218,7 +218,7 @@ class CommonTestHelper(context: Context) {
                     .createAccount(userName, password, null, it)
         }
 
-        // Preform dummy step
+        // Perform dummy step
         val registrationResult = doSync<RegistrationResult> {
             matrix.authenticationService
                     .getRegistrationWizard()

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/common/TestMatrixComponent.kt

@@ -25,8 +25,16 @@ import org.matrix.android.sdk.internal.di.MatrixComponent
 import org.matrix.android.sdk.internal.di.MatrixModule
 import org.matrix.android.sdk.internal.di.MatrixScope
 import org.matrix.android.sdk.internal.di.NetworkModule
+import org.matrix.android.sdk.internal.raw.RawModule
 
-@Component(modules = [TestModule::class, MatrixModule::class, NetworkModule::class, AuthModule::class, TestNetworkModule::class])
+@Component(modules = [
+    TestModule::class,
+    MatrixModule::class,
+    NetworkModule::class,
+    AuthModule::class,
+    RawModule::class,
+    TestNetworkModule::class
+])
 @MatrixScope
 internal interface TestMatrixComponent : MatrixComponent {
 

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/UnwedgingTest.kt

@@ -18,7 +18,7 @@ package org.matrix.android.sdk.internal.crypto
 
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import org.matrix.android.sdk.InstrumentedTest
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.toModel
@@ -212,7 +212,7 @@ class UnwedgingTest : InstrumentedTest {
         mTestHelper.waitWithLatch {
             mTestHelper.retryPeriodicallyWithLatch(it) {
                 // we should get back the key and be able to decrypt
-                val result = tryThis {
+                val result = tryOrNull {
                     bobSession.cryptoService().decryptEvent(messagesReceivedByBob[0].root, "")
                 }
                 Timber.i("## CRYPTO | testUnwedging: decrypt result  ${result?.clearEvent}")

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/gossiping/WithHeldTests.kt

@@ -20,7 +20,7 @@ import android.util.Log
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import org.matrix.android.sdk.InstrumentedTest
 import org.matrix.android.sdk.api.NoOpMatrixCallback
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.toModel
@@ -227,7 +227,7 @@ class WithHeldTests : InstrumentedTest {
             mTestHelper.retryPeriodicallyWithLatch(latch) {
                 val timeLineEvent = bobSecondSession.getRoom(testData.roomId)?.getTimeLineEvent(eventId)?.also {
                     // try to decrypt and force key request
-                    tryThis { bobSecondSession.cryptoService().decryptEvent(it.root, "") }
+                    tryOrNull { bobSecondSession.cryptoService().decryptEvent(it.root, "") }
                 }
                 sessionId = timeLineEvent?.root?.content?.toModel<EncryptedEventContent>()?.sessionId
                 timeLineEvent != null

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/Matrix.kt

@@ -25,6 +25,7 @@ import com.zhuinden.monarchy.Monarchy
 import org.matrix.android.sdk.BuildConfig
 import org.matrix.android.sdk.api.auth.AuthenticationService
 import org.matrix.android.sdk.api.legacy.LegacySessionImporter
+import org.matrix.android.sdk.api.raw.RawService
 import org.matrix.android.sdk.internal.SessionManager
 import org.matrix.android.sdk.internal.di.DaggerMatrixComponent
 import org.matrix.android.sdk.internal.network.UserAgentHolder
@@ -42,6 +43,7 @@ class Matrix private constructor(context: Context, matrixConfiguration: MatrixCo
 
     @Inject internal lateinit var legacySessionImporter: LegacySessionImporter
     @Inject internal lateinit var authenticationService: AuthenticationService
+    @Inject internal lateinit var rawService: RawService
     @Inject internal lateinit var userAgentHolder: UserAgentHolder
     @Inject internal lateinit var backgroundDetectionObserver: BackgroundDetectionObserver
     @Inject internal lateinit var olmManager: OlmManager
@@ -62,6 +64,8 @@ class Matrix private constructor(context: Context, matrixConfiguration: MatrixCo
         return authenticationService
     }
 
+    fun rawService() = rawService
+
     fun legacySessionImporter(): LegacySessionImporter {
         return legacySessionImporter
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/auth/data/LoginFlowResult.kt

@@ -17,13 +17,11 @@
 
 package org.matrix.android.sdk.api.auth.data
 
-// Either a list of supported login types, or an error if the homeserver is outdated
 sealed class LoginFlowResult {
     data class Success(
             val supportedLoginTypes: List<String>,
             val isLoginAndRegistrationSupported: Boolean,
-            val homeServerUrl: String
+            val homeServerUrl: String,
+            val isOutdatedHomeserver: Boolean
     ) : LoginFlowResult()
-
-    object OutdatedHomeserver : LoginFlowResult()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/auth/data/WellKnown.kt

@@ -42,9 +42,6 @@ import org.matrix.android.sdk.api.util.JsonDict
  *              }
  *          ]
  *    }
- *     "im.vector.riot.jitsi": {
- *         "preferredDomain": "https://jitsi.riot.im/"
- *     }
  * }
  * </pre>
  */
@@ -57,24 +54,5 @@ data class WellKnown(
         val identityServer: WellKnownBaseConfig? = null,
 
         @Json(name = "m.integrations")
-        val integrations: JsonDict? = null,
+        val integrations: JsonDict? = null
-
-        @Json(name = "im.vector.riot.e2ee")
-        val e2eAdminSetting: E2EWellKnownConfig? = null,
-
-        @Json(name = "im.vector.riot.jitsi")
-        val jitsiServer: WellKnownPreferredConfig? = null
-
-)
-
-@JsonClass(generateAdapter = true)
-data class E2EWellKnownConfig(
-        @Json(name = "default")
-        val e2eDefault: Boolean = true
-)
-
-@JsonClass(generateAdapter = true)
-data class WellKnownPreferredConfig(
-        @Json(name = "preferredDomain")
-        val preferredDomain: String? = null
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/extensions/Try.kt

@@ -19,7 +19,7 @@ package org.matrix.android.sdk.api.extensions
 
 import timber.log.Timber
 
-inline fun <A> tryThis(message: String? = null, operation: () -> A): A? {
+inline fun <A> tryOrNull(message: String? = null, operation: () -> A): A? {
     return try {
         operation()
     } catch (any: Throwable) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/failure/Extensions.kt

@@ -17,7 +17,7 @@
 
 package org.matrix.android.sdk.api.failure
 
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.internal.auth.registration.RegistrationFlowResponse
 import org.matrix.android.sdk.internal.di.MoshiProvider
 import java.io.IOException
@@ -49,7 +49,7 @@ fun Throwable.isInvalidPassword(): Boolean {
  */
 fun Throwable.toRegistrationFlowResponse(): RegistrationFlowResponse? {
     return if (this is Failure.OtherServerError && this.httpCode == 401) {
-        tryThis {
+        tryOrNull {
             MoshiProvider.providesMoshi()
                     .adapter(RegistrationFlowResponse::class.java)
                     .fromJson(this.errorBody)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/failure/MatrixError.kt

@@ -132,6 +132,8 @@ data class MatrixError(
         const val M_CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
         /** (Not documented yet) */
         const val M_WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
+        /** (Not documented yet) */
+        const val M_WEAK_PASSWORD = "M_WEAK_PASSWORD"
 
         const val M_TERMS_NOT_SIGNED = "M_TERMS_NOT_SIGNED"
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/raw/RawCacheStrategy.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.api.raw
+
+sealed class RawCacheStrategy {
+    // Data is always fetched from the server
+    object NoCache: RawCacheStrategy()
+
+    // Once data is retrieved, it is stored for the provided amount of time.
+    // In case of error, and if strict is set to false, the cache can be returned if available
+    data class TtlCache(val validityDurationInMillis: Long, val strict: Boolean): RawCacheStrategy()
+
+    // Once retrieved, the data is stored in cache and will be always get from the cache
+    object InfiniteCache: RawCacheStrategy()
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/raw/RawService.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.api.raw
+
+import org.matrix.android.sdk.api.MatrixCallback
+import org.matrix.android.sdk.api.util.Cancelable
+
+/**
+ * Useful methods to fetch raw data from the server. The access token will not be used to fetched the data
+ */
+interface RawService {
+    /**
+     * Get a URL, either from cache or from the remote server, depending on the cache strategy
+     */
+    fun getUrl(url: String,
+               rawCacheStrategy: RawCacheStrategy,
+               matrixCallback: MatrixCallback<String>): Cancelable
+
+    /**
+     * Specific case for the well-known file. Cache validity is 8 hours
+     */
+    fun getWellknown(userId: String, matrixCallback: MatrixCallback<String>): Cancelable
+
+    /**
+     * Clear all the cache data
+     */
+    fun clearCache(matrixCallback: MatrixCallback<Unit>): Cancelable
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt

@@ -110,7 +110,7 @@ interface Session :
      * This does not work in doze mode :/
      * If battery optimization is on it can work in app standby but that's all :/
      */
-    fun startAutomaticBackgroundSync(repeatDelay: Long = 30_000L)
+    fun startAutomaticBackgroundSync(timeOutInSeconds: Long, repeatDelayInSeconds: Long)
 
     fun stopAnyBackgroundSync()
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/homeserver/HomeServerCapabilities.kt

@@ -33,16 +33,7 @@ data class HomeServerCapabilities(
         /**
          * Default identity server url, provided in Wellknown
          */
-        val defaultIdentityServerUrl: String? = null,
+        val defaultIdentityServerUrl: String? = null
-        /**
-         * Option to allow homeserver admins to set the default E2EE behaviour back to disabled for DMs / private rooms
-         * (as it was before) for various environments where this is desired.
-         */
-        val adminE2EByDefault: Boolean = true,
-        /**
-         * Preferred Jitsi domain, provided in Wellknown
-         */
-        val preferredJitsiDomain: String? = null
 ) {
     companion object {
         const val MAX_UPLOAD_FILE_SIZE_UNKNOWN = -1L

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/summary/RoomSummaryConstants.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.api.session.room.summary
+
+import org.matrix.android.sdk.api.session.events.model.EventType
+
+object RoomSummaryConstants {
+
+    val PREVIEWABLE_TYPES = listOf(
+            // TODO filter message type (KEY_VERIFICATION_READY, etc.)
+            EventType.MESSAGE,
+            EventType.CALL_INVITE,
+            EventType.CALL_HANGUP,
+            EventType.CALL_ANSWER,
+            EventType.ENCRYPTED,
+            EventType.STICKER,
+            EventType.REACTION
+    )
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/timeline/TimelineEventFilters.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.api.session.room.timeline
+
+data class TimelineEventFilters(
+        /**
+         * A flag to filter edit events
+         */
+        val filterEdits: Boolean = false,
+        /**
+         * A flag to filter redacted events
+         */
+        val filterRedacted: Boolean = false,
+        /**
+         * A flag to filter useless events, such as membership events without any change
+         */
+        val filterUseless: Boolean = false,
+        /**
+         * A flag to filter by types. It should be used with [allowedTypes] field
+         */
+        val filterTypes: Boolean = false,
+        /**
+         * If [filterTypes] is true, the list of types allowed by the list.
+         */
+        val allowedTypes: List<String> = emptyList()
+)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/timeline/TimelineSettings.kt

@@ -26,25 +26,9 @@ data class TimelineSettings(
          */
         val initialSize: Int,
         /**
-         * A flag to filter edit events
+         * Filters for timeline event
          */
-        val filterEdits: Boolean = false,
+        val filters: TimelineEventFilters = TimelineEventFilters(),
-        /**
-         * A flag to filter redacted events
-         */
-        val filterRedacted: Boolean = false,
-        /**
-         * A flag to filter useless events, such as membership events without any change
-         */
-        val filterUseless: Boolean = false,
-        /**
-         * A flag to filter by types. It should be used with [allowedTypes] field
-         */
-        val filterTypes: Boolean = false,
-        /**
-         * If [filterTypes] is true, the list of types allowed by the list.
-         */
-        val allowedTypes: List<String> = emptyList(),
         /**
          * If true, will build read receipts for each event.
          */

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/auth/DefaultAuthenticationService.kt

@@ -273,16 +273,16 @@ internal class DefaultAuthenticationService @Inject constructor(
     }
 
     private suspend fun getLoginFlowResult(authAPI: AuthAPI, versions: Versions, homeServerUrl: String): LoginFlowResult {
-        return if (versions.isSupportedBySdk()) {
+        // Get the login flow
-            // Get the login flow
+        val loginFlowResponse = executeRequest<LoginFlowResponse>(null) {
-            val loginFlowResponse = executeRequest<LoginFlowResponse>(null) {
+            apiCall = authAPI.getLoginFlows()
-                apiCall = authAPI.getLoginFlows()
-            }
-            LoginFlowResult.Success(loginFlowResponse.flows.orEmpty().mapNotNull { it.type }, versions.isLoginAndRegistrationSupportedBySdk(), homeServerUrl)
-        } else {
-            // Not supported
-            LoginFlowResult.OutdatedHomeserver
         }
+        return LoginFlowResult.Success(
+                loginFlowResponse.flows.orEmpty().mapNotNull { it.type },
+                versions.isLoginAndRegistrationSupportedBySdk(),
+                homeServerUrl,
+                !versions.isSupportedBySdk()
+        )
     }
 
     override fun getRegistrationWizard(): RegistrationWizard {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/CancelGossipRequestWorker.kt

@@ -18,10 +18,9 @@
 package org.matrix.android.sdk.internal.crypto
 
 import android.content.Context
-import androidx.work.CoroutineWorker
-import androidx.work.Data
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.auth.data.Credentials
 import org.matrix.android.sdk.api.failure.shouldBeRetried
 import org.matrix.android.sdk.api.session.events.model.Event
@@ -32,28 +31,29 @@ import org.matrix.android.sdk.internal.crypto.model.MXUsersDevicesMap
 import org.matrix.android.sdk.internal.crypto.model.rest.ShareRequestCancellation
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
 import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
+import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.worker.getSessionComponent
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
-import org.greenrobot.eventbus.EventBus
+import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import timber.log.Timber
 import javax.inject.Inject
 
 internal class CancelGossipRequestWorker(context: Context,
                                          params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<CancelGossipRequestWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
-            val sessionId: String,
+            override val sessionId: String,
             val requestId: String,
-            val recipients: Map<String, List<String>>
+            val recipients: Map<String, List<String>>,
-    ) {
+            override val lastFailureMessage: String? = null
+    ) : SessionWorkerParams {
         companion object {
             fun fromRequest(sessionId: String, request: OutgoingGossipingRequest): Params {
                 return Params(
                         sessionId = sessionId,
                         requestId = request.requestId,
-                        recipients = request.recipients
+                        recipients = request.recipients,
+                        lastFailureMessage = null
                 )
             }
         }
@@ -64,18 +64,11 @@ internal class CancelGossipRequestWorker(context: Context,
     @Inject lateinit var eventBus: EventBus
     @Inject lateinit var credentials: Credentials
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val errorOutputData = Data.Builder().putBoolean("failed", true).build()
+        injector.inject(this)
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+    }
-                ?: return Result.success(errorOutputData)
-
-        val sessionComponent = getSessionComponent(params.sessionId)
-                ?: return Result.success(errorOutputData).also {
-                    // TODO, can this happen? should I update local echo?
-                    Timber.e("Unknown Session, cannot send message, sessionId: ${params.sessionId}")
-                }
-        sessionComponent.inject(this)
 
+    override suspend fun doSafeWork(params: Params): Result {
         val localId = LocalEcho.createLocalEchoId()
         val contentMap = MXUsersDevicesMap<Any>()
         val toDeviceContent = ShareRequestCancellation(
@@ -107,13 +100,17 @@ internal class CancelGossipRequestWorker(context: Context,
             )
             cryptoStore.updateOutgoingGossipingRequestState(params.requestId, OutgoingGossipingRequestState.CANCELLED)
             return Result.success()
-        } catch (exception: Throwable) {
+        } catch (throwable: Throwable) {
-            return if (exception.shouldBeRetried()) {
+            return if (throwable.shouldBeRetried()) {
                 Result.retry()
             } else {
                 cryptoStore.updateOutgoingGossipingRequestState(params.requestId, OutgoingGossipingRequestState.FAILED_TO_CANCEL)
-                Result.success(errorOutputData)
+                buildErrorResult(params, throwable.localizedMessage ?: "error")
             }
         }
     }
+
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt

@@ -27,10 +27,16 @@ import androidx.lifecycle.LiveData
 import com.squareup.moshi.Types
 import com.zhuinden.monarchy.Monarchy
 import dagger.Lazy
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.cancelChildren
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
 import org.matrix.android.sdk.api.MatrixCallback
 import org.matrix.android.sdk.api.NoOpMatrixCallback
 import org.matrix.android.sdk.api.crypto.MXCryptoConfig
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.failure.Failure
 import org.matrix.android.sdk.api.listeners.ProgressListener
 import org.matrix.android.sdk.api.session.crypto.CryptoService
@@ -102,12 +108,6 @@ import org.matrix.android.sdk.internal.task.launchToCallback
 import org.matrix.android.sdk.internal.util.JsonCanonicalizer
 import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
 import org.matrix.android.sdk.internal.util.fetchCopied
-import kotlinx.coroutines.CancellationException
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.cancelChildren
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.runBlocking
-import kotlinx.coroutines.withContext
 import org.matrix.olm.OlmManager
 import timber.log.Timber
 import java.util.concurrent.atomic.AtomicBoolean
@@ -345,13 +345,13 @@ internal class DefaultCryptoService @Inject constructor(
             // Open the store
             cryptoStore.open()
             // this can throw if no network
-            tryThis {
+            tryOrNull {
                 uploadDeviceKeys()
             }
 
             oneTimeKeysUploader.maybeUploadOneTimeKeys()
             // this can throw if no backup
-            tryThis {
+            tryOrNull {
                 keysBackupService.checkAndStartKeysBackup()
             }
         }
@@ -1072,7 +1072,11 @@ internal class DefaultCryptoService @Inject constructor(
                         throw Exception("Error")
                     }
 
-                    megolmSessionDataImporter.handle(importedSessions, true, progressListener)
+                    megolmSessionDataImporter.handle(
+                            megolmSessionsData = importedSessions,
+                            fromBackup = false,
+                            progressListener = progressListener
+                    )
                 }
             }.foldToCallback(callback)
         }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OutgoingGossipingRequestManager.kt

@@ -126,7 +126,7 @@ internal class OutgoingGossipingRequestManager @Inject constructor(
      * @param request the request
      */
     private fun sendOutgoingGossipingRequest(request: OutgoingGossipingRequest) {
-        Timber.v("## CRYPTO - GOSSIP sendOutgoingRoomKeyRequest() : Requesting keys $request")
+        Timber.v("## CRYPTO - GOSSIP sendOutgoingGossipingRequest() : Requesting keys $request")
 
         val params = SendGossipRequestWorker.Params(
                 sessionId = sessionId,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/SendGossipRequestWorker.kt

@@ -18,10 +18,9 @@
 package org.matrix.android.sdk.internal.crypto
 
 import android.content.Context
-import androidx.work.CoroutineWorker
-import androidx.work.Data
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.auth.data.Credentials
 import org.matrix.android.sdk.api.failure.shouldBeRetried
 import org.matrix.android.sdk.api.session.events.model.Event
@@ -34,40 +33,34 @@ import org.matrix.android.sdk.internal.crypto.model.rest.RoomKeyShareRequest
 import org.matrix.android.sdk.internal.crypto.model.rest.SecretShareRequest
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
 import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
+import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.worker.getSessionComponent
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
-import org.greenrobot.eventbus.EventBus
+import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import timber.log.Timber
 import javax.inject.Inject
 
 internal class SendGossipRequestWorker(context: Context,
                                        params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<SendGossipRequestWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
-            val sessionId: String,
+            override val sessionId: String,
             val keyShareRequest: OutgoingRoomKeyRequest? = null,
-            val secretShareRequest: OutgoingSecretRequest? = null
+            val secretShareRequest: OutgoingSecretRequest? = null,
-    )
+            override val lastFailureMessage: String? = null
+    ) : SessionWorkerParams
 
     @Inject lateinit var sendToDeviceTask: SendToDeviceTask
     @Inject lateinit var cryptoStore: IMXCryptoStore
     @Inject lateinit var eventBus: EventBus
     @Inject lateinit var credentials: Credentials
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val errorOutputData = Data.Builder().putBoolean("failed", true).build()
+        injector.inject(this)
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+    }
-                ?: return Result.success(errorOutputData)
-
-        val sessionComponent = getSessionComponent(params.sessionId)
-                ?: return Result.success(errorOutputData).also {
-                    // TODO, can this happen? should I update local echo?
-                    Timber.e("Unknown Session, cannot send message, sessionId: ${params.sessionId}")
-                }
-        sessionComponent.inject(this)
 
+    override suspend fun doSafeWork(params: Params): Result {
         val localId = LocalEcho.createLocalEchoId()
         val contentMap = MXUsersDevicesMap<Any>()
         val eventType: String
@@ -121,7 +114,7 @@ internal class SendGossipRequestWorker(context: Context,
                 }
             }
             else                              -> {
-                return Result.success(errorOutputData).also {
+                return buildErrorResult(params, "Unknown empty gossiping request").also {
                     Timber.e("Unknown empty gossiping request: $params")
                 }
             }
@@ -137,13 +130,17 @@ internal class SendGossipRequestWorker(context: Context,
             )
             cryptoStore.updateOutgoingGossipingRequestState(requestId, OutgoingGossipingRequestState.SENT)
             return Result.success()
-        } catch (exception: Throwable) {
+        } catch (throwable: Throwable) {
-            return if (exception.shouldBeRetried()) {
+            return if (throwable.shouldBeRetried()) {
                 Result.retry()
             } else {
                 cryptoStore.updateOutgoingGossipingRequestState(requestId, OutgoingGossipingRequestState.FAILED_TO_SEND)
-                Result.success(errorOutputData)
+                buildErrorResult(params, throwable.localizedMessage ?: "error")
             }
         }
     }
+
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/SendGossipWorker.kt

@@ -18,10 +18,9 @@
 package org.matrix.android.sdk.internal.crypto
 
 import android.content.Context
-import androidx.work.CoroutineWorker
-import androidx.work.Data
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.auth.data.Credentials
 import org.matrix.android.sdk.api.failure.shouldBeRetried
 import org.matrix.android.sdk.api.session.events.model.Event
@@ -34,22 +33,23 @@ import org.matrix.android.sdk.internal.crypto.model.MXUsersDevicesMap
 import org.matrix.android.sdk.internal.crypto.model.event.SecretSendEventContent
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
 import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
+import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.worker.getSessionComponent
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
-import org.greenrobot.eventbus.EventBus
+import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import timber.log.Timber
 import javax.inject.Inject
 
 internal class SendGossipWorker(context: Context,
                                 params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<SendGossipWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
-            val sessionId: String,
+            override val sessionId: String,
             val secretValue: String,
-            val request: IncomingSecretShareRequest
+            val request: IncomingSecretShareRequest,
-    )
+            override val lastFailureMessage: String? = null
+    ) : SessionWorkerParams
 
     @Inject lateinit var sendToDeviceTask: SendToDeviceTask
     @Inject lateinit var cryptoStore: IMXCryptoStore
@@ -58,18 +58,11 @@ internal class SendGossipWorker(context: Context,
     @Inject lateinit var messageEncrypter: MessageEncrypter
     @Inject lateinit var ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val errorOutputData = Data.Builder().putBoolean("failed", true).build()
+        injector.inject(this)
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+    }
-                ?: return Result.success(errorOutputData)
-
-        val sessionComponent = getSessionComponent(params.sessionId)
-                ?: return Result.success(errorOutputData).also {
-                    // TODO, can this happen? should I update local echo?
-                    Timber.e("Unknown Session, cannot send message, sessionId: ${params.sessionId}")
-                }
-        sessionComponent.inject(this)
 
+    override suspend fun doSafeWork(params: Params): Result {
         val localId = LocalEcho.createLocalEchoId()
         val eventType: String = EventType.SEND_SECRET
 
@@ -81,7 +74,7 @@ internal class SendGossipWorker(context: Context,
         val requestingUserId = params.request.userId ?: ""
         val requestingDeviceId = params.request.deviceId ?: ""
         val deviceInfo = cryptoStore.getUserDevice(requestingUserId, requestingDeviceId)
-                ?: return Result.success(errorOutputData).also {
+                ?: return buildErrorResult(params, "Unknown deviceInfo, cannot send message").also {
                     cryptoStore.updateGossipingRequestState(params.request, GossipingRequestState.FAILED_TO_ACCEPTED)
                     Timber.e("Unknown deviceInfo, cannot send message, sessionId: ${params.request.deviceId}")
                 }
@@ -94,7 +87,7 @@ internal class SendGossipWorker(context: Context,
         if (olmSessionResult?.sessionId == null) {
             // no session with this device, probably because there
             // were no one-time keys.
-            return Result.success(errorOutputData).also {
+            return buildErrorResult(params, "no session with this device").also {
                 cryptoStore.updateGossipingRequestState(params.request, GossipingRequestState.FAILED_TO_ACCEPTED)
                 Timber.e("no session with this device, probably because there were no one-time keys.")
             }
@@ -130,13 +123,17 @@ internal class SendGossipWorker(context: Context,
             )
             cryptoStore.updateGossipingRequestState(params.request, GossipingRequestState.ACCEPTED)
             return Result.success()
-        } catch (exception: Throwable) {
+        } catch (throwable: Throwable) {
-            return if (exception.shouldBeRetried()) {
+            return if (throwable.shouldBeRetried()) {
                 Result.retry()
             } else {
                 cryptoStore.updateGossipingRequestState(params.request, GossipingRequestState.FAILED_TO_ACCEPTED)
-                Result.success(errorOutputData)
+                buildErrorResult(params, throwable.localizedMessage ?: "error")
             }
         }
     }
+
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/actions/MegolmSessionDataImporter.kt

@@ -39,7 +39,7 @@ internal class MegolmSessionDataImporter @Inject constructor(private val olmDevi
      * Must be call on the crypto coroutine thread
      *
      * @param megolmSessionsData megolm sessions.
-     * @param backUpKeys         true to back up them to the homeserver.
+     * @param fromBackup         true if the imported keys are already backed up on the server.
      * @param progressListener   the progress listener
      * @return import room keys result
      */

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStore.kt

@@ -20,6 +20,10 @@ package org.matrix.android.sdk.internal.crypto.store.db
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.Transformations
 import com.zhuinden.monarchy.Monarchy
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import io.realm.Sort
+import io.realm.kotlin.where
 import org.matrix.android.sdk.api.auth.data.Credentials
 import org.matrix.android.sdk.api.session.crypto.crosssigning.MXCrossSigningInfo
 import org.matrix.android.sdk.api.session.events.model.Event
@@ -85,10 +89,6 @@ import org.matrix.android.sdk.internal.database.mapper.ContentMapper
 import org.matrix.android.sdk.internal.di.CryptoDatabase
 import org.matrix.android.sdk.internal.di.MoshiProvider
 import org.matrix.android.sdk.internal.session.SessionScope
-import io.realm.Realm
-import io.realm.RealmConfiguration
-import io.realm.Sort
-import io.realm.kotlin.where
 import org.matrix.olm.OlmAccount
 import org.matrix.olm.OlmException
 import timber.log.Timber
@@ -372,6 +372,7 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun storePrivateKeysInfo(msk: String?, usk: String?, ssk: String?) {
+        Timber.v("## CRYPTO | *** storePrivateKeysInfo ${msk != null}, ${usk != null}, ${ssk != null}")
         doRealmTransaction(realmConfiguration) { realm ->
             realm.where<CryptoMetadataEntity>().findFirst()?.apply {
                 xSignMasterPrivateKey = msk
@@ -407,6 +408,7 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun storeMSKPrivateKey(msk: String?) {
+        Timber.v("## CRYPTO | *** storeMSKPrivateKey ${msk != null} ")
         doRealmTransaction(realmConfiguration) { realm ->
             realm.where<CryptoMetadataEntity>().findFirst()?.apply {
                 xSignMasterPrivateKey = msk
@@ -415,6 +417,7 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun storeSSKPrivateKey(ssk: String?) {
+        Timber.v("## CRYPTO | *** storeSSKPrivateKey ${ssk != null} ")
         doRealmTransaction(realmConfiguration) { realm ->
             realm.where<CryptoMetadataEntity>().findFirst()?.apply {
                 xSignSelfSignedPrivateKey = ssk
@@ -423,6 +426,7 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun storeUSKPrivateKey(usk: String?) {
+        Timber.v("## CRYPTO | *** storeUSKPrivateKey ${usk != null} ")
         doRealmTransaction(realmConfiguration) { realm ->
             realm.where<CryptoMetadataEntity>().findFirst()?.apply {
                 xSignUserPrivateKey = usk
@@ -541,7 +545,7 @@ internal class RealmCryptoStore @Inject constructor(
                     deviceId = it.deviceId
             )
         }
-        monarchy.writeAsync { realm ->
+        doRealmTransactionAsync(realmConfiguration) { realm ->
             realm.where<MyDeviceLastSeenInfoEntity>().findAll().deleteAllFromRealm()
             entities.forEach {
                 realm.insertOrUpdate(it)
@@ -1191,7 +1195,7 @@ internal class RealmCryptoStore @Inject constructor(
                     .findAll()
                     .mapNotNull { entity ->
                         when (entity.type) {
-                            GossipRequestType.KEY    -> {
+                            GossipRequestType.KEY -> {
                                 IncomingRoomKeyRequest(
                                         userId = entity.otherUserId,
                                         deviceId = entity.otherDeviceId,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt

@@ -19,7 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.db
 
 import com.squareup.moshi.Moshi
 import com.squareup.moshi.Types
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.internal.crypto.model.MXDeviceInfo
 import org.matrix.android.sdk.internal.crypto.model.OlmInboundGroupSessionWrapper
@@ -398,7 +398,7 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
                 ?.addField(DeviceInfoEntityFields.FIRST_TIME_SEEN_LOCAL_TS, Long::class.java)
                 ?.setNullable(DeviceInfoEntityFields.FIRST_TIME_SEEN_LOCAL_TS, true)
                 ?.transform { deviceInfoEntity ->
-                    tryThis {
+                    tryOrNull {
                         deviceInfoEntity.setLong(DeviceInfoEntityFields.FIRST_TIME_SEEN_LOCAL_TS, now)
                     }
                 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/model/IncomingGossipingRequestEntity.kt

@@ -17,7 +17,7 @@
 
 package org.matrix.android.sdk.internal.crypto.store.db.model
 
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.internal.crypto.GossipRequestType
 import org.matrix.android.sdk.internal.crypto.GossipingRequestState
 import org.matrix.android.sdk.internal.crypto.IncomingRoomKeyRequest
@@ -45,7 +45,7 @@ internal open class IncomingGossipingRequestEntity(@Index var requestId: String?
 
     var type: GossipRequestType
         get() {
-            return tryThis { typeStr?.let { GossipRequestType.valueOf(it) } } ?: GossipRequestType.KEY
+            return tryOrNull { typeStr?.let { GossipRequestType.valueOf(it) } } ?: GossipRequestType.KEY
         }
         set(value) {
             typeStr = value.name
@@ -55,7 +55,7 @@ internal open class IncomingGossipingRequestEntity(@Index var requestId: String?
 
     var requestState: GossipingRequestState
         get() {
-            return tryThis { GossipingRequestState.valueOf(requestStateStr) }
+            return tryOrNull { GossipingRequestState.valueOf(requestStateStr) }
                     ?: GossipingRequestState.NONE
         }
         set(value) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/model/OutgoingGossipingRequestEntity.kt

@@ -19,7 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.db.model
 
 import com.squareup.moshi.JsonAdapter
 import com.squareup.moshi.Types
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.internal.crypto.GossipRequestType
 import org.matrix.android.sdk.internal.crypto.OutgoingGossipingRequest
 import org.matrix.android.sdk.internal.crypto.OutgoingGossipingRequestState
@@ -47,7 +47,7 @@ internal open class OutgoingGossipingRequestEntity(
 
     var type: GossipRequestType
         get() {
-            return tryThis { typeStr?.let { GossipRequestType.valueOf(it) } } ?: GossipRequestType.KEY
+            return tryOrNull { typeStr?.let { GossipRequestType.valueOf(it) } } ?: GossipRequestType.KEY
         }
         set(value) {
             typeStr = value.name
@@ -57,7 +57,7 @@ internal open class OutgoingGossipingRequestEntity(
 
     var requestState: OutgoingGossipingRequestState
         get() {
-            return tryThis { OutgoingGossipingRequestState.valueOf(requestStateStr) }
+            return tryOrNull { OutgoingGossipingRequestState.valueOf(requestStateStr) }
                     ?: OutgoingGossipingRequestState.UNSENT
         }
         set(value) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/verification/SendVerificationMessageWorker.kt

@@ -17,17 +17,17 @@
 package org.matrix.android.sdk.internal.crypto.verification
 
 import android.content.Context
-import androidx.work.CoroutineWorker
 import androidx.work.Data
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
 import org.matrix.android.sdk.api.failure.shouldBeRetried
 import org.matrix.android.sdk.api.session.crypto.CryptoService
-import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.internal.crypto.tasks.SendVerificationMessageTask
+import org.matrix.android.sdk.internal.session.SessionComponent
+import org.matrix.android.sdk.internal.session.room.send.CancelSendTracker
+import org.matrix.android.sdk.internal.session.room.send.LocalEchoRepository
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -37,56 +37,56 @@ import javax.inject.Inject
  */
 internal class SendVerificationMessageWorker(context: Context,
                                              params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<SendVerificationMessageWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
             override val sessionId: String,
-            val event: Event,
+            val eventId: String,
             override val lastFailureMessage: String? = null
     ) : SessionWorkerParams
 
-    @Inject
+    @Inject lateinit var sendVerificationMessageTask: SendVerificationMessageTask
-    lateinit var sendVerificationMessageTask: SendVerificationMessageTask
+    @Inject lateinit var localEchoRepository: LocalEchoRepository
+    @Inject lateinit var cryptoService: CryptoService
+    @Inject lateinit var cancelSendTracker: CancelSendTracker
 
-    @Inject
+    override fun injectWith(injector: SessionComponent) {
-    lateinit var cryptoService: CryptoService
+        injector.inject(this)
+    }
 
-    override suspend fun doWork(): Result {
+    override suspend fun doSafeWork(params: Params): Result {
-        val errorOutputData = Data.Builder().putBoolean(OUTPUT_KEY_FAILED, true).build()
+        val localEvent = localEchoRepository.getUpToDateEcho(params.eventId) ?: return buildErrorResult(params, "Event not found")
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+        val localEventId = localEvent.eventId ?: ""
-                ?: return Result.success(errorOutputData)
+        val roomId = localEvent.roomId ?: ""
+
+        if (cancelSendTracker.isCancelRequestedFor(localEventId, roomId)) {
+            return Result.success()
+                    .also {
+                        cancelSendTracker.markCancelled(localEventId, roomId)
+                        Timber.e("## SendEvent: Event sending has been cancelled $localEventId")
+                    }
+        }
 
-        val sessionComponent = getSessionComponent(params.sessionId)
-                ?: return Result.success(errorOutputData).also {
-                    // TODO, can this happen? should I update local echo?
-                    Timber.e("Unknown Session, cannot send message, sessionId: ${params.sessionId}")
-                }
-        sessionComponent.inject(this)
-        val localId = params.event.eventId ?: ""
         return try {
-            val eventId = sendVerificationMessageTask.execute(
+            val resultEventId = sendVerificationMessageTask.execute(
                     SendVerificationMessageTask.Params(
-                            event = params.event,
+                            event = localEvent,
                             cryptoService = cryptoService
                     )
             )
 
-            Result.success(Data.Builder().putString(localId, eventId).build())
+            Result.success(Data.Builder().putString(localEventId, resultEventId).build())
-        } catch (exception: Throwable) {
+        } catch (throwable: Throwable) {
-            if (exception.shouldBeRetried()) {
+            if (throwable.shouldBeRetried()) {
                 Result.retry()
             } else {
-                Result.success(errorOutputData)
+                buildErrorResult(params, throwable.localizedMessage ?: "error")
             }
         }
     }
 
-    companion object {
+    override fun buildErrorParams(params: Params, message: String): Params {
-        private const val OUTPUT_KEY_FAILED = "failed"
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
-
-        fun hasFailed(outputData: Data): Boolean {
-            return outputData.getBoolean(OUTPUT_KEY_FAILED, false)
-        }
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/verification/VerificationEmoji.kt

@@ -55,14 +55,14 @@ internal fun getEmojiForCode(code: Int): EmojiRepresentation {
         31            -> EmojiRepresentation("🤖", R.string.verification_emoji_robot, R.drawable.ic_verification_robot)
         32            -> EmojiRepresentation("🎩", R.string.verification_emoji_hat, R.drawable.ic_verification_hat)
         33            -> EmojiRepresentation("👓", R.string.verification_emoji_glasses, R.drawable.ic_verification_glasses)
-        34            -> EmojiRepresentation("🔧", R.string.verification_emoji_wrench, R.drawable.ic_verification_wrench)
+        34            -> EmojiRepresentation("🔧", R.string.verification_emoji_spanner, R.drawable.ic_verification_spanner)
         35            -> EmojiRepresentation("🎅", R.string.verification_emoji_santa, R.drawable.ic_verification_santa)
-        36            -> EmojiRepresentation("👍", R.string.verification_emoji_thumbsup, R.drawable.ic_verification_thumbs_up)
+        36            -> EmojiRepresentation("👍", R.string.verification_emoji_thumbs_up, R.drawable.ic_verification_thumbs_up)
         37            -> EmojiRepresentation("☂️", R.string.verification_emoji_umbrella, R.drawable.ic_verification_umbrella)
         38            -> EmojiRepresentation("⌛", R.string.verification_emoji_hourglass, R.drawable.ic_verification_hourglass)
         39            -> EmojiRepresentation("⏰", R.string.verification_emoji_clock, R.drawable.ic_verification_clock)
         40            -> EmojiRepresentation("🎁", R.string.verification_emoji_gift, R.drawable.ic_verification_gift)
-        41            -> EmojiRepresentation("💡", R.string.verification_emoji_lightbulb, R.drawable.ic_verification_light_bulb)
+        41            -> EmojiRepresentation("💡", R.string.verification_emoji_light_bulb, R.drawable.ic_verification_light_bulb)
         42            -> EmojiRepresentation("📕", R.string.verification_emoji_book, R.drawable.ic_verification_book)
         43            -> EmojiRepresentation("✏️", R.string.verification_emoji_pencil, R.drawable.ic_verification_pencil)
         44            -> EmojiRepresentation("📎", R.string.verification_emoji_paperclip, R.drawable.ic_verification_paperclip)
@@ -74,7 +74,7 @@ internal fun getEmojiForCode(code: Int): EmojiRepresentation {
         50            -> EmojiRepresentation("🏁", R.string.verification_emoji_flag, R.drawable.ic_verification_flag)
         51            -> EmojiRepresentation("🚂", R.string.verification_emoji_train, R.drawable.ic_verification_train)
         52            -> EmojiRepresentation("🚲", R.string.verification_emoji_bicycle, R.drawable.ic_verification_bicycle)
-        53            -> EmojiRepresentation("✈️", R.string.verification_emoji_airplane, R.drawable.ic_verification_airplane)
+        53            -> EmojiRepresentation("✈️", R.string.verification_emoji_aeroplane, R.drawable.ic_verification_aeroplane)
         54            -> EmojiRepresentation("🚀", R.string.verification_emoji_rocket, R.drawable.ic_verification_rocket)
         55            -> EmojiRepresentation("🏆", R.string.verification_emoji_trophy, R.drawable.ic_verification_trophy)
         56            -> EmojiRepresentation("⚽", R.string.verification_emoji_ball, R.drawable.ic_verification_ball)
@@ -82,7 +82,7 @@ internal fun getEmojiForCode(code: Int): EmojiRepresentation {
         58            -> EmojiRepresentation("🎺", R.string.verification_emoji_trumpet, R.drawable.ic_verification_trumpet)
         59            -> EmojiRepresentation("🔔", R.string.verification_emoji_bell, R.drawable.ic_verification_bell)
         60            -> EmojiRepresentation("⚓", R.string.verification_emoji_anchor, R.drawable.ic_verification_anchor)
-        61            -> EmojiRepresentation("🎧", R.string.verification_emoji_headphone, R.drawable.ic_verification_headphone)
+        61            -> EmojiRepresentation("🎧", R.string.verification_emoji_headphones, R.drawable.ic_verification_headphones)
         62            -> EmojiRepresentation("📁", R.string.verification_emoji_folder, R.drawable.ic_verification_folder)
         /* 63 */ else -> EmojiRepresentation("📌", R.string.verification_emoji_pin, R.drawable.ic_verification_pin)
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/verification/VerificationTransportRoomMessage.kt

@@ -22,6 +22,9 @@ import androidx.work.Data
 import androidx.work.ExistingWorkPolicy
 import androidx.work.Operation
 import androidx.work.WorkInfo
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import org.matrix.android.sdk.R
 import org.matrix.android.sdk.api.session.crypto.verification.CancelCode
 import org.matrix.android.sdk.api.session.crypto.verification.ValidVerificationInfoRequest
@@ -51,10 +54,8 @@ import org.matrix.android.sdk.internal.di.WorkManagerProvider
 import org.matrix.android.sdk.internal.session.room.send.LocalEchoEventFactory
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.util.StringProvider
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.launch
 import timber.log.Timber
 import java.util.UUID
 import java.util.concurrent.TimeUnit
@@ -87,7 +88,7 @@ internal class VerificationTransportRoomMessage(
 
         val workerParams = WorkerParamsFactory.toData(SendVerificationMessageWorker.Params(
                 sessionId = sessionId,
-                event = event
+                eventId = event.eventId ?: ""
         ))
         val enqueueInfo = enqueueSendWork(workerParams)
 
@@ -115,20 +116,30 @@ internal class VerificationTransportRoomMessage(
         val observer = object : Observer<List<WorkInfo>> {
             override fun onChanged(workInfoList: List<WorkInfo>?) {
                 workInfoList
-                        ?.filter { it.state == WorkInfo.State.SUCCEEDED }
                         ?.firstOrNull { it.id == enqueueInfo.second }
                         ?.let { wInfo ->
-                            if (SendVerificationMessageWorker.hasFailed(wInfo.outputData)) {
+                            when (wInfo.state) {
-                                Timber.e("## SAS verification [${tx?.transactionId}] failed to send verification message in state : ${tx?.state}")
+                                WorkInfo.State.FAILED -> {
-                                tx?.cancel(onErrorReason)
+                                    tx?.cancel(onErrorReason)
-                            } else {
+                                    workLiveData.removeObserver(this)
-                                if (onDone != null) {
+                                }
-                                    onDone()
+                                WorkInfo.State.SUCCEEDED -> {
-                                } else {
+                                    if (SessionSafeCoroutineWorker.hasFailed(wInfo.outputData)) {
-                                    tx?.state = nextState
+                                        Timber.e("## SAS verification [${tx?.transactionId}] failed to send verification message in state : ${tx?.state}")
+                                        tx?.cancel(onErrorReason)
+                                    } else {
+                                        if (onDone != null) {
+                                            onDone()
+                                        } else {
+                                            tx?.state = nextState
+                                        }
+                                    }
+                                    workLiveData.removeObserver(this)
+                                }
+                                else -> {
+                                    // nop
                                 }
                             }
-                            workLiveData.removeObserver(this)
                         }
             }
         }
@@ -174,7 +185,7 @@ internal class VerificationTransportRoomMessage(
 
         val workerParams = WorkerParamsFactory.toData(SendVerificationMessageWorker.Params(
                 sessionId = sessionId,
-                event = event
+                eventId = event.eventId ?: ""
         ))
 
         val workRequest = workManagerProvider.matrixOneTimeWorkRequestBuilder<SendVerificationMessageWorker>()
@@ -184,7 +195,7 @@ internal class VerificationTransportRoomMessage(
                 .build()
 
         workManagerProvider.workManager
-                .beginUniqueWork("${roomId}_VerificationWork", ExistingWorkPolicy.APPEND, workRequest)
+                .beginUniqueWork("${roomId}_VerificationWork", ExistingWorkPolicy.APPEND_OR_REPLACE, workRequest)
                 .enqueue()
 
         // I cannot just listen to the given work request, because when used in a uniqueWork,
@@ -199,7 +210,7 @@ internal class VerificationTransportRoomMessage(
                         ?.filter { it.state == WorkInfo.State.SUCCEEDED }
                         ?.firstOrNull { it.id == workRequest.id }
                         ?.let { wInfo ->
-                            if (SendVerificationMessageWorker.hasFailed(wInfo.outputData)) {
+                            if (SessionSafeCoroutineWorker.hasFailed(wInfo.outputData)) {
                                 callback(null, null)
                             } else {
                                 val eventId = wInfo.outputData.getString(localId)
@@ -229,7 +240,7 @@ internal class VerificationTransportRoomMessage(
         )
         val workerParams = WorkerParamsFactory.toData(SendVerificationMessageWorker.Params(
                 sessionId = sessionId,
-                event = event
+                eventId = event.eventId ?: ""
         ))
         enqueueSendWork(workerParams)
     }
@@ -249,7 +260,7 @@ internal class VerificationTransportRoomMessage(
         )
         val workerParams = WorkerParamsFactory.toData(SendVerificationMessageWorker.Params(
                 sessionId = sessionId,
-                event = event
+                eventId = event.eventId ?: ""
         ))
         val enqueueInfo = enqueueSendWork(workerParams)
 
@@ -280,7 +291,7 @@ internal class VerificationTransportRoomMessage(
                 .setBackoffCriteria(BackoffPolicy.LINEAR, 2_000L, TimeUnit.MILLISECONDS)
                 .build()
         return workManagerProvider.workManager
-                .beginUniqueWork(uniqueQueueName(), ExistingWorkPolicy.APPEND, workRequest)
+                .beginUniqueWork(uniqueQueueName(), ExistingWorkPolicy.APPEND_OR_REPLACE, workRequest)
                 .enqueue() to workRequest.id
     }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/AsyncTransaction.kt

@@ -16,31 +16,52 @@
  */
 package org.matrix.android.sdk.internal.database
 
+import com.zhuinden.monarchy.Monarchy
 import io.realm.Realm
 import io.realm.RealmConfiguration
+import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
 import kotlinx.coroutines.withContext
 import timber.log.Timber
 
-suspend fun <T> awaitTransaction(config: RealmConfiguration, transaction: suspend (realm: Realm) -> T) = withContext(Dispatchers.Default) {
+internal fun <T> CoroutineScope.asyncTransaction(monarchy: Monarchy, transaction: suspend (realm: Realm) -> T) {
-    Realm.getInstance(config).use { bgRealm ->
+    asyncTransaction(monarchy.realmConfiguration, transaction)
-        bgRealm.beginTransaction()
+}
-        val result: T
+
-        try {
+internal fun <T> CoroutineScope.asyncTransaction(realmConfiguration: RealmConfiguration, transaction: suspend (realm: Realm) -> T) {
-            val start = System.currentTimeMillis()
+    launch {
-            result = transaction(bgRealm)
+        awaitTransaction(realmConfiguration, transaction)
-            if (isActive) {
+    }
-                bgRealm.commitTransaction()
+}
-                val end = System.currentTimeMillis()
+
-                val time = end - start
+private val realmSemaphore = Semaphore(1)
-                Timber.v("Execute transaction in $time millis")
+
-            }
+suspend fun <T> awaitTransaction(config: RealmConfiguration, transaction: suspend (realm: Realm) -> T): T {
-        } finally {
+    return realmSemaphore.withPermit {
-            if (bgRealm.isInTransaction) {
+        withContext(Dispatchers.IO) {
-                bgRealm.cancelTransaction()
+            Realm.getInstance(config).use { bgRealm ->
-            }
+                bgRealm.beginTransaction()
-        }
+                val result: T
-        result
+                try {
+                    val start = System.currentTimeMillis()
+                    result = transaction(bgRealm)
+                    if (isActive) {
+                        bgRealm.commitTransaction()
+                        val end = System.currentTimeMillis()
+                        val time = end - start
+                        Timber.v("Execute transaction in $time millis")
+                    }
+                } finally {
+                    if (bgRealm.isInTransaction) {
+                        bgRealm.cancelTransaction()
+                    }
+                }
+                result
+            }
+        }
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/RealmInstanceWrapper.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database
+
+import io.realm.Realm
+import java.io.Closeable
+
+internal class RealmInstanceWrapper(private val realm: Realm, private val closeRealmOnClose: Boolean) : Closeable {
+
+    override fun close() {
+        if (closeRealmOnClose) {
+            realm.close()
+        }
+    }
+
+    fun <R> withRealm(block: (Realm) -> R): R {
+        return use {
+            block(it.realm)
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/RealmSessionProvider.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database
+
+import android.os.Looper
+import androidx.annotation.MainThread
+import com.zhuinden.monarchy.Monarchy
+import io.realm.Realm
+import org.matrix.android.sdk.internal.di.SessionDatabase
+import org.matrix.android.sdk.internal.session.SessionLifecycleObserver
+import org.matrix.android.sdk.internal.session.SessionScope
+import javax.inject.Inject
+import kotlin.concurrent.getOrSet
+
+/**
+ * This class keeps an instance of realm open in the main thread so you can grab it whenever you want to get a realm
+ * instance. This does check each time if you are on the main thread or not and returns the appropriate realm instance.
+ */
+@SessionScope
+internal class RealmSessionProvider @Inject constructor(@SessionDatabase private val monarchy: Monarchy)
+    : SessionLifecycleObserver {
+
+    private val realmThreadLocal = ThreadLocal<Realm>()
+
+    /**
+     * Allow you to execute a block with an opened realm. It automatically closes it if necessary (ie. when not in main thread)
+     */
+    fun <R> withRealm(block: (Realm) -> R): R {
+        return getRealmWrapper().withRealm(block)
+    }
+
+    @MainThread
+    override fun onStart() {
+        realmThreadLocal.getOrSet {
+            Realm.getInstance(monarchy.realmConfiguration)
+        }
+    }
+
+    @MainThread
+    override fun onStop() {
+        realmThreadLocal.get()?.close()
+        realmThreadLocal.remove()
+    }
+
+    private fun getRealmWrapper(): RealmInstanceWrapper {
+        val isOnMainThread = isOnMainThread()
+        val realm = if (isOnMainThread) {
+            realmThreadLocal.getOrSet {
+                Realm.getInstance(monarchy.realmConfiguration)
+            }
+        } else {
+            Realm.getInstance(monarchy.realmConfiguration)
+        }
+        return RealmInstanceWrapper(realm, closeRealmOnClose = !isOnMainThread)
+    }
+
+    private fun isOnMainThread() = Looper.myLooper() == Looper.getMainLooper()
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/RealmSessionStoreMigration.kt

@@ -28,7 +28,7 @@ import javax.inject.Inject
 class RealmSessionStoreMigration @Inject constructor() : RealmMigration {
 
     companion object {
-        const val SESSION_STORE_SCHEMA_VERSION = 4L
+        const val SESSION_STORE_SCHEMA_VERSION = 5L
     }
 
     override fun migrate(realm: DynamicRealm, oldVersion: Long, newVersion: Long) {
@@ -38,6 +38,7 @@ class RealmSessionStoreMigration @Inject constructor() : RealmMigration {
         if (oldVersion <= 1) migrateTo2(realm)
         if (oldVersion <= 2) migrateTo3(realm)
         if (oldVersion <= 3) migrateTo4(realm)
+        if (oldVersion <= 4) migrateTo5(realm)
     }
 
     private fun migrateTo1(realm: DynamicRealm) {
@@ -54,16 +55,16 @@ class RealmSessionStoreMigration @Inject constructor() : RealmMigration {
     private fun migrateTo2(realm: DynamicRealm) {
         Timber.d("Step 1 -> 2")
         realm.schema.get("HomeServerCapabilitiesEntity")
-                ?.addField(HomeServerCapabilitiesEntityFields.ADMIN_E2_E_BY_DEFAULT, Boolean::class.java)
+                ?.addField("adminE2EByDefault", Boolean::class.java)
                 ?.transform { obj ->
-                    obj.setBoolean(HomeServerCapabilitiesEntityFields.ADMIN_E2_E_BY_DEFAULT, true)
+                    obj.setBoolean("adminE2EByDefault", true)
                 }
     }
 
     private fun migrateTo3(realm: DynamicRealm) {
         Timber.d("Step 2 -> 3")
         realm.schema.get("HomeServerCapabilitiesEntity")
-                ?.addField(HomeServerCapabilitiesEntityFields.PREFERRED_JITSI_DOMAIN, String::class.java)
+                ?.addField("preferredJitsiDomain", String::class.java)
                 ?.transform { obj ->
                     // Schedule a refresh of the capabilities
                     obj.setLong(HomeServerCapabilitiesEntityFields.LAST_UPDATED_TIMESTAMP, 0)
@@ -82,4 +83,11 @@ class RealmSessionStoreMigration @Inject constructor() : RealmMigration {
                 .setRequired(PendingThreePidEntityFields.SID, true)
                 .addField(PendingThreePidEntityFields.SUBMIT_URL, String::class.java)
     }
+
+    private fun migrateTo5(realm: DynamicRealm) {
+        Timber.d("Step 4 -> 5")
+        realm.schema.get("HomeServerCapabilitiesEntity")
+                ?.removeField("adminE2EByDefault")
+                ?.removeField("preferredJitsiDomain")
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/mapper/ContentMapper.kt

@@ -20,21 +20,34 @@ package org.matrix.android.sdk.internal.database.mapper
 import org.matrix.android.sdk.api.session.events.model.Content
 import org.matrix.android.sdk.api.util.JSON_DICT_PARAMETERIZED_TYPE
 import org.matrix.android.sdk.internal.di.MoshiProvider
+import org.matrix.android.sdk.internal.network.parsing.CheckNumberType
 
 internal object ContentMapper {
 
     private val moshi = MoshiProvider.providesMoshi()
-    private val adapter = moshi.adapter<Content>(JSON_DICT_PARAMETERIZED_TYPE)
+    private val castJsonNumberMoshi by lazy {
+        // We are adding the CheckNumberType as we are serializing/deserializing multiple time in a row
+        // and we lost typing information doing so.
+        // We don't want this check to be done on all adapters, so we create a new moshi just for that.
+        MoshiProvider.providesMoshi()
+                .newBuilder()
+                .add(CheckNumberType.JSON_ADAPTER_FACTORY)
+                .build()
+    }
 
-    fun map(content: String?): Content? {
+    fun map(content: String?, castJsonNumbers: Boolean = false): Content? {
         return content?.let {
-            adapter.fromJson(it)
+            if (castJsonNumbers) {
+                castJsonNumberMoshi
+            } else {
+                moshi
+            }.adapter<Content>(JSON_DICT_PARAMETERIZED_TYPE).fromJson(it)
         }
     }
 
     fun map(content: Content?): String? {
         return content?.let {
-            adapter.toJson(it)
+            moshi.adapter<Content>(JSON_DICT_PARAMETERIZED_TYPE).toJson(it)
         }
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/mapper/EventMapper.kt

@@ -54,7 +54,7 @@ internal object EventMapper {
         return eventEntity
     }
 
-    fun map(eventEntity: EventEntity): Event {
+    fun map(eventEntity: EventEntity, castJsonNumbers: Boolean = false): Event {
         val ud = eventEntity.unsignedData
                 ?.takeIf { it.isNotBlank() }
                 ?.let {
@@ -69,8 +69,8 @@ internal object EventMapper {
         return Event(
                 type = eventEntity.type,
                 eventId = eventEntity.eventId,
-                content = ContentMapper.map(eventEntity.content),
+                content = ContentMapper.map(eventEntity.content, castJsonNumbers),
-                prevContent = ContentMapper.map(eventEntity.prevContent),
+                prevContent = ContentMapper.map(eventEntity.prevContent, castJsonNumbers),
                 originServerTs = eventEntity.originServerTs,
                 senderId = eventEntity.sender,
                 stateKey = eventEntity.stateKey,
@@ -96,8 +96,8 @@ internal object EventMapper {
     }
 }
 
-internal fun EventEntity.asDomain(): Event {
+internal fun EventEntity.asDomain(castJsonNumbers: Boolean = false): Event {
-    return EventMapper.map(this)
+    return EventMapper.map(this, castJsonNumbers)
 }
 
 internal fun Event.toEntity(roomId: String, sendState: SendState, ageLocalTs: Long?): EventEntity {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/mapper/HomeServerCapabilitiesMapper.kt

@@ -30,9 +30,7 @@ internal object HomeServerCapabilitiesMapper {
                 canChangePassword = entity.canChangePassword,
                 maxUploadFileSize = entity.maxUploadFileSize,
                 lastVersionIdentityServerSupported = entity.lastVersionIdentityServerSupported,
-                defaultIdentityServerUrl = entity.defaultIdentityServerUrl,
+                defaultIdentityServerUrl = entity.defaultIdentityServerUrl
-                adminE2EByDefault = entity.adminE2EByDefault,
-                preferredJitsiDomain = entity.preferredJitsiDomain
         )
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/mapper/ReadReceiptsSummaryMapper.kt

@@ -18,26 +18,24 @@
 package org.matrix.android.sdk.internal.database.mapper
 
 import org.matrix.android.sdk.api.session.room.model.ReadReceipt
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.model.ReadReceiptsSummaryEntity
 import org.matrix.android.sdk.internal.database.model.UserEntity
 import org.matrix.android.sdk.internal.database.query.where
-import org.matrix.android.sdk.internal.di.SessionDatabase
-import io.realm.Realm
-import io.realm.RealmConfiguration
 import javax.inject.Inject
 
-internal class ReadReceiptsSummaryMapper @Inject constructor(@SessionDatabase private val realmConfiguration: RealmConfiguration) {
+internal class ReadReceiptsSummaryMapper @Inject constructor(private val realmSessionProvider: RealmSessionProvider) {
 
     fun map(readReceiptsSummaryEntity: ReadReceiptsSummaryEntity?): List<ReadReceipt> {
         if (readReceiptsSummaryEntity == null) {
             return emptyList()
         }
-        return Realm.getInstance(realmConfiguration).use { realm ->
+        return realmSessionProvider.withRealm { realm ->
             val readReceipts = readReceiptsSummaryEntity.readReceipts
             readReceipts
                     .mapNotNull {
                         val user = UserEntity.where(realm, it.userId).findFirst()
-                                   ?: return@mapNotNull null
+                                ?: return@mapNotNull null
                         ReadReceipt(user.asDomain(), it.originServerTs.toLong())
                     }
         }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/HomeServerCapabilitiesEntity.kt

@@ -17,17 +17,15 @@
 
 package org.matrix.android.sdk.internal.database.model
 
-import org.matrix.android.sdk.api.session.homeserver.HomeServerCapabilities
 import io.realm.RealmObject
+import org.matrix.android.sdk.api.session.homeserver.HomeServerCapabilities
 
 internal open class HomeServerCapabilitiesEntity(
         var canChangePassword: Boolean = true,
         var maxUploadFileSize: Long = HomeServerCapabilities.MAX_UPLOAD_FILE_SIZE_UNKNOWN,
         var lastVersionIdentityServerSupported: Boolean = false,
         var defaultIdentityServerUrl: String? = null,
-        var adminE2EByDefault: Boolean = true,
+        var lastUpdatedTimestamp: Long = 0L
-        var lastUpdatedTimestamp: Long = 0L,
-        var preferredJitsiDomain: String? = null
 ) : RealmObject() {
 
     companion object

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/RawCacheEntity.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database.model
+
+import io.realm.RealmObject
+import io.realm.annotations.PrimaryKey
+
+internal open class RawCacheEntity(
+        @PrimaryKey
+        var url: String = "",
+        var data: String = "",
+        var lastUpdatedTimestamp: Long = 0L
+) : RealmObject() {
+
+    companion object
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/query/RawCacheQueries.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database.query
+
+import io.realm.Realm
+import io.realm.kotlin.createObject
+import io.realm.kotlin.where
+import org.matrix.android.sdk.internal.database.model.RawCacheEntity
+import org.matrix.android.sdk.internal.database.model.RawCacheEntityFields
+
+/**
+ * Get the current RawCacheEntity, return null if it does not exist
+ */
+internal fun RawCacheEntity.Companion.get(realm: Realm, url: String): RawCacheEntity? {
+    return realm.where<RawCacheEntity>()
+            .equalTo(RawCacheEntityFields.URL, url)
+            .findFirst()
+}
+
+/**
+ * Get the current RawCacheEntity, create one if it does not exist
+ */
+internal fun RawCacheEntity.Companion.getOrCreate(realm: Realm, url: String): RawCacheEntity {
+    return get(realm, url) ?: realm.createObject(url)
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/query/TimelineEventEntityQueries.kt

@@ -17,17 +17,18 @@
 
 package org.matrix.android.sdk.internal.database.query
 
-import org.matrix.android.sdk.api.session.room.send.SendState
-import org.matrix.android.sdk.internal.database.model.ChunkEntity
-import org.matrix.android.sdk.internal.database.model.RoomEntity
-import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
-import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
 import io.realm.Realm
 import io.realm.RealmList
 import io.realm.RealmQuery
 import io.realm.RealmResults
 import io.realm.Sort
 import io.realm.kotlin.where
+import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.api.session.room.timeline.TimelineEventFilters
+import org.matrix.android.sdk.internal.database.model.ChunkEntity
+import org.matrix.android.sdk.internal.database.model.RoomEntity
+import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
+import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
 
 internal fun TimelineEventEntity.Companion.where(realm: Realm, roomId: String, eventId: String): RealmQuery<TimelineEventEntity> {
     return realm.where<TimelineEventEntity>()
@@ -56,16 +57,10 @@ internal fun TimelineEventEntity.Companion.findWithSenderMembershipEvent(realm:
 internal fun TimelineEventEntity.Companion.latestEvent(realm: Realm,
                                                        roomId: String,
                                                        includesSending: Boolean,
-                                                       filterContentRelation: Boolean = false,
+                                                       filters: TimelineEventFilters = TimelineEventFilters()): TimelineEventEntity? {
-                                                       filterTypes: List<String> = emptyList()): TimelineEventEntity? {
     val roomEntity = RoomEntity.where(realm, roomId).findFirst() ?: return null
-    val sendingTimelineEvents = roomEntity.sendingTimelineEvents.where().filterTypes(filterTypes)
+    val sendingTimelineEvents = roomEntity.sendingTimelineEvents.where().filterEvents(filters)
-    val liveEvents = ChunkEntity.findLastForwardChunkOfRoom(realm, roomId)?.timelineEvents?.where()?.filterTypes(filterTypes)
+    val liveEvents = ChunkEntity.findLastForwardChunkOfRoom(realm, roomId)?.timelineEvents?.where()?.filterEvents(filters)
-    if (filterContentRelation) {
-        liveEvents
-                ?.not()?.like(TimelineEventEntityFields.ROOT.CONTENT, TimelineEventFilter.Content.EDIT)
-                ?.not()?.like(TimelineEventEntityFields.ROOT.CONTENT, TimelineEventFilter.Content.RESPONSE)
-    }
     val query = if (includesSending && sendingTimelineEvents.findAll().isNotEmpty()) {
         sendingTimelineEvents
     } else {
@@ -76,6 +71,24 @@ internal fun TimelineEventEntity.Companion.latestEvent(realm: Realm,
             ?.findFirst()
 }
 
+internal fun RealmQuery<TimelineEventEntity>.filterEvents(filters: TimelineEventFilters): RealmQuery<TimelineEventEntity> {
+    if (filters.filterTypes) {
+        `in`(TimelineEventEntityFields.ROOT.TYPE, filters.allowedTypes.toTypedArray())
+    }
+    if (filters.filterUseless) {
+        not()
+                .equalTo(TimelineEventEntityFields.ROOT.IS_USELESS, true)
+    }
+    if (filters.filterEdits) {
+        not().like(TimelineEventEntityFields.ROOT.CONTENT, TimelineEventFilter.Content.EDIT)
+        not().like(TimelineEventEntityFields.ROOT.CONTENT, TimelineEventFilter.Content.RESPONSE)
+    }
+    if (filters.filterRedacted) {
+        not().like(TimelineEventEntityFields.ROOT.UNSIGNED_DATA, TimelineEventFilter.Unsigned.REDACTED)
+    }
+    return this
+}
+
 internal fun RealmQuery<TimelineEventEntity>.filterTypes(filterTypes: List<String>): RealmQuery<TimelineEventEntity> {
     return if (filterTypes.isEmpty()) {
         this

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/DbQualifiers.kt

@@ -23,6 +23,10 @@ import javax.inject.Qualifier
 @Retention(AnnotationRetention.RUNTIME)
 internal annotation class AuthDatabase
 
+@Qualifier
+@Retention(AnnotationRetention.RUNTIME)
+internal annotation class GlobalDatabase
+
 @Qualifier
 @Retention(AnnotationRetention.RUNTIME)
 internal annotation class SessionDatabase

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MatrixComponent.kt

@@ -22,22 +22,30 @@ import android.content.res.Resources
 import com.squareup.moshi.Moshi
 import dagger.BindsInstance
 import dagger.Component
+import okhttp3.OkHttpClient
 import org.matrix.android.sdk.api.Matrix
 import org.matrix.android.sdk.api.MatrixConfiguration
 import org.matrix.android.sdk.api.auth.AuthenticationService
+import org.matrix.android.sdk.api.raw.RawService
 import org.matrix.android.sdk.internal.SessionManager
 import org.matrix.android.sdk.internal.auth.AuthModule
 import org.matrix.android.sdk.internal.auth.SessionParamsStore
+import org.matrix.android.sdk.internal.raw.RawModule
 import org.matrix.android.sdk.internal.session.MockHttpInterceptor
 import org.matrix.android.sdk.internal.session.TestInterceptor
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.util.BackgroundDetectionObserver
 import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
-import okhttp3.OkHttpClient
 import org.matrix.olm.OlmManager
 import java.io.File
 
-@Component(modules = [MatrixModule::class, NetworkModule::class, AuthModule::class, NoOpTestModule::class])
+@Component(modules = [
+    MatrixModule::class,
+    NetworkModule::class,
+    AuthModule::class,
+    RawModule::class,
+    NoOpTestModule::class
+])
 @MatrixScope
 internal interface MatrixComponent {
 
@@ -53,6 +61,8 @@ internal interface MatrixComponent {
 
     fun authenticationService(): AuthenticationService
 
+    fun rawService(): RawService
+
     fun context(): Context
 
     fun matrixConfiguration(): MatrixConfiguration

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/network/ProgressRequestBody.kt

@@ -24,7 +24,7 @@ import okio.BufferedSink
 import okio.ForwardingSink
 import okio.Sink
 import okio.buffer
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import java.io.IOException
 
 internal class ProgressRequestBody(private val delegate: RequestBody,
@@ -40,7 +40,7 @@ internal class ProgressRequestBody(private val delegate: RequestBody,
 
     override fun isDuplex() = delegate.isDuplex()
 
-    val length = tryThis { delegate.contentLength() } ?: -1
+    val length = tryOrNull { delegate.contentLength() } ?: -1
 
     override fun contentLength() = length
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/network/parsing/CheckNumberType.kt

@@ -49,7 +49,7 @@ interface CheckNumberType {
                             val numberAsString = reader.nextString()
                             val decimal = BigDecimal(numberAsString)
                             if (decimal.scale() <= 0) {
-                                decimal.intValueExact()
+                                decimal.longValueExact()
                             } else {
                                 decimal.toDouble()
                             }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/raw/DefaultCleanRawCacheTask.kt

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.raw
+
+import com.zhuinden.monarchy.Monarchy
+import io.realm.kotlin.where
+import org.matrix.android.sdk.internal.database.model.RawCacheEntity
+import org.matrix.android.sdk.internal.di.GlobalDatabase
+import org.matrix.android.sdk.internal.task.Task
+import org.matrix.android.sdk.internal.util.awaitTransaction
+import javax.inject.Inject
+
+internal interface CleanRawCacheTask : Task<Unit, Unit>
+
+internal class DefaultCleanRawCacheTask @Inject constructor(
+        @GlobalDatabase private val monarchy: Monarchy
+) : CleanRawCacheTask {
+
+    override suspend fun execute(params: Unit) {
+        monarchy.awaitTransaction { realm ->
+            realm.where<RawCacheEntity>()
+                    .findAll()
+                    .deleteAllFromRealm()
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/raw/DefaultGetUrlTask.kt

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.raw
+
+import com.zhuinden.monarchy.Monarchy
+import okhttp3.ResponseBody
+import org.matrix.android.sdk.api.raw.RawCacheStrategy
+import org.matrix.android.sdk.internal.database.model.RawCacheEntity
+import org.matrix.android.sdk.internal.database.query.get
+import org.matrix.android.sdk.internal.database.query.getOrCreate
+import org.matrix.android.sdk.internal.di.GlobalDatabase
+import org.matrix.android.sdk.internal.network.executeRequest
+import org.matrix.android.sdk.internal.task.Task
+import org.matrix.android.sdk.internal.util.awaitTransaction
+import java.util.Date
+import javax.inject.Inject
+
+internal interface GetUrlTask : Task<GetUrlTask.Params, String> {
+    data class Params(
+            val url: String,
+            val rawCacheStrategy: RawCacheStrategy
+    )
+}
+
+internal class DefaultGetUrlTask @Inject constructor(
+        private val rawAPI: RawAPI,
+        @GlobalDatabase private val monarchy: Monarchy
+) : GetUrlTask {
+
+    override suspend fun execute(params: GetUrlTask.Params): String {
+        return when (params.rawCacheStrategy) {
+            RawCacheStrategy.NoCache -> doRequest(params.url)
+            is RawCacheStrategy.TtlCache -> doRequestWithCache(
+                    params.url,
+                    params.rawCacheStrategy.validityDurationInMillis,
+                    params.rawCacheStrategy.strict
+            )
+            RawCacheStrategy.InfiniteCache -> doRequestWithCache(
+                    params.url,
+                    Long.MAX_VALUE,
+                    true
+            )
+        }
+    }
+
+    private suspend fun doRequest(url: String): String {
+        return executeRequest<ResponseBody>(null) {
+            apiCall = rawAPI.getUrl(url)
+        }
+                .string()
+    }
+
+    private suspend fun doRequestWithCache(url: String, validityDurationInMillis: Long, strict: Boolean): String {
+        // Get data from cache
+        var dataFromCache: String? = null
+        var isCacheValid = false
+        monarchy.doWithRealm { realm ->
+            val entity = RawCacheEntity.get(realm, url)
+            dataFromCache = entity?.data
+            isCacheValid = entity != null && Date().time < entity.lastUpdatedTimestamp + validityDurationInMillis
+        }
+
+        if (dataFromCache != null && isCacheValid) {
+            return dataFromCache as String
+        }
+
+        // No cache or outdated cache
+        val data = try {
+            doRequest(url)
+        } catch (throwable: Throwable) {
+            // In case of error, we can return value from cache even if outdated
+            return dataFromCache
+                    ?.takeIf { !strict }
+                    ?: throw throwable
+        }
+
+        // Store cache
+        monarchy.awaitTransaction { realm ->
+            val rawCacheEntity = RawCacheEntity.getOrCreate(realm, url)
+            rawCacheEntity.data = data
+            rawCacheEntity.lastUpdatedTimestamp = Date().time
+        }
+
+        return data
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/raw/DefaultRawService.kt

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.raw
+
+import org.matrix.android.sdk.api.MatrixCallback
+import org.matrix.android.sdk.api.raw.RawCacheStrategy
+import org.matrix.android.sdk.api.raw.RawService
+import org.matrix.android.sdk.api.util.Cancelable
+import org.matrix.android.sdk.internal.task.TaskExecutor
+import org.matrix.android.sdk.internal.task.configureWith
+import java.util.concurrent.TimeUnit
+import javax.inject.Inject
+
+internal class DefaultRawService @Inject constructor(
+        private val taskExecutor: TaskExecutor,
+        private val getUrlTask: GetUrlTask,
+        private val cleanRawCacheTask: CleanRawCacheTask
+) : RawService {
+    override fun getUrl(url: String,
+                        rawCacheStrategy: RawCacheStrategy,
+                        matrixCallback: MatrixCallback<String>): Cancelable {
+        return getUrlTask
+                .configureWith(GetUrlTask.Params(url, rawCacheStrategy)) {
+                    callback = matrixCallback
+                }
+                .executeBy(taskExecutor)
+    }
+
+    override fun getWellknown(userId: String,
+                              matrixCallback: MatrixCallback<String>): Cancelable {
+        val homeServerDomain = userId.substringAfter(":")
+        return getUrl(
+                "https://$homeServerDomain/.well-known/matrix/client",
+                RawCacheStrategy.TtlCache(TimeUnit.HOURS.toMillis(8), false),
+                matrixCallback
+        )
+    }
+
+    override fun clearCache(matrixCallback: MatrixCallback<Unit>): Cancelable {
+        return cleanRawCacheTask
+                .configureWith(Unit) {
+                    callback = matrixCallback
+                }
+                .executeBy(taskExecutor)
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/raw/GlobalRealmModule.kt

@@ -1,5 +1,6 @@
 /*
  * Copyright 2019 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,16 +15,16 @@
  * limitations under the License.
  */
 
-package im.vector.app.features.home
+package org.matrix.android.sdk.internal.raw
 
-import androidx.annotation.ColorRes
+import io.realm.annotations.RealmModule
-import im.vector.app.R
+import org.matrix.android.sdk.internal.database.model.RawCacheEntity
 
-@ColorRes
+/**
-fun getColorFromRoomId(roomId: String?): Int {
+ * Realm module for global classes
-    return when ((roomId?.toList()?.sumBy { it.toInt() } ?: 0) % 3) {
+ */
-        1    -> R.color.riotx_avatar_fill_2
+@RealmModule(library = true,
-        2    -> R.color.riotx_avatar_fill_3
+        classes = [
-        else -> R.color.riotx_avatar_fill_1
+            RawCacheEntity::class
-    }
+        ])
-}
+internal class GlobalRealmModule

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/raw/RawAPI.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.matrix.android.sdk.internal.raw
+
+import okhttp3.ResponseBody
+import retrofit2.Call
+import retrofit2.http.GET
+import retrofit2.http.Url
+
+internal interface RawAPI {
+    @GET
+    fun getUrl(@Url url: String): Call<ResponseBody>
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/raw/RawModule.kt

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2020 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.matrix.android.sdk.internal.raw
+
+import com.zhuinden.monarchy.Monarchy
+import dagger.Binds
+import dagger.Lazy
+import dagger.Module
+import dagger.Provides
+import io.realm.RealmConfiguration
+import okhttp3.OkHttpClient
+import org.matrix.android.sdk.api.raw.RawService
+import org.matrix.android.sdk.internal.database.RealmKeysUtils
+import org.matrix.android.sdk.internal.di.GlobalDatabase
+import org.matrix.android.sdk.internal.di.MatrixScope
+import org.matrix.android.sdk.internal.di.Unauthenticated
+import org.matrix.android.sdk.internal.network.RetrofitFactory
+
+@Module
+internal abstract class RawModule {
+
+    @Module
+    companion object {
+        private const val DB_ALIAS = "matrix-sdk-global"
+
+        @JvmStatic
+        @Provides
+        @GlobalDatabase
+        fun providesMonarchy(@GlobalDatabase realmConfiguration: RealmConfiguration): Monarchy {
+            return Monarchy.Builder()
+                    .setRealmConfiguration(realmConfiguration)
+                    .build()
+        }
+
+        @JvmStatic
+        @Provides
+        @GlobalDatabase
+        @MatrixScope
+        fun providesRealmConfiguration(realmKeysUtils: RealmKeysUtils): RealmConfiguration {
+            return RealmConfiguration.Builder()
+                    .apply {
+                        realmKeysUtils.configureEncryption(this, DB_ALIAS)
+                    }
+                    .name("matrix-sdk-global.realm")
+                    .modules(GlobalRealmModule())
+                    .build()
+        }
+
+        @Provides
+        @JvmStatic
+        fun providesRawAPI(@Unauthenticated okHttpClient: Lazy<OkHttpClient>,
+                           retrofitFactory: RetrofitFactory): RawAPI {
+            return retrofitFactory.create(okHttpClient, "https://example.org").create(RawAPI::class.java)
+        }
+    }
+
+    @Binds
+    abstract fun bindRawService(service: DefaultRawService): RawService
+
+    @Binds
+    abstract fun bindGetUrlTask(task: DefaultGetUrlTask): GetUrlTask
+
+    @Binds
+    abstract fun bindCleanRawCacheTask(task: DefaultCleanRawCacheTask): CleanRawCacheTask
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultFileService.kt

@@ -23,7 +23,7 @@ import android.webkit.MimeTypeMap
 import androidx.core.content.FileProvider
 import arrow.core.Try
 import org.matrix.android.sdk.api.MatrixCallback
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.content.ContentUrlResolver
 import org.matrix.android.sdk.api.session.file.FileService
 import org.matrix.android.sdk.api.util.Cancelable
@@ -144,11 +144,13 @@ internal class DefaultFileService @Inject constructor(
 
                         if (elementToDecrypt != null) {
                             Timber.v("## FileService: decrypt file")
-                            val decryptSuccess = MXEncryptedAttachments.decryptAttachment(
+                            val decryptSuccess = destFile.outputStream().buffered().use {
-                                    source.inputStream(),
+                                MXEncryptedAttachments.decryptAttachment(
-                                    elementToDecrypt,
+                                        source.inputStream(),
-                                    destFile.outputStream().buffered()
+                                        elementToDecrypt,
-                            )
+                                        it
+                                )
+                            }
                             response.close()
                             if (!decryptSuccess) {
                                 return@flatMap Try.Failure(IllegalStateException("Decryption error"))
@@ -172,7 +174,7 @@ internal class DefaultFileService @Inject constructor(
                     }
                 }
                 toNotify?.forEach { otherCallbacks ->
-                    tryThis { otherCallbacks.onFailure(it) }
+                    tryOrNull { otherCallbacks.onFailure(it) }
                 }
             }, { file ->
                 callback.onSuccess(file)
@@ -184,7 +186,7 @@ internal class DefaultFileService @Inject constructor(
                 }
                 Timber.v("## FileService additional to notify ${toNotify?.size ?: 0} ")
                 toNotify?.forEach { otherCallbacks ->
-                    tryThis { otherCallbacks.onSuccess(file) }
+                    tryOrNull { otherCallbacks.onSuccess(file) }
                 }
             })
         }.toCancelable()

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt

@@ -166,8 +166,8 @@ internal class DefaultSession @Inject constructor(
         SyncWorker.requireBackgroundSync(workManagerProvider, sessionId)
     }
 
-    override fun startAutomaticBackgroundSync(repeatDelay: Long) {
+    override fun startAutomaticBackgroundSync(timeOutInSeconds: Long, repeatDelayInSeconds: Long) {
-        SyncWorker.automaticallyBackgroundSync(workManagerProvider, sessionId, 0, repeatDelay)
+        SyncWorker.automaticallyBackgroundSync(workManagerProvider, sessionId, timeOutInSeconds, repeatDelayInSeconds)
     }
 
     override fun stopAnyBackgroundSync() {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt

@@ -47,6 +47,7 @@ import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorage
 import org.matrix.android.sdk.internal.crypto.verification.VerificationMessageProcessor
 import org.matrix.android.sdk.internal.database.DatabaseCleaner
 import org.matrix.android.sdk.internal.database.EventInsertLiveObserver
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory
 import org.matrix.android.sdk.internal.di.Authenticated
 import org.matrix.android.sdk.internal.di.DeviceId
@@ -325,23 +326,27 @@ internal abstract class SessionModule {
 
     @Binds
     @IntoSet
-    abstract fun bindIntegrationManager(observer: IntegrationManager): SessionLifecycleObserver
+    abstract fun bindIntegrationManager(manager: IntegrationManager): SessionLifecycleObserver
 
     @Binds
     @IntoSet
-    abstract fun bindWidgetUrlFormatter(observer: DefaultWidgetURLFormatter): SessionLifecycleObserver
+    abstract fun bindWidgetUrlFormatter(formatter: DefaultWidgetURLFormatter): SessionLifecycleObserver
 
     @Binds
     @IntoSet
-    abstract fun bindShieldTrustUpdated(observer: ShieldTrustUpdater): SessionLifecycleObserver
+    abstract fun bindShieldTrustUpdated(updater: ShieldTrustUpdater): SessionLifecycleObserver
 
     @Binds
     @IntoSet
-    abstract fun bindIdentityService(observer: DefaultIdentityService): SessionLifecycleObserver
+    abstract fun bindIdentityService(service: DefaultIdentityService): SessionLifecycleObserver
 
     @Binds
     @IntoSet
-    abstract fun bindDatabaseCleaner(observer: DatabaseCleaner): SessionLifecycleObserver
+    abstract fun bindDatabaseCleaner(cleaner: DatabaseCleaner): SessionLifecycleObserver
+
+    @Binds
+    @IntoSet
+    abstract fun bindRealmSessionProvider(provider: RealmSessionProvider): SessionLifecycleObserver
 
     @Binds
     abstract fun bindInitialSyncProgressService(service: DefaultInitialSyncProgressService): InitialSyncProgressService

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt

@@ -19,7 +19,7 @@ package org.matrix.android.sdk.internal.session.call
 
 import android.os.SystemClock
 import org.matrix.android.sdk.api.MatrixCallback
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.call.CallSignalingService
 import org.matrix.android.sdk.api.session.call.CallState
 import org.matrix.android.sdk.api.session.call.CallsListener
@@ -210,7 +210,7 @@ internal class DefaultCallSignalingService @Inject constructor(
 
     private fun onCallHangup(hangup: CallHangupContent) {
         callListeners.toList().forEach {
-            tryThis {
+            tryOrNull {
                 it.onCallHangupReceived(hangup)
             }
         }
@@ -218,7 +218,7 @@ internal class DefaultCallSignalingService @Inject constructor(
 
     private fun onCallAnswer(answer: CallAnswerContent) {
         callListeners.toList().forEach {
-            tryThis {
+            tryOrNull {
                 it.onCallAnswerReceived(answer)
             }
         }
@@ -226,7 +226,7 @@ internal class DefaultCallSignalingService @Inject constructor(
 
     private fun onCallManageByOtherSession(callId: String) {
         callListeners.toList().forEach {
-            tryThis {
+            tryOrNull {
                 it.onCallManagedByOtherSession(callId)
             }
         }
@@ -237,7 +237,7 @@ internal class DefaultCallSignalingService @Inject constructor(
         if (incomingCall.otherUserId == userId) return
 
         callListeners.toList().forEach {
-            tryThis {
+            tryOrNull {
                 it.onCallInviteReceived(incomingCall, invite)
             }
         }
@@ -245,7 +245,7 @@ internal class DefaultCallSignalingService @Inject constructor(
 
     private fun onCallIceCandidate(incomingCall: MxCall, candidates: CallCandidatesContent) {
         callListeners.toList().forEach {
-            tryThis {
+            tryOrNull {
                 it.onCallIceCandidateReceived(incomingCall, candidates)
             }
         }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/FileUploader.kt

@@ -32,7 +32,7 @@ import okhttp3.RequestBody.Companion.toRequestBody
 import okio.BufferedSink
 import okio.source
 import org.greenrobot.eventbus.EventBus
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.content.ContentUrlResolver
 import org.matrix.android.sdk.internal.di.Authenticated
 import org.matrix.android.sdk.internal.network.ProgressRequestBody
@@ -96,7 +96,7 @@ internal class FileUploader @Inject constructor(@Authenticated
             inputStream.copyTo(it)
         }
         return uploadFile(workingFile, filename, mimeType, progressListener).also {
-            tryThis { workingFile.delete() }
+            tryOrNull { workingFile.delete() }
         }
     }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/ImageCompressor.kt

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.content
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.graphics.Matrix
+import androidx.exifinterface.media.ExifInterface
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import timber.log.Timber
+import java.io.File
+import java.util.UUID
+import javax.inject.Inject
+
+internal class ImageCompressor @Inject constructor() {
+    suspend fun compress(
+            context: Context,
+            imageFile: File,
+            desiredWidth: Int,
+            desiredHeight: Int,
+            desiredQuality: Int = 80): File {
+        return withContext(Dispatchers.IO) {
+            val compressedBitmap = BitmapFactory.Options().run {
+                inJustDecodeBounds = true
+                decodeBitmap(imageFile, this)
+                inSampleSize = calculateInSampleSize(outWidth, outHeight, desiredWidth, desiredHeight)
+                inJustDecodeBounds = false
+                decodeBitmap(imageFile, this)?.let {
+                    rotateBitmap(imageFile, it)
+                }
+            } ?: return@withContext imageFile
+
+            val destinationFile = createDestinationFile(context)
+
+            runCatching {
+                destinationFile.outputStream().use {
+                    compressedBitmap.compress(Bitmap.CompressFormat.JPEG, desiredQuality, it)
+                }
+            }
+
+            return@withContext destinationFile
+        }
+    }
+
+    private fun rotateBitmap(file: File, bitmap: Bitmap): Bitmap {
+        file.inputStream().use { inputStream ->
+            try {
+                ExifInterface(inputStream).let { exifInfo ->
+                    val orientation = exifInfo.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
+                    val matrix = Matrix()
+                    when (orientation) {
+                        ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f)
+                        ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f)
+                        ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f)
+                        ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.preScale(-1f, 1f)
+                        ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.preScale(1f, -1f)
+                        ExifInterface.ORIENTATION_TRANSPOSE -> {
+                            matrix.preRotate(-90f)
+                            matrix.preScale(-1f, 1f)
+                        }
+                        ExifInterface.ORIENTATION_TRANSVERSE -> {
+                            matrix.preRotate(90f)
+                            matrix.preScale(-1f, 1f)
+                        }
+                        else                                      -> return bitmap
+                    }
+                    return Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
+                }
+            } catch (e: Exception) {
+                Timber.e(e, "Cannot read orientation")
+            }
+        }
+        return bitmap
+    }
+
+    // https://developer.android.com/topic/performance/graphics/load-bitmap
+    private fun calculateInSampleSize(width: Int, height: Int, desiredWidth: Int, desiredHeight: Int): Int {
+        var inSampleSize = 1
+
+        if (width > desiredWidth || height > desiredHeight) {
+            val halfHeight: Int = height / 2
+            val halfWidth: Int = width / 2
+
+            // Calculate the largest inSampleSize value that is a power of 2 and keeps both
+            // height and width larger than the requested height and width.
+            while (halfHeight / inSampleSize >= desiredHeight && halfWidth / inSampleSize >= desiredWidth) {
+                inSampleSize *= 2
+            }
+        }
+
+        return inSampleSize
+    }
+
+    private fun decodeBitmap(file: File, options: BitmapFactory.Options = BitmapFactory.Options()): Bitmap? {
+        return try {
+            file.inputStream().use { inputStream ->
+                BitmapFactory.decodeStream(inputStream, null, options)
+            }
+        } catch (e: Exception) {
+            Timber.e(e, "Cannot decode Bitmap")
+            null
+        }
+    }
+
+    private fun createDestinationFile(context: Context): File {
+        return File.createTempFile(UUID.randomUUID().toString(), null, context.cacheDir)
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/content/UploadContentWorker.kt

@@ -19,14 +19,10 @@ package org.matrix.android.sdk.internal.session.content
 
 import android.content.Context
 import android.graphics.BitmapFactory
-import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
-import id.zelory.compressor.Compressor
+import org.matrix.android.sdk.api.extensions.tryOrNull
-import id.zelory.compressor.constraint.default
-import org.matrix.android.sdk.api.extensions.tryThis
 import org.matrix.android.sdk.api.session.content.ContentAttachmentData
-import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.toContent
 import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.room.model.message.MessageAudioContent
@@ -36,13 +32,18 @@ import org.matrix.android.sdk.api.session.room.model.message.MessageImageContent
 import org.matrix.android.sdk.api.session.room.model.message.MessageVideoContent
 import org.matrix.android.sdk.internal.crypto.attachments.MXEncryptedAttachments
 import org.matrix.android.sdk.internal.crypto.model.rest.EncryptedFileInfo
+import org.matrix.android.sdk.internal.database.mapper.ContentMapper
+import org.matrix.android.sdk.internal.database.mapper.asDomain
 import org.matrix.android.sdk.internal.network.ProgressRequestBody
 import org.matrix.android.sdk.internal.session.DefaultFileService
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.session.room.send.CancelSendTracker
+import org.matrix.android.sdk.internal.session.room.send.LocalEchoIdentifiers
+import org.matrix.android.sdk.internal.session.room.send.LocalEchoRepository
 import org.matrix.android.sdk.internal.session.room.send.MultipleEventSendingDispatcherWorker
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
 import timber.log.Timber
 import java.io.File
 import java.util.UUID
@@ -58,12 +59,13 @@ private data class NewImageAttributes(
  * Possible previous worker: None
  * Possible next worker    : Always [MultipleEventSendingDispatcherWorker]
  */
-internal class UploadContentWorker(val context: Context, params: WorkerParameters) : CoroutineWorker(context, params) {
+internal class UploadContentWorker(val context: Context, params: WorkerParameters)
+    : SessionSafeCoroutineWorker<UploadContentWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
             override val sessionId: String,
-            val events: List<Event>,
+            val localEchoIds: List<LocalEchoIdentifiers>,
             val attachment: ContentAttachmentData,
             val isEncrypted: Boolean,
             val compressBeforeSending: Boolean,
@@ -74,20 +76,15 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
     @Inject lateinit var contentUploadStateTracker: DefaultContentUploadStateTracker
     @Inject lateinit var fileService: DefaultFileService
     @Inject lateinit var cancelSendTracker: CancelSendTracker
+    @Inject lateinit var imageCompressor: ImageCompressor
+    @Inject lateinit var localEchoRepository: LocalEchoRepository
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+        injector.inject(this)
-                ?: return Result.success()
+    }
-                        .also { Timber.e("Unable to parse work parameters") }
 
+    override suspend fun doSafeWork(params: Params): Result {
         Timber.v("Starting upload media work with params $params")
-
-        if (params.lastFailureMessage != null) {
-            // Transmit the error
-            return Result.success(inputData)
-                    .also { Timber.e("Work cancelled due to input error from parent") }
-        }
-
         // Just defensive code to ensure that we never have an uncaught exception that could break the queue
         return try {
             internalDoWork(params)
@@ -97,21 +94,21 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
         }
     }
 
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
+
     private suspend fun internalDoWork(params: Params): Result {
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
+        val allCancelled = params.localEchoIds.all { cancelSendTracker.isCancelRequestedFor(it.eventId, it.roomId) }
-        sessionComponent.inject(this)
-
-        val attachment = params.attachment
-
-        var newImageAttributes: NewImageAttributes? = null
-
-        val allCancelled = params.events.all { cancelSendTracker.isCancelRequestedFor(it.eventId, it.roomId) }
         if (allCancelled) {
             // there is no point in uploading the image!
             return Result.success(inputData)
                     .also { Timber.e("## Send: Work cancelled by user") }
         }
 
+        val attachment = params.attachment
+        val filesToDelete = mutableListOf<File>()
+
         try {
             val inputStream = context.contentResolver.openInputStream(attachment.queryUri)
                     ?: return Result.success(
@@ -124,44 +121,15 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
 
             // always use a temporary file, it guaranties that we could report progress on upload and simplifies the flows
             val workingFile = File.createTempFile(UUID.randomUUID().toString(), null, context.cacheDir)
-            workingFile.outputStream().use {
+                    .also { filesToDelete.add(it) }
-                inputStream.copyTo(it)
+            workingFile.outputStream().use { outputStream ->
-            }
+                inputStream.use { inputStream ->
-
+                    inputStream.copyTo(outputStream)
-//            inputStream.use {
-            var uploadedThumbnailUrl: String? = null
-            var uploadedThumbnailEncryptedFileInfo: EncryptedFileInfo? = null
-
-            ThumbnailExtractor.extractThumbnail(context, params.attachment)?.let { thumbnailData ->
-                val thumbnailProgressListener = object : ProgressRequestBody.Listener {
-                    override fun onProgress(current: Long, total: Long) {
-                        notifyTracker(params) { contentUploadStateTracker.setProgressThumbnail(it, current, total) }
-                    }
-                }
-
-                    try {
-                        val contentUploadResponse = if (params.isEncrypted) {
-                            Timber.v("Encrypt thumbnail")
-                            notifyTracker(params) { contentUploadStateTracker.setEncryptingThumbnail(it) }
-                            val encryptionResult = MXEncryptedAttachments.encryptAttachment(thumbnailData.bytes.inputStream(), thumbnailData.mimeType)
-                            uploadedThumbnailEncryptedFileInfo = encryptionResult.encryptedFileInfo
-                            fileUploader.uploadByteArray(encryptionResult.encryptedByteArray,
-                                    "thumb_${attachment.name}",
-                                    "application/octet-stream",
-                                    thumbnailProgressListener)
-                        } else {
-                            fileUploader.uploadByteArray(thumbnailData.bytes,
-                                    "thumb_${attachment.name}",
-                                    thumbnailData.mimeType,
-                                    thumbnailProgressListener)
-                        }
-
-                    uploadedThumbnailUrl = contentUploadResponse.contentUri
-                } catch (t: Throwable) {
-                    Timber.e(t, "Thumbnail update failed")
                 }
             }
 
+            val uploadThumbnailResult = dealWithThumbnail(params)
+
             val progressListener = object : ProgressRequestBody.Listener {
                 override fun onProgress(current: Long, total: Long) {
                     notifyTracker(params) {
@@ -177,40 +145,37 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
             var uploadedFileEncryptedFileInfo: EncryptedFileInfo? = null
 
             return try {
-                val fileToUplaod: File
+                val fileToUpload: File
+                var newImageAttributes: NewImageAttributes? = null
 
                 if (attachment.type == ContentAttachmentData.Type.IMAGE && params.compressBeforeSending) {
-                    // Compressor library works with File instead of Uri for now. Since Scoped Storage doesn't allow us to access files directly, we should
+                    fileToUpload = imageCompressor.compress(context, workingFile, MAX_IMAGE_SIZE, MAX_IMAGE_SIZE)
-                    // copy it to a cache folder by using InputStream and OutputStream.
+                            .also { compressedFile ->
-                    // https://github.com/zetbaitsu/Compressor/pull/150
+                                // Get new Bitmap size
-                    // As soon as the above PR is merged, we can use attachment.queryUri instead of creating a cacheFile.
+                                compressedFile.inputStream().use {
-                    val compressedFile = Compressor.compress(context, workingFile) {
+                                    val options = BitmapFactory.Options().apply { inJustDecodeBounds = true }
-                        default(
+                                    val bitmap = BitmapFactory.decodeStream(it, null, options)
-                                width = MAX_IMAGE_SIZE,
+                                    val fileSize = bitmap?.byteCount ?: 0
-                                height = MAX_IMAGE_SIZE
+                                    newImageAttributes = NewImageAttributes(
-                        )
+                                            options.outWidth,
-                    }
+                                            options.outHeight,
-
+                                            fileSize
-                    val options = BitmapFactory.Options().apply { inJustDecodeBounds = true }
+                                    )
-                    BitmapFactory.decodeFile(compressedFile.absolutePath, options)
+                                }
-                    val fileSize = compressedFile.length().toInt()
+                            }
-                    newImageAttributes = NewImageAttributes(
+                            .also { filesToDelete.add(it) }
-                            options.outWidth,
-                            options.outHeight,
-                            fileSize
-                    )
-                    fileToUplaod = compressedFile
                 } else {
-                    fileToUplaod = workingFile
+                    fileToUpload = workingFile
                 }
 
                 val contentUploadResponse = if (params.isEncrypted) {
                     Timber.v("## FileService: Encrypt file")
 
                     val tmpEncrypted = File.createTempFile(UUID.randomUUID().toString(), null, context.cacheDir)
+                            .also { filesToDelete.add(it) }
 
                     uploadedFileEncryptedFileInfo =
-                            MXEncryptedAttachments.encrypt(fileToUplaod.inputStream(), attachment.getSafeMimeType(), tmpEncrypted) { read, total ->
+                            MXEncryptedAttachments.encrypt(fileToUpload.inputStream(), attachment.getSafeMimeType(), tmpEncrypted) { read, total ->
                                 notifyTracker(params) {
                                     contentUploadStateTracker.setEncrypting(it, read.toLong(), total.toLong())
                                 }
@@ -220,14 +185,10 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
 
                     fileUploader
                             .uploadFile(tmpEncrypted, attachment.name, "application/octet-stream", progressListener)
-                            .also {
-                                // we can delete?
-                                tryThis { tmpEncrypted.delete() }
-                            }
                 } else {
                     Timber.v("## FileService: Clear file")
                     fileUploader
-                            .uploadFile(fileToUplaod, attachment.name, attachment.getSafeMimeType(), progressListener)
+                            .uploadFile(fileToUpload, attachment.name, attachment.getSafeMimeType(), progressListener)
                 }
 
                 Timber.v("## FileService: Update cache storage for ${contentUploadResponse.contentUri}")
@@ -237,14 +198,14 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
                     }
                     Timber.v("## FileService: cache storage updated")
                 } catch (failure: Throwable) {
-                    Timber.e(failure, "## FileService: Failed to update fileservice cache")
+                    Timber.e(failure, "## FileService: Failed to update file cache")
                 }
 
                 handleSuccess(params,
                         contentUploadResponse.contentUri,
                         uploadedFileEncryptedFileInfo,
-                        uploadedThumbnailUrl,
+                        uploadThumbnailResult?.uploadedThumbnailUrl,
-                        uploadedThumbnailEncryptedFileInfo,
+                        uploadThumbnailResult?.uploadedThumbnailEncryptedFileInfo,
                         newImageAttributes)
             } catch (t: Throwable) {
                 Timber.e(t, "## FileService: ERROR ${t.localizedMessage}")
@@ -252,17 +213,62 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
             }
         } catch (e: Exception) {
             Timber.e(e, "## FileService: ERROR")
-            notifyTracker(params) { contentUploadStateTracker.setFailure(it, e) }
+            return handleFailure(params, e)
-            return Result.success(
+        } finally {
-                    WorkerParamsFactory.toData(
+            // Delete all temporary files
-                            params.copy(
+            filesToDelete.forEach {
-                                    lastFailureMessage = e.localizedMessage
+                tryOrNull { it.delete() }
-                            )
+            }
-                    )
-            )
         }
     }
 
+    private data class UploadThumbnailResult(
+            val uploadedThumbnailUrl: String,
+            val uploadedThumbnailEncryptedFileInfo: EncryptedFileInfo?
+    )
+
+    /**
+     * If appropriate, it will create and upload a thumbnail
+     */
+    private suspend fun dealWithThumbnail(params: Params): UploadThumbnailResult? {
+        return ThumbnailExtractor.extractThumbnail(context, params.attachment)
+                ?.let { thumbnailData ->
+                    val thumbnailProgressListener = object : ProgressRequestBody.Listener {
+                        override fun onProgress(current: Long, total: Long) {
+                            notifyTracker(params) { contentUploadStateTracker.setProgressThumbnail(it, current, total) }
+                        }
+                    }
+
+                    try {
+                        if (params.isEncrypted) {
+                            Timber.v("Encrypt thumbnail")
+                            notifyTracker(params) { contentUploadStateTracker.setEncryptingThumbnail(it) }
+                            val encryptionResult = MXEncryptedAttachments.encryptAttachment(thumbnailData.bytes.inputStream(), thumbnailData.mimeType)
+                            val contentUploadResponse = fileUploader.uploadByteArray(encryptionResult.encryptedByteArray,
+                                    "thumb_${params.attachment.name}",
+                                    "application/octet-stream",
+                                    thumbnailProgressListener)
+                            UploadThumbnailResult(
+                                    contentUploadResponse.contentUri,
+                                    encryptionResult.encryptedFileInfo
+                            )
+                        } else {
+                            val contentUploadResponse = fileUploader.uploadByteArray(thumbnailData.bytes,
+                                    "thumb_${params.attachment.name}",
+                                    thumbnailData.mimeType,
+                                    thumbnailProgressListener)
+                            UploadThumbnailResult(
+                                    contentUploadResponse.contentUri,
+                                    null
+                            )
+                        }
+                    } catch (t: Throwable) {
+                        Timber.e(t, "Thumbnail upload failed")
+                        null
+                    }
+                }
+    }
+
     private fun handleFailure(params: Params, failure: Throwable): Result {
         notifyTracker(params) { contentUploadStateTracker.setFailure(it, failure) }
 
@@ -275,46 +281,48 @@ internal class UploadContentWorker(val context: Context, params: WorkerParameter
         )
     }
 
-    private fun handleSuccess(params: Params,
+    private suspend fun handleSuccess(params: Params,
-                              attachmentUrl: String,
+                                      attachmentUrl: String,
-                              encryptedFileInfo: EncryptedFileInfo?,
+                                      encryptedFileInfo: EncryptedFileInfo?,
-                              thumbnailUrl: String?,
+                                      thumbnailUrl: String?,
-                              thumbnailEncryptedFileInfo: EncryptedFileInfo?,
+                                      thumbnailEncryptedFileInfo: EncryptedFileInfo?,
-                              newImageAttributes: NewImageAttributes?): Result {
+                                      newImageAttributes: NewImageAttributes?): Result {
         notifyTracker(params) { contentUploadStateTracker.setSuccess(it) }
+        params.localEchoIds.forEach {
+            updateEvent(it.eventId, attachmentUrl, encryptedFileInfo, thumbnailUrl, thumbnailEncryptedFileInfo, newImageAttributes)
+        }
 
-        val updatedEvents = params.events
+        val sendParams = MultipleEventSendingDispatcherWorker.Params(
-                .map {
+                sessionId = params.sessionId,
-                    updateEvent(it, attachmentUrl, encryptedFileInfo, thumbnailUrl, thumbnailEncryptedFileInfo, newImageAttributes)
+                localEchoIds = params.localEchoIds,
-                }
+                isEncrypted = params.isEncrypted
-
+        )
-        val sendParams = MultipleEventSendingDispatcherWorker.Params(params.sessionId, updatedEvents, params.isEncrypted)
         return Result.success(WorkerParamsFactory.toData(sendParams)).also {
             Timber.v("## handleSuccess $attachmentUrl, work is stopped $isStopped")
         }
     }
 
-    private fun updateEvent(event: Event,
+    private suspend fun updateEvent(eventId: String,
-                            url: String,
+                                    url: String,
-                            encryptedFileInfo: EncryptedFileInfo?,
+                                    encryptedFileInfo: EncryptedFileInfo?,
-                            thumbnailUrl: String? = null,
+                                    thumbnailUrl: String? = null,
-                            thumbnailEncryptedFileInfo: EncryptedFileInfo?,
+                                    thumbnailEncryptedFileInfo: EncryptedFileInfo?,
-                            newImageAttributes: NewImageAttributes?): Event {
+                                    newImageAttributes: NewImageAttributes?) {
-        val messageContent: MessageContent = event.content.toModel() ?: return event
+        localEchoRepository.updateEcho(eventId) { _, event ->
-        val updatedContent = when (messageContent) {
+            val messageContent: MessageContent? = event.asDomain().content.toModel()
-            is MessageImageContent -> messageContent.update(url, encryptedFileInfo, newImageAttributes)
+            val updatedContent = when (messageContent) {
-            is MessageVideoContent -> messageContent.update(url, encryptedFileInfo, thumbnailUrl, thumbnailEncryptedFileInfo)
+                is MessageImageContent -> messageContent.update(url, encryptedFileInfo, newImageAttributes)
-            is MessageFileContent  -> messageContent.update(url, encryptedFileInfo)
+                is MessageVideoContent -> messageContent.update(url, encryptedFileInfo, thumbnailUrl, thumbnailEncryptedFileInfo)
-            is MessageAudioContent -> messageContent.update(url, encryptedFileInfo)
+                is MessageFileContent -> messageContent.update(url, encryptedFileInfo)
-            else                   -> messageContent
+                is MessageAudioContent -> messageContent.update(url, encryptedFileInfo)
+                else                   -> messageContent
+            }
+            event.content = ContentMapper.map(updatedContent.toContent())
         }
-        return event.copy(content = updatedContent.toContent())
     }
 
     private fun notifyTracker(params: Params, function: (String) -> Unit) {
-        params.events
+        params.localEchoIds.forEach { function.invoke(it.eventId) }
-                .mapNotNull { it.eventId }
-                .forEach { eventId -> function.invoke(eventId) }
     }
 
     private fun MessageImageContent.update(url: String,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/download/DefaultContentDownloadStateTracker.kt

@@ -19,7 +19,7 @@ package org.matrix.android.sdk.internal.session.download
 
 import android.os.Handler
 import android.os.Looper
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.file.ContentDownloadStateTracker
 import org.matrix.android.sdk.internal.session.SessionScope
 import timber.log.Timber
@@ -76,7 +76,7 @@ internal class DefaultContentDownloadStateTracker @Inject constructor() : Progre
             Timber.v("## DL Progress Error code:$errorCode")
             updateState(url, ContentDownloadStateTracker.State.Failure(errorCode))
             listeners[url]?.forEach {
-                tryThis { it.onDownloadStateUpdate(ContentDownloadStateTracker.State.Failure(errorCode)) }
+                tryOrNull { it.onDownloadStateUpdate(ContentDownloadStateTracker.State.Failure(errorCode)) }
             }
         }
     }
@@ -84,7 +84,7 @@ internal class DefaultContentDownloadStateTracker @Inject constructor() : Progre
     private fun updateState(url: String, state: ContentDownloadStateTracker.State) {
         states[url] = state
         listeners[url]?.forEach {
-            tryThis { it.onDownloadStateUpdate(state) }
+            tryOrNull { it.onDownloadStateUpdate(state) }
         }
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/group/GetGroupDataWorker.kt

@@ -18,20 +18,19 @@
 package org.matrix.android.sdk.internal.session.group
 
 import android.content.Context
-import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
+import org.matrix.android.sdk.internal.session.SessionComponent
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
-import timber.log.Timber
 import javax.inject.Inject
 
 /**
  * Possible previous worker: None
  * Possible next worker    : None
  */
-internal class GetGroupDataWorker(context: Context, params: WorkerParameters) : CoroutineWorker(context, params) {
+internal class GetGroupDataWorker(context: Context, params: WorkerParameters)
+    : SessionSafeCoroutineWorker<GetGroupDataWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
@@ -41,13 +40,11 @@ internal class GetGroupDataWorker(context: Context, params: WorkerParameters) :
 
     @Inject lateinit var getGroupDataTask: GetGroupDataTask
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+        injector.inject(this)
-                ?: return Result.failure()
+    }
-                        .also { Timber.e("Unable to parse work parameters") }
 
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
+    override suspend fun doSafeWork(params: Params): Result {
-        sessionComponent.inject(this)
         return runCatching {
             getGroupDataTask.execute(GetGroupDataTask.Params.FetchAllActive)
         }.fold(
@@ -55,4 +52,8 @@ internal class GetGroupDataWorker(context: Context, params: WorkerParameters) :
                 { Result.retry() }
         )
     }
+
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/homeserver/DefaultGetHomeServerCapabilitiesTask.kt

@@ -18,6 +18,7 @@
 package org.matrix.android.sdk.internal.session.homeserver
 
 import com.zhuinden.monarchy.Monarchy
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.auth.data.HomeServerConnectionConfig
 import org.matrix.android.sdk.api.auth.wellknown.WellknownResult
 import org.matrix.android.sdk.api.session.homeserver.HomeServerCapabilities
@@ -32,7 +33,6 @@ import org.matrix.android.sdk.internal.session.integrationmanager.IntegrationMan
 import org.matrix.android.sdk.internal.task.Task
 import org.matrix.android.sdk.internal.util.awaitTransaction
 import org.matrix.android.sdk.internal.wellknown.GetWellknownTask
-import org.greenrobot.eventbus.EventBus
 import timber.log.Timber
 import java.util.Date
 import javax.inject.Inject
@@ -109,16 +109,12 @@ internal class DefaultGetHomeServerCapabilitiesTask @Inject constructor(
 
             if (getWellknownResult != null && getWellknownResult is WellknownResult.Prompt) {
                 homeServerCapabilitiesEntity.defaultIdentityServerUrl = getWellknownResult.identityServerUrl
-                homeServerCapabilitiesEntity.adminE2EByDefault = getWellknownResult.wellKnown.e2eAdminSetting?.e2eDefault ?: true
-                homeServerCapabilitiesEntity.preferredJitsiDomain = getWellknownResult.wellKnown.jitsiServer?.preferredDomain
                 // We are also checking for integration manager configurations
                 val config = configExtractor.extract(getWellknownResult.wellKnown)
                 if (config != null) {
                     Timber.v("Extracted integration config : $config")
                     realm.insertOrUpdate(config)
                 }
-            } else {
-                homeServerCapabilitiesEntity.adminE2EByDefault = true
             }
             homeServerCapabilitiesEntity.lastUpdatedTimestamp = Date().time
         }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/identity/DefaultIdentityService.kt

@@ -23,7 +23,7 @@ import androidx.lifecycle.LifecycleRegistry
 import dagger.Lazy
 import org.matrix.android.sdk.api.MatrixCallback
 import org.matrix.android.sdk.api.auth.data.SessionParams
-import org.matrix.android.sdk.api.extensions.tryThis
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.failure.Failure
 import org.matrix.android.sdk.api.failure.MatrixError
 import org.matrix.android.sdk.api.session.events.model.toModel
@@ -113,7 +113,7 @@ internal class DefaultIdentityService @Inject constructor(
             // Url has changed, we have to reset our store, update internal configuration and notify listeners
             identityStore.setUrl(baseUrl)
             updateIdentityAPI(baseUrl)
-            listeners.toList().forEach { tryThis { it.onIdentityServerChange() } }
+            listeners.toList().forEach { tryOrNull { it.onIdentityServerChange() } }
         }
     }
 
@@ -236,7 +236,7 @@ internal class DefaultIdentityService @Inject constructor(
     private suspend fun updateAccountData(url: String?) {
         // Also notify the listener
         withContext(coroutineDispatchers.main) {
-            listeners.toList().forEach { tryThis { it.onIdentityServerChange() } }
+            listeners.toList().forEach { tryOrNull { it.onIdentityServerChange() } }
         }
 
         updateUserAccountDataTask.execute(UpdateUserAccountDataTask.IdentityParams(

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/profile/RefreshUserThreePidsTask.kt

@@ -23,6 +23,7 @@ import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.network.executeRequest
 import org.matrix.android.sdk.internal.task.Task
 import org.greenrobot.eventbus.EventBus
+import org.matrix.android.sdk.internal.util.awaitTransaction
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -39,7 +40,7 @@ internal class DefaultRefreshUserThreePidsTask @Inject constructor(private val p
 
         Timber.d("Get ${accountThreePidsResponse.threePids?.size} threePids")
         // Store the list in DB
-        monarchy.writeAsync { realm ->
+        monarchy.awaitTransaction { realm ->
             realm.where(UserThreePidEntity::class.java).findAll().deleteAllFromRealm()
             accountThreePidsResponse.threePids?.forEach {
                 val entity = UserThreePidEntity(

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/pushers/AddHttpPusherWorker.kt

@@ -17,10 +17,10 @@
 package org.matrix.android.sdk.internal.session.pushers
 
 import android.content.Context
-import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
 import com.zhuinden.monarchy.Monarchy
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.failure.Failure
 import org.matrix.android.sdk.api.session.pushers.PusherState
 import org.matrix.android.sdk.internal.database.mapper.toEntity
@@ -28,16 +28,14 @@ import org.matrix.android.sdk.internal.database.model.PusherEntity
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.network.executeRequest
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.util.awaitTransaction
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
-import org.greenrobot.eventbus.EventBus
-import timber.log.Timber
 import javax.inject.Inject
 
 internal class AddHttpPusherWorker(context: Context, params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<AddHttpPusherWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
@@ -50,14 +48,11 @@ internal class AddHttpPusherWorker(context: Context, params: WorkerParameters)
     @Inject @SessionDatabase lateinit var monarchy: Monarchy
     @Inject lateinit var eventBus: EventBus
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+        injector.inject(this)
-                ?: return Result.failure()
+    }
-                        .also { Timber.e("Unable to parse work parameters") }
-
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
-        sessionComponent.inject(this)
 
+    override suspend fun doSafeWork(params: Params): Result {
         val pusher = params.pusher
 
         if (pusher.pushKey.isBlank()) {
@@ -82,6 +77,10 @@ internal class AddHttpPusherWorker(context: Context, params: WorkerParameters)
         }
     }
 
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
+
     private suspend fun setPusher(pusher: JsonPusher) {
         executeRequest<Unit>(eventBus) {
             apiCall = pushersAPI.setPusher(pusher)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/EventRelationsAggregationProcessor.kt

@@ -347,7 +347,7 @@ internal class EventRelationsAggregationProcessor @Inject constructor(@UserId pr
                 if (userId == senderId) {
                     sumModel.myVote = optionIndex
                 }
-                Timber.v("## POLL adding vote $optionIndex for user $senderId in poll :$relatedEventId ")
+                Timber.v("## POLL adding vote $optionIndex for user $senderId in poll :$targetEventId ")
             } else {
                 Timber.v("## POLL Ignoring vote (older than known one)  eventId:$eventId ")
             }
@@ -356,7 +356,7 @@ internal class EventRelationsAggregationProcessor @Inject constructor(@UserId pr
             if (userId == senderId) {
                 sumModel.myVote = optionIndex
             }
-            Timber.v("## POLL adding vote $optionIndex for user $senderId in poll :$relatedEventId ")
+            Timber.v("## POLL adding vote $optionIndex for user $senderId in poll :$targetEventId ")
         }
         sumModel.votes = votes
         if (isLocalEcho) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/RoomGetter.kt

@@ -17,17 +17,16 @@
 
 package org.matrix.android.sdk.internal.session.room
 
-import com.zhuinden.monarchy.Monarchy
+import io.realm.Realm
 import org.matrix.android.sdk.api.session.room.Room
 import org.matrix.android.sdk.api.session.room.model.Membership
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.model.RoomEntity
 import org.matrix.android.sdk.internal.database.model.RoomSummaryEntity
 import org.matrix.android.sdk.internal.database.model.RoomSummaryEntityFields
 import org.matrix.android.sdk.internal.database.query.where
-import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.session.SessionScope
 import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
-import io.realm.Realm
 import javax.inject.Inject
 
 internal interface RoomGetter {
@@ -38,18 +37,18 @@ internal interface RoomGetter {
 
 @SessionScope
 internal class DefaultRoomGetter @Inject constructor(
-        @SessionDatabase private val monarchy: Monarchy,
+        private val realmSessionProvider: RealmSessionProvider,
         private val roomFactory: RoomFactory
 ) : RoomGetter {
 
     override fun getRoom(roomId: String): Room? {
-        return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+        return realmSessionProvider.withRealm { realm ->
             createRoom(realm, roomId)
         }
     }
 
     override fun getDirectRoomWith(otherUserId: String): Room? {
-        return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+        return realmSessionProvider.withRealm { realm ->
             RoomSummaryEntity.where(realm)
                     .equalTo(RoomSummaryEntityFields.IS_DIRECT, true)
                     .equalTo(RoomSummaryEntityFields.MEMBERSHIP_STR, Membership.JOIN.name)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/relation/DefaultRelationService.kt

@@ -202,13 +202,13 @@ internal class DefaultRelationService @AssistedInject constructor(
 
     private fun createEncryptEventWork(event: Event, keepKeys: List<String>?): OneTimeWorkRequest {
         // Same parameter
-        val params = EncryptEventWorker.Params(sessionId, event, keepKeys)
+        val params = EncryptEventWorker.Params(sessionId, event.eventId!!, keepKeys)
         val sendWorkData = WorkerParamsFactory.toData(params)
         return timeLineSendEventWorkCommon.createWork<EncryptEventWorker>(sendWorkData, true)
     }
 
     private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, event = event)
+        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, eventId = event.eventId!!)
         val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
         return timeLineSendEventWorkCommon.createWork<SendEventWorker>(sendWorkData, startChain)
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/relation/SendRelationWorker.kt

@@ -17,7 +17,6 @@
 package org.matrix.android.sdk.internal.session.room.relation
 
 import android.content.Context
-import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
 import org.greenrobot.eventbus.EventBus
@@ -27,45 +26,38 @@ import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.room.model.relation.ReactionContent
 import org.matrix.android.sdk.api.session.room.model.relation.ReactionInfo
 import org.matrix.android.sdk.internal.network.executeRequest
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.session.room.RoomAPI
+import org.matrix.android.sdk.internal.session.room.send.LocalEchoRepository
 import org.matrix.android.sdk.internal.session.room.send.SendResponse
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
-import timber.log.Timber
 import javax.inject.Inject
 
 // TODO This is not used. Delete?
-internal class SendRelationWorker(context: Context, params: WorkerParameters) : CoroutineWorker(context, params) {
+internal class SendRelationWorker(context: Context, params: WorkerParameters)
+    : SessionSafeCoroutineWorker<SendRelationWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
             override val sessionId: String,
             val roomId: String,
-            val event: Event,
+            val eventId: String,
             val relationType: String? = null,
             override val lastFailureMessage: String? = null
     ) : SessionWorkerParams
 
     @Inject lateinit var roomAPI: RoomAPI
     @Inject lateinit var eventBus: EventBus
+    @Inject lateinit var localEchoRepository: LocalEchoRepository
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+        injector.inject(this)
-                ?: return Result.failure()
+    }
-                        .also { Timber.e("Unable to parse work parameters") }
 
-        if (params.lastFailureMessage != null) {
+    override suspend fun doSafeWork(params: Params): Result {
-            // Transmit the error
+        val localEvent = localEchoRepository.getUpToDateEcho(params.eventId)
-            return Result.success(inputData)
+        if (localEvent?.eventId == null) {
-                    .also { Timber.e("Work cancelled due to input error from parent") }
-        }
-
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
-        sessionComponent.inject(this)
-
-        val localEvent = params.event
-        if (localEvent.eventId == null) {
             return Result.failure()
         }
         val relationContent = localEvent.content.toModel<ReactionContent>()
@@ -88,6 +80,10 @@ internal class SendRelationWorker(context: Context, params: WorkerParameters) :
         }
     }
 
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
+
     private suspend fun sendRelation(roomId: String, relationType: String, relatedEventId: String, localEvent: Event) {
         executeRequest<SendResponse>(eventBus) {
             apiCall = roomAPI.sendRelation(

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/DefaultSendService.kt

@@ -336,7 +336,7 @@ internal class DefaultSendService @AssistedInject constructor(
 
     private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
         // Same parameter
-        return EncryptEventWorker.Params(sessionId, event)
+        return EncryptEventWorker.Params(sessionId, event.eventId ?: "")
                 .let { WorkerParamsFactory.toData(it) }
                 .let {
                     workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
@@ -360,7 +360,10 @@ internal class DefaultSendService @AssistedInject constructor(
                                       attachment: ContentAttachmentData,
                                       isRoomEncrypted: Boolean,
                                       compressBeforeSending: Boolean): OneTimeWorkRequest {
-        val uploadMediaWorkerParams = UploadContentWorker.Params(sessionId, allLocalEchos, attachment, isRoomEncrypted, compressBeforeSending)
+        val localEchoIds = allLocalEchos.map {
+            LocalEchoIdentifiers(it.roomId!!, it.eventId!!)
+        }
+        val uploadMediaWorkerParams = UploadContentWorker.Params(sessionId, localEchoIds, attachment, isRoomEncrypted, compressBeforeSending)
         val uploadWorkData = WorkerParamsFactory.toData(uploadMediaWorkerParams)
 
         return workManagerProvider.matrixOneTimeWorkRequestBuilder<UploadContentWorker>()

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/EncryptEventWorker.kt

@@ -18,21 +18,23 @@
 package org.matrix.android.sdk.internal.session.room.send
 
 import android.content.Context
-import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
 import org.matrix.android.sdk.api.failure.Failure
 import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.events.model.Event
+import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.toContent
 import org.matrix.android.sdk.api.session.room.send.SendState
 import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
 import org.matrix.android.sdk.internal.crypto.MXEventDecryptionResult
 import org.matrix.android.sdk.internal.crypto.model.MXEncryptEventContentResult
+import org.matrix.android.sdk.internal.database.mapper.ContentMapper
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.util.awaitCallback
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -41,12 +43,12 @@ import javax.inject.Inject
  * Possible next worker    : Always [SendEventWorker]
  */
 internal class EncryptEventWorker(context: Context, params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<EncryptEventWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
             override val sessionId: String,
-            val event: Event,
+            val eventId: String,
             /** Do not encrypt these keys, keep them as is in encrypted content (e.g. m.relates_to) */
             val keepKeys: List<String>? = null,
             override val lastFailureMessage: String? = null
@@ -56,24 +58,15 @@ internal class EncryptEventWorker(context: Context, params: WorkerParameters)
     @Inject lateinit var localEchoRepository: LocalEchoRepository
     @Inject lateinit var cancelSendTracker: CancelSendTracker
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        Timber.v("Start Encrypt work")
+        injector.inject(this)
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+    }
-                ?: return Result.success()
-                        .also { Timber.e("Unable to parse work parameters") }
 
-        Timber.v("## SendEvent: Start Encrypt work for event ${params.event.eventId}")
+    override suspend fun doSafeWork(params: Params): Result {
-        if (params.lastFailureMessage != null) {
+        Timber.v("## SendEvent: Start Encrypt work for event ${params.eventId}")
-            // Transmit the error
-            return Result.success(inputData)
-                    .also { Timber.e("Work cancelled due to input error from parent") }
-        }
 
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
+        val localEvent = localEchoRepository.getUpToDateEcho(params.eventId)
-        sessionComponent.inject(this)
+        if (localEvent?.eventId == null) {
-
-        val localEvent = params.event
-        if (localEvent.eventId == null) {
             return Result.success()
         }
 
@@ -106,15 +99,10 @@ internal class EncryptEventWorker(context: Context, params: WorkerParameters)
                     modifiedContent[toKeep] = it
                 }
             }
-            val safeResult = result.copy(eventContent = modifiedContent)
-            val encryptedEvent = localEvent.copy(
-                    type = safeResult.eventType,
-                    content = safeResult.eventContent
-            )
             // Better handling of local echo, to avoid decrypting transition on remote echo
             // Should I only do it for text messages?
-            if (result.eventContent["algorithm"] == MXCRYPTO_ALGORITHM_MEGOLM) {
+            val decryptionLocalEcho = if (result.eventContent["algorithm"] == MXCRYPTO_ALGORITHM_MEGOLM) {
-                val decryptionLocalEcho = MXEventDecryptionResult(
+                MXEventDecryptionResult(
                         clearEvent = Event(
                                 type = localEvent.type,
                                 content = localEvent.content,
@@ -124,10 +112,18 @@ internal class EncryptEventWorker(context: Context, params: WorkerParameters)
                         senderCurve25519Key = result.eventContent["sender_key"] as? String,
                         claimedEd25519Key = crypto.getMyDevice().fingerprint()
                 )
-                localEchoRepository.updateEncryptedEcho(localEvent.eventId, safeResult.eventContent, decryptionLocalEcho)
+            } else {
+                null
+            }
+            localEchoRepository.updateEcho(localEvent.eventId) { _, localEcho ->
+                localEcho.type = EventType.ENCRYPTED
+                localEcho.content = ContentMapper.map(modifiedContent)
+                decryptionLocalEcho?.also {
+                    localEcho.setDecryptionResult(it)
+                }
             }
 
-            val nextWorkerParams = SendEventWorker.Params(sessionId = params.sessionId, event = encryptedEvent)
+            val nextWorkerParams = SendEventWorker.Params(sessionId = params.sessionId, eventId = params.eventId)
             return Result.success(WorkerParamsFactory.toData(nextWorkerParams))
         } else {
             val sendState = when (error) {
@@ -138,10 +134,14 @@ internal class EncryptEventWorker(context: Context, params: WorkerParameters)
             // always return success, or the chain will be stuck for ever!
             val nextWorkerParams = SendEventWorker.Params(
                     sessionId = params.sessionId,
-                    event = localEvent,
+                    eventId = localEvent.eventId,
                     lastFailureMessage = error?.localizedMessage ?: "Error"
             )
             return Result.success(WorkerParamsFactory.toData(nextWorkerParams))
         }
     }
+
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoIdentifiers.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.send
+
+import com.squareup.moshi.JsonClass
+
+/**
+ * This is used as a holder to pass necessary data to some workers params.
+ */
+@JsonClass(generateAdapter = true)
+internal data class LocalEchoIdentifiers(val roomId: String, val eventId: String)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoRepository.kt

@@ -18,7 +18,8 @@
 package org.matrix.android.sdk.internal.session.room.send
 
 import com.zhuinden.monarchy.Monarchy
-import org.matrix.android.sdk.api.session.events.model.Content
+import io.realm.Realm
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.toModel
@@ -26,10 +27,11 @@ import org.matrix.android.sdk.api.session.room.model.message.MessageContent
 import org.matrix.android.sdk.api.session.room.model.message.MessageType
 import org.matrix.android.sdk.api.session.room.send.SendState
 import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
-import org.matrix.android.sdk.internal.crypto.MXEventDecryptionResult
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
+import org.matrix.android.sdk.internal.database.asyncTransaction
 import org.matrix.android.sdk.internal.database.helper.nextId
-import org.matrix.android.sdk.internal.database.mapper.ContentMapper
 import org.matrix.android.sdk.internal.database.mapper.TimelineEventMapper
+import org.matrix.android.sdk.internal.database.mapper.asDomain
 import org.matrix.android.sdk.internal.database.mapper.toEntity
 import org.matrix.android.sdk.internal.database.model.EventEntity
 import org.matrix.android.sdk.internal.database.model.EventInsertEntity
@@ -42,13 +44,14 @@ import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
 import org.matrix.android.sdk.internal.session.room.summary.RoomSummaryUpdater
 import org.matrix.android.sdk.internal.session.room.timeline.DefaultTimeline
+import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.util.awaitTransaction
-import io.realm.Realm
-import org.greenrobot.eventbus.EventBus
 import timber.log.Timber
 import javax.inject.Inject
 
 internal class LocalEchoRepository @Inject constructor(@SessionDatabase private val monarchy: Monarchy,
+                                                       private val taskExecutor: TaskExecutor,
+                                                       private val realmSessionProvider: RealmSessionProvider,
                                                        private val roomSummaryUpdater: RoomSummaryUpdater,
                                                        private val eventBus: EventBus,
                                                        private val timelineEventMapper: TimelineEventMapper) {
@@ -59,7 +62,7 @@ internal class LocalEchoRepository @Inject constructor(@SessionDatabase private
         if (event.eventId == null) {
             throw IllegalStateException("You should have set an eventId for your event")
         }
-        val timelineEventEntity = Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+        val timelineEventEntity = realmSessionProvider.withRealm { realm ->
             val eventEntity = event.toEntity(roomId, SendState.UNSENT, System.currentTimeMillis())
             val roomMemberHelper = RoomMemberHelper(realm, roomId)
             val myUser = roomMemberHelper.getLastRoomMember(senderId)
@@ -75,12 +78,12 @@ internal class LocalEchoRepository @Inject constructor(@SessionDatabase private
         }
         val timelineEvent = timelineEventMapper.map(timelineEventEntity)
         eventBus.post(DefaultTimeline.OnLocalEchoCreated(roomId = roomId, timelineEvent = timelineEvent))
-        monarchy.writeAsync { realm ->
+        taskExecutor.executorScope.asyncTransaction(monarchy) { realm ->
             val eventInsertEntity = EventInsertEntity(event.eventId, event.type).apply {
                 this.insertType = EventInsertType.LOCAL_ECHO
             }
             realm.insert(eventInsertEntity)
-            val roomEntity = RoomEntity.where(realm, roomId = roomId).findFirst() ?: return@writeAsync
+            val roomEntity = RoomEntity.where(realm, roomId = roomId).findFirst() ?: return@asyncTransaction
             roomEntity.sendingTimelineEvents.add(0, timelineEventEntity)
             roomSummaryUpdater.updateSendingInformation(realm, roomId)
         }
@@ -88,30 +91,41 @@ internal class LocalEchoRepository @Inject constructor(@SessionDatabase private
 
     fun updateSendState(eventId: String, sendState: SendState) {
         Timber.v("## SendEvent: [${System.currentTimeMillis()}] Update local state of $eventId to ${sendState.name}")
-        monarchy.writeAsync { realm ->
+        updateEchoAsync(eventId) { realm, sendingEventEntity ->
+            if (sendState == SendState.SENT && sendingEventEntity.sendState == SendState.SYNCED) {
+                // If already synced, do not put as sent
+            } else {
+                sendingEventEntity.sendState = sendState
+            }
+            roomSummaryUpdater.updateSendingInformation(realm, sendingEventEntity.roomId)
+        }
+    }
+
+    suspend fun updateEcho(eventId: String, block: (realm: Realm, eventEntity: EventEntity) -> Unit) {
+        monarchy.awaitTransaction { realm ->
             val sendingEventEntity = EventEntity.where(realm, eventId).findFirst()
             if (sendingEventEntity != null) {
-                if (sendState == SendState.SENT && sendingEventEntity.sendState == SendState.SYNCED) {
+                block(realm, sendingEventEntity)
-                    // If already synced, do not put as sent
-                } else {
-                    sendingEventEntity.sendState = sendState
-                }
-                roomSummaryUpdater.updateSendingInformation(realm, sendingEventEntity.roomId)
             }
         }
     }
 
-    fun updateEncryptedEcho(eventId: String, encryptedContent: Content, mxEventDecryptionResult: MXEventDecryptionResult) {
+    fun updateEchoAsync(eventId: String, block: (realm: Realm, eventEntity: EventEntity) -> Unit) {
-        monarchy.writeAsync { realm ->
+        taskExecutor.executorScope.asyncTransaction(monarchy) { realm ->
             val sendingEventEntity = EventEntity.where(realm, eventId).findFirst()
             if (sendingEventEntity != null) {
-                sendingEventEntity.type = EventType.ENCRYPTED
+                block(realm, sendingEventEntity)
-                sendingEventEntity.content = ContentMapper.map(encryptedContent)
-                sendingEventEntity.setDecryptionResult(mxEventDecryptionResult)
             }
         }
     }
 
+    suspend fun getUpToDateEcho(eventId: String): Event? {
+        // We are using awaitTransaction here to make sure this executes after other transactions
+        return monarchy.awaitTransaction { realm ->
+            EventEntity.where(realm, eventId).findFirst()?.asDomain(castJsonNumbers = true)
+        }
+    }
+
     suspend fun deleteFailedEcho(roomId: String, localEcho: TimelineEvent) {
         deleteFailedEcho(roomId, localEcho.eventId)
     }
@@ -149,8 +163,8 @@ internal class LocalEchoRepository @Inject constructor(@SessionDatabase private
         return getAllEventsWithStates(roomId, SendState.HAS_FAILED_STATES)
     }
 
-    fun getAllEventsWithStates(roomId: String, states : List<SendState>): List<TimelineEvent> {
+    fun getAllEventsWithStates(roomId: String, states: List<SendState>): List<TimelineEvent> {
-        return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+        return realmSessionProvider.withRealm { realm ->
             TimelineEventEntity
                     .findAllInRoomWithSendStates(realm, roomId, states)
                     .sortedByDescending { it.displayIndex }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/MultipleEventSendingDispatcherWorker.kt

@@ -19,18 +19,17 @@ package org.matrix.android.sdk.internal.session.room.send
 
 import android.content.Context
 import androidx.work.BackoffPolicy
-import androidx.work.CoroutineWorker
 import androidx.work.OneTimeWorkRequest
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.room.send.SendState
 import org.matrix.android.sdk.internal.di.WorkManagerProvider
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.session.content.UploadContentWorker
 import org.matrix.android.sdk.internal.session.room.timeline.TimelineSendEventWorkCommon
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
 import org.matrix.android.sdk.internal.worker.startChain
 import timber.log.Timber
 import java.util.concurrent.TimeUnit
@@ -43,12 +42,12 @@ import javax.inject.Inject
  * Possible next worker    : None, but it will post new work to send events, encrypted or not
  */
 internal class MultipleEventSendingDispatcherWorker(context: Context, params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<MultipleEventSendingDispatcherWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
             override val sessionId: String,
-            val events: List<Event>,
+            val localEchoIds: List<LocalEchoIdentifiers>,
             val isEncrypted: Boolean,
             override val lastFailureMessage: String? = null
     ) : SessionWorkerParams
@@ -57,46 +56,48 @@ internal class MultipleEventSendingDispatcherWorker(context: Context, params: Wo
     @Inject lateinit var timelineSendEventWorkCommon: TimelineSendEventWorkCommon
     @Inject lateinit var localEchoRepository: LocalEchoRepository
 
-    override suspend fun doWork(): Result {
+    override fun doOnError(params: Params): Result {
-        Timber.v("## SendEvent: Start dispatch sending multiple event work")
+        params.localEchoIds.forEach { localEchoIds ->
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+            localEchoRepository.updateSendState(localEchoIds.eventId, SendState.UNDELIVERED)
-                ?: return Result.success()
-                        .also { Timber.e("Unable to parse work parameters") }
-
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
-        sessionComponent.inject(this)
-
-        if (params.lastFailureMessage != null) {
-            params.events.forEach { event ->
-                event.eventId?.let { localEchoRepository.updateSendState(it, SendState.UNDELIVERED) }
-            }
-            // Transmit the error if needed?
-            return Result.success(inputData)
-                    .also { Timber.e("## SendEvent: Work cancelled due to input error from parent ${params.lastFailureMessage}") }
         }
 
+        return super.doOnError(params)
+    }
+
+    override fun injectWith(injector: SessionComponent) {
+        injector.inject(this)
+    }
+
+    override suspend fun doSafeWork(params: Params): Result {
+        Timber.v("## SendEvent: Start dispatch sending multiple event work")
         // Create a work for every event
-        params.events.forEach { event ->
+        params.localEchoIds.forEach { localEchoIds ->
+            val roomId = localEchoIds.roomId
+            val eventId = localEchoIds.eventId
             if (params.isEncrypted) {
-                localEchoRepository.updateSendState(event.eventId ?: "", SendState.ENCRYPTING)
+                localEchoRepository.updateSendState(eventId, SendState.ENCRYPTING)
-                Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule encrypt and send event ${event.eventId}")
+                Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule encrypt and send event $eventId")
-                val encryptWork = createEncryptEventWork(params.sessionId, event, true)
+                val encryptWork = createEncryptEventWork(params.sessionId, eventId, true)
                 // Note that event will be replaced by the result of the previous work
-                val sendWork = createSendEventWork(params.sessionId, event, false)
+                val sendWork = createSendEventWork(params.sessionId, eventId, false)
-                timelineSendEventWorkCommon.postSequentialWorks(event.roomId!!, encryptWork, sendWork)
+                timelineSendEventWorkCommon.postSequentialWorks(roomId, encryptWork, sendWork)
             } else {
-                localEchoRepository.updateSendState(event.eventId ?: "", SendState.SENDING)
+                localEchoRepository.updateSendState(eventId, SendState.SENDING)
-                Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule send event ${event.eventId}")
+                Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule send event $eventId")
-                val sendWork = createSendEventWork(params.sessionId, event, true)
+                val sendWork = createSendEventWork(params.sessionId, eventId, true)
-                timelineSendEventWorkCommon.postWork(event.roomId!!, sendWork)
+                timelineSendEventWorkCommon.postWork(roomId, sendWork)
             }
         }
 
         return Result.success()
     }
 
-    private fun createEncryptEventWork(sessionId: String, event: Event, startChain: Boolean): OneTimeWorkRequest {
+    override fun buildErrorParams(params: Params, message: String): Params {
-        val params = EncryptEventWorker.Params(sessionId, event)
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
+
+    private fun createEncryptEventWork(sessionId: String, eventId: String, startChain: Boolean): OneTimeWorkRequest {
+        val params = EncryptEventWorker.Params(sessionId, eventId)
         val sendWorkData = WorkerParamsFactory.toData(params)
 
         return workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
@@ -107,8 +108,8 @@ internal class MultipleEventSendingDispatcherWorker(context: Context, params: Wo
                 .build()
     }
 
-    private fun createSendEventWork(sessionId: String, event: Event, startChain: Boolean): OneTimeWorkRequest {
+    private fun createSendEventWork(sessionId: String, eventId: String, startChain: Boolean): OneTimeWorkRequest {
-        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, event = event)
+        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, eventId = eventId)
         val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
 
         return timelineSendEventWorkCommon.createWork<SendEventWorker>(sendWorkData, startChain)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/RedactEventWorker.kt

@@ -17,24 +17,24 @@
 package org.matrix.android.sdk.internal.session.room.send
 
 import android.content.Context
-import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.failure.Failure
 import org.matrix.android.sdk.internal.network.executeRequest
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.session.room.RoomAPI
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
-import org.greenrobot.eventbus.EventBus
-import timber.log.Timber
 import javax.inject.Inject
 
 /**
  * Possible previous worker: None
  * Possible next worker    : None
  */
-internal class RedactEventWorker(context: Context, params: WorkerParameters) : CoroutineWorker(context, params) {
+internal class RedactEventWorker(context: Context, params: WorkerParameters)
+    : SessionSafeCoroutineWorker<RedactEventWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
@@ -49,20 +49,11 @@ internal class RedactEventWorker(context: Context, params: WorkerParameters) : C
     @Inject lateinit var roomAPI: RoomAPI
     @Inject lateinit var eventBus: EventBus
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+        injector.inject(this)
-                ?: return Result.failure()
+    }
-                        .also { Timber.e("Unable to parse work parameters") }
-
-        if (params.lastFailureMessage != null) {
-            // Transmit the error
-            return Result.success(inputData)
-                    .also { Timber.e("Work cancelled due to input error from parent") }
-        }
-
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
-        sessionComponent.inject(this)
 
+    override suspend fun doSafeWork(params: Params): Result {
         val eventId = params.eventId
         return runCatching {
             executeRequest<SendResponse>(eventBus) {
@@ -91,4 +82,8 @@ internal class RedactEventWorker(context: Context, params: WorkerParameters) : C
                 }
         )
     }
+
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/RoomEventSender.kt

@@ -56,7 +56,7 @@ internal class RoomEventSender @Inject constructor(
 
     private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
         // Same parameter
-        val params = EncryptEventWorker.Params(sessionId, event)
+        val params = EncryptEventWorker.Params(sessionId, event.eventId!!)
         val sendWorkData = WorkerParamsFactory.toData(params)
 
         return workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
@@ -68,7 +68,7 @@ internal class RoomEventSender @Inject constructor(
     }
 
     private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, event = event)
+        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, eventId = event.eventId!!)
         val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
 
         return timelineSendEventWorkCommon.createWork<SendEventWorker>(sendWorkData, startChain)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/SendEventWorker.kt

@@ -18,19 +18,19 @@
 package org.matrix.android.sdk.internal.session.room.send
 
 import android.content.Context
-import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
+import io.realm.RealmConfiguration
 import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.failure.shouldBeRetried
 import org.matrix.android.sdk.api.session.events.model.Content
-import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.network.executeRequest
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.session.room.RoomAPI
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -42,35 +42,29 @@ import javax.inject.Inject
  */
 internal class SendEventWorker(context: Context,
                                params: WorkerParameters)
-    : CoroutineWorker(context, params) {
+    : SessionSafeCoroutineWorker<SendEventWorker.Params>(context, params, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
             override val sessionId: String,
             override val lastFailureMessage: String? = null,
-            val event: Event? = null,
+            val eventId: String
-            // Keep for compat at the moment, will be removed later
-            val eventId: String? = null
     ) : SessionWorkerParams
 
     @Inject lateinit var localEchoRepository: LocalEchoRepository
     @Inject lateinit var roomAPI: RoomAPI
     @Inject lateinit var eventBus: EventBus
     @Inject lateinit var cancelSendTracker: CancelSendTracker
+    @SessionDatabase @Inject lateinit var realmConfiguration: RealmConfiguration
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
+        injector.inject(this)
-                ?: return Result.success()
+    }
-                        .also { Timber.e("## SendEvent: Unable to parse work parameters") }
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
-        sessionComponent.inject(this)
 
-        val event = params.event
+    override suspend fun doSafeWork(params: Params): Result {
+        val event = localEchoRepository.getUpToDateEcho(params.eventId)
         if (event?.eventId == null || event.roomId == null) {
-            // Old way of sending
+            localEchoRepository.updateSendState(params.eventId, SendState.UNDELIVERED)
-            if (params.eventId != null) {
-                localEchoRepository.updateSendState(params.eventId, SendState.UNDELIVERED)
-            }
             return Result.success()
                     .also { Timber.e("Work cancelled due to bad input data") }
         }
@@ -106,6 +100,10 @@ internal class SendEventWorker(context: Context,
         }
     }
 
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
+
     private suspend fun sendEvent(eventId: String, roomId: String, type: String, content: Content?) {
         localEchoRepository.updateSendState(eventId, SendState.SENDING)
         executeRequest<SendResponse>(eventBus) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/state/StateEventDataSource.kt

@@ -20,24 +20,26 @@ package org.matrix.android.sdk.internal.session.room.state
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.Transformations
 import com.zhuinden.monarchy.Monarchy
+import io.realm.Realm
+import io.realm.RealmQuery
+import io.realm.kotlin.where
 import org.matrix.android.sdk.api.query.QueryStringValue
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.util.Optional
 import org.matrix.android.sdk.api.util.toOptional
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.mapper.asDomain
 import org.matrix.android.sdk.internal.database.model.CurrentStateEventEntity
 import org.matrix.android.sdk.internal.database.model.CurrentStateEventEntityFields
 import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.query.process
-import io.realm.Realm
-import io.realm.RealmQuery
-import io.realm.kotlin.where
 import javax.inject.Inject
 
-internal class StateEventDataSource @Inject constructor(@SessionDatabase private val monarchy: Monarchy) {
+internal class StateEventDataSource @Inject constructor(@SessionDatabase private val monarchy: Monarchy,
+                                                        private val realmSessionProvider: RealmSessionProvider) {
 
     fun getStateEvent(roomId: String, eventType: String, stateKey: QueryStringValue): Event? {
-        return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+        return realmSessionProvider.withRealm { realm ->
             buildStateEventQuery(realm, roomId, setOf(eventType), stateKey).findFirst()?.root?.asDomain()
         }
     }
@@ -53,7 +55,7 @@ internal class StateEventDataSource @Inject constructor(@SessionDatabase private
     }
 
     fun getStateEvents(roomId: String, eventTypes: Set<String>, stateKey: QueryStringValue): List<Event> {
-        return Realm.getInstance(monarchy.realmConfiguration).use { realm ->
+        return realmSessionProvider.withRealm { realm ->
             buildStateEventQuery(realm, roomId, eventTypes, stateKey)
                     .findAll()
                     .mapNotNull {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/summary/RoomSummaryEventsHelper.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.summary
+
+import io.realm.Realm
+import org.matrix.android.sdk.api.session.room.summary.RoomSummaryConstants
+import org.matrix.android.sdk.api.session.room.timeline.TimelineEventFilters
+import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
+import org.matrix.android.sdk.internal.database.query.latestEvent
+
+internal object RoomSummaryEventsHelper {
+
+    private val previewFilters = TimelineEventFilters(
+            filterTypes = true,
+            allowedTypes = RoomSummaryConstants.PREVIEWABLE_TYPES,
+            filterUseless = true,
+            filterRedacted = false,
+            filterEdits = true
+    )
+
+    fun getLatestPreviewableEvent(realm: Realm, roomId: String): TimelineEventEntity? {
+        return TimelineEventEntity.latestEvent(
+                realm = realm,
+                roomId = roomId,
+                includesSending = true,
+                filters = previewFilters
+        )
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/summary/RoomSummaryUpdater.kt

@@ -18,6 +18,8 @@
 package org.matrix.android.sdk.internal.session.room.summary
 
 import dagger.Lazy
+import io.realm.Realm
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.crypto.RoomEncryptionTrustLevel
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.toModel
@@ -40,7 +42,6 @@ import org.matrix.android.sdk.internal.database.query.findAllInRoomWithSendState
 import org.matrix.android.sdk.internal.database.query.getOrCreate
 import org.matrix.android.sdk.internal.database.query.getOrNull
 import org.matrix.android.sdk.internal.database.query.isEventRead
-import org.matrix.android.sdk.internal.database.query.latestEvent
 import org.matrix.android.sdk.internal.database.query.whereType
 import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.session.room.RoomAvatarResolver
@@ -49,8 +50,6 @@ import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
 import org.matrix.android.sdk.internal.session.room.timeline.TimelineEventDecryptor
 import org.matrix.android.sdk.internal.session.sync.model.RoomSyncSummary
 import org.matrix.android.sdk.internal.session.sync.model.RoomSyncUnreadNotifications
-import io.realm.Realm
-import org.greenrobot.eventbus.EventBus
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -61,28 +60,6 @@ internal class RoomSummaryUpdater @Inject constructor(
         private val timelineEventDecryptor: Lazy<TimelineEventDecryptor>,
         private val eventBus: EventBus) {
 
-    companion object {
-        // TODO: maybe allow user of SDK to give that list
-        val PREVIEWABLE_TYPES = listOf(
-                // TODO filter message type (KEY_VERIFICATION_READY, etc.)
-                EventType.MESSAGE,
-                EventType.STATE_ROOM_NAME,
-                EventType.STATE_ROOM_TOPIC,
-                EventType.STATE_ROOM_AVATAR,
-                EventType.STATE_ROOM_MEMBER,
-                EventType.STATE_ROOM_HISTORY_VISIBILITY,
-                EventType.CALL_INVITE,
-                EventType.CALL_HANGUP,
-                EventType.CALL_ANSWER,
-                EventType.ENCRYPTED,
-                EventType.STATE_ROOM_ENCRYPTION,
-                EventType.STATE_ROOM_THIRD_PARTY_INVITE,
-                EventType.STICKER,
-                EventType.REACTION,
-                EventType.STATE_ROOM_CREATE
-        )
-    }
-
     fun update(realm: Realm,
                roomId: String,
                membership: Membership? = null,
@@ -110,9 +87,6 @@ internal class RoomSummaryUpdater @Inject constructor(
             roomSummaryEntity.membership = membership
         }
 
-        val latestPreviewableEvent = TimelineEventEntity.latestEvent(realm, roomId, includesSending = true,
-                filterTypes = PREVIEWABLE_TYPES, filterContentRelation = true)
-
         val lastNameEvent = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_NAME, stateKey = "")?.root
         val lastTopicEvent = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_TOPIC, stateKey = "")?.root
         val lastCanonicalAliasEvent = CurrentStateEventEntity.getOrNull(realm, roomId, type = EventType.STATE_ROOM_CANONICAL_ALIAS, stateKey = "")?.root
@@ -123,6 +97,8 @@ internal class RoomSummaryUpdater @Inject constructor(
                 .contains(EventEntityFields.CONTENT, "\"algorithm\":\"$MXCRYPTO_ALGORITHM_MEGOLM\"")
                 .findFirst()
 
+        val latestPreviewableEvent = RoomSummaryEventsHelper.getLatestPreviewableEvent(realm, roomId)
+
         roomSummaryEntity.hasUnreadMessages = roomSummaryEntity.notificationCount > 0
                 // avoid this call if we are sure there are unread events
                 || !isEventRead(realm.configuration, userId, roomId, latestPreviewableEvent?.eventId)
@@ -178,8 +154,7 @@ internal class RoomSummaryUpdater @Inject constructor(
     fun updateSendingInformation(realm: Realm, roomId: String) {
         val roomSummaryEntity = RoomSummaryEntity.getOrCreate(realm, roomId)
         roomSummaryEntity.updateHasFailedSending()
-        roomSummaryEntity.latestPreviewableEvent = TimelineEventEntity.latestEvent(realm, roomId, includesSending = true,
+        roomSummaryEntity.latestPreviewableEvent = RoomSummaryEventsHelper.getLatestPreviewableEvent(realm, roomId)
-                filterTypes = PREVIEWABLE_TYPES, filterContentRelation = true)
     }
 
     fun updateShieldTrust(realm: Realm,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/DefaultTimeline.kt

@@ -17,33 +17,6 @@
 
 package org.matrix.android.sdk.internal.session.room.timeline
 
-import org.matrix.android.sdk.api.MatrixCallback
-import org.matrix.android.sdk.api.extensions.orFalse
-import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.RelationType
-import org.matrix.android.sdk.api.session.events.model.toModel
-import org.matrix.android.sdk.api.session.room.model.ReadReceipt
-import org.matrix.android.sdk.api.session.room.model.message.MessageContent
-import org.matrix.android.sdk.api.session.room.send.SendState
-import org.matrix.android.sdk.api.session.room.timeline.Timeline
-import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
-import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
-import org.matrix.android.sdk.api.util.CancelableBag
-import org.matrix.android.sdk.internal.database.mapper.TimelineEventMapper
-import org.matrix.android.sdk.internal.database.model.ChunkEntity
-import org.matrix.android.sdk.internal.database.model.ChunkEntityFields
-import org.matrix.android.sdk.internal.database.model.RoomEntity
-import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
-import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
-import org.matrix.android.sdk.internal.database.query.TimelineEventFilter
-import org.matrix.android.sdk.internal.database.query.findAllInRoomWithSendStates
-import org.matrix.android.sdk.internal.database.query.where
-import org.matrix.android.sdk.internal.database.query.whereRoomId
-import org.matrix.android.sdk.internal.task.TaskExecutor
-import org.matrix.android.sdk.internal.task.configureWith
-import org.matrix.android.sdk.internal.util.Debouncer
-import org.matrix.android.sdk.internal.util.createBackgroundHandler
-import org.matrix.android.sdk.internal.util.createUIHandler
 import io.realm.OrderedCollectionChangeSet
 import io.realm.OrderedRealmCollectionChangeListener
 import io.realm.Realm
@@ -54,6 +27,35 @@ import io.realm.Sort
 import org.greenrobot.eventbus.EventBus
 import org.greenrobot.eventbus.Subscribe
 import org.greenrobot.eventbus.ThreadMode
+import org.matrix.android.sdk.api.MatrixCallback
+import org.matrix.android.sdk.api.extensions.orFalse
+import org.matrix.android.sdk.api.extensions.tryOrNull
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.RelationType
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.room.model.ReadReceipt
+import org.matrix.android.sdk.api.session.room.model.message.MessageContent
+import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.api.session.room.timeline.Timeline
+import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
+import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
+import org.matrix.android.sdk.api.util.CancelableBag
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
+import org.matrix.android.sdk.internal.database.mapper.TimelineEventMapper
+import org.matrix.android.sdk.internal.database.model.ChunkEntity
+import org.matrix.android.sdk.internal.database.model.ChunkEntityFields
+import org.matrix.android.sdk.internal.database.model.RoomEntity
+import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
+import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
+import org.matrix.android.sdk.internal.database.query.filterEvents
+import org.matrix.android.sdk.internal.database.query.findAllInRoomWithSendStates
+import org.matrix.android.sdk.internal.database.query.where
+import org.matrix.android.sdk.internal.database.query.whereRoomId
+import org.matrix.android.sdk.internal.task.TaskExecutor
+import org.matrix.android.sdk.internal.task.configureWith
+import org.matrix.android.sdk.internal.util.Debouncer
+import org.matrix.android.sdk.internal.util.createBackgroundHandler
+import org.matrix.android.sdk.internal.util.createUIHandler
 import timber.log.Timber
 import java.util.Collections
 import java.util.UUID
@@ -76,7 +78,8 @@ internal class DefaultTimeline(
         private val settings: TimelineSettings,
         private val hiddenReadReceipts: TimelineHiddenReadReceipts,
         private val eventBus: EventBus,
-        private val eventDecryptor: TimelineEventDecryptor
+        private val eventDecryptor: TimelineEventDecryptor,
+        private val realmSessionProvider: RealmSessionProvider
 ) : Timeline, TimelineHiddenReadReceipts.Delegate {
 
     data class OnNewTimelineEvents(val roomId: String, val eventIds: List<String>)
@@ -136,13 +139,13 @@ internal class DefaultTimeline(
     }
 
     override fun pendingEventCount(): Int {
-        return Realm.getInstance(realmConfiguration).use {
+        return realmSessionProvider.withRealm {
             RoomEntity.where(it, roomId).findFirst()?.sendingTimelineEvents?.count() ?: 0
         }
     }
 
     override fun failedToDeliverEventCount(): Int {
-        return Realm.getInstance(realmConfiguration).use {
+        return realmSessionProvider.withRealm {
             TimelineEventEntity.findAllInRoomWithSendStates(it, roomId, SendState.HAS_FAILED_STATES).count()
         }
     }
@@ -182,7 +185,7 @@ internal class DefaultTimeline(
     }
 
     private fun TimelineSettings.shouldHandleHiddenReadReceipts(): Boolean {
-        return buildReadReceipts && (filterEdits || filterTypes)
+        return buildReadReceipts && (filters.filterEdits || filters.filterTypes)
     }
 
     override fun dispose() {
@@ -239,7 +242,7 @@ internal class DefaultTimeline(
             return eventId
         }
         // Otherwise, we should check if the event is in the db, but is hidden because of filters
-        return Realm.getInstance(realmConfiguration).use { localRealm ->
+        return realmSessionProvider.withRealm { localRealm ->
             val nonFilteredEvents = buildEventQuery(localRealm)
                     .sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.DESCENDING)
                     .findAll()
@@ -317,23 +320,36 @@ internal class DefaultTimeline(
     @Subscribe(threadMode = ThreadMode.MAIN)
     fun onLocalEchoCreated(onLocalEchoCreated: OnLocalEchoCreated) {
         if (isLive && onLocalEchoCreated.roomId == roomId) {
-            listeners.forEach {
+            // do not add events that would have been filtered
-                it.onNewTimelineEvents(listOf(onLocalEchoCreated.timelineEvent.eventId))
+            if (listOf(onLocalEchoCreated.timelineEvent).filterEventsWithSettings().isNotEmpty()) {
+                listeners.forEach {
+                    it.onNewTimelineEvents(listOf(onLocalEchoCreated.timelineEvent.eventId))
+                }
+                Timber.v("On local echo created: ${onLocalEchoCreated.timelineEvent.eventId}")
+                inMemorySendingEvents.add(0, onLocalEchoCreated.timelineEvent)
+                postSnapshot()
             }
-            Timber.v("On local echo created: $onLocalEchoCreated")
-            inMemorySendingEvents.add(0, onLocalEchoCreated.timelineEvent)
-            postSnapshot()
         }
     }
 
 // Private methods *****************************************************************************
 
-    private fun rebuildEvent(eventId: String, builder: (TimelineEvent) -> TimelineEvent): Boolean {
+    private fun rebuildEvent(eventId: String, builder: (TimelineEvent) -> TimelineEvent?): Boolean {
-        return builtEventsIdMap[eventId]?.let { builtIndex ->
+        return tryOrNull {
-            // Update the relation of existing event
+            builtEventsIdMap[eventId]?.let { builtIndex ->
-            builtEvents[builtIndex]?.let { te ->
+                // Update the relation of existing event
-                builtEvents[builtIndex] = builder(te)
+                builtEvents[builtIndex]?.let { te ->
-                true
+                    val rebuiltEvent = builder(te)
+                    // If rebuilt event is filtered its returned as null and should be removed.
+                    if (rebuiltEvent == null) {
+                        builtEventsIdMap.remove(eventId)
+                        builtEventsIdMap.entries.filter { it.value > builtIndex }.forEach { it.setValue(it.value - 1) }
+                        builtEvents.removeAt(builtIndex)
+                    } else {
+                        builtEvents[builtIndex] = rebuiltEvent
+                    }
+                    true
+                }
             }
         } ?: false
     }
@@ -408,14 +424,14 @@ internal class DefaultTimeline(
 
     private fun getState(direction: Timeline.Direction): State {
         return when (direction) {
-            Timeline.Direction.FORWARDS  -> forwardsState.get()
+            Timeline.Direction.FORWARDS -> forwardsState.get()
             Timeline.Direction.BACKWARDS -> backwardsState.get()
         }
     }
 
     private fun updateState(direction: Timeline.Direction, update: (State) -> State) {
         val stateReference = when (direction) {
-            Timeline.Direction.FORWARDS  -> forwardsState
+            Timeline.Direction.FORWARDS -> forwardsState
             Timeline.Direction.BACKWARDS -> backwardsState
         }
         val currentValue = stateReference.get()
@@ -484,7 +500,8 @@ internal class DefaultTimeline(
             val eventEntity = results[index]
             eventEntity?.eventId?.let { eventId ->
                 postSnapshot = rebuildEvent(eventId) {
-                    buildTimelineEvent(eventEntity)
+                    val builtEvent = buildTimelineEvent(eventEntity)
+                    listOf(builtEvent).filterEventsWithSettings().firstOrNull()
                 } || postSnapshot
             }
         }
@@ -725,10 +742,10 @@ internal class DefaultTimeline(
         return object : MatrixCallback<TokenChunkEventPersistor.Result> {
             override fun onSuccess(data: TokenChunkEventPersistor.Result) {
                 when (data) {
-                    TokenChunkEventPersistor.Result.SUCCESS           -> {
+                    TokenChunkEventPersistor.Result.SUCCESS -> {
                         Timber.v("Success fetching $limit items $direction from pagination request")
                     }
-                    TokenChunkEventPersistor.Result.REACHED_END       -> {
+                    TokenChunkEventPersistor.Result.REACHED_END -> {
                         postSnapshot()
                     }
                     TokenChunkEventPersistor.Result.SHOULD_FETCH_MORE ->
@@ -754,39 +771,24 @@ internal class DefaultTimeline(
     }
 
     private fun RealmQuery<TimelineEventEntity>.filterEventsWithSettings(): RealmQuery<TimelineEventEntity> {
-        if (settings.filterTypes) {
+        return filterEvents(settings.filters)
-            `in`(TimelineEventEntityFields.ROOT.TYPE, settings.allowedTypes.toTypedArray())
-        }
-        if (settings.filterUseless) {
-            not()
-                    .equalTo(TimelineEventEntityFields.ROOT.IS_USELESS, true)
-        }
-        if (settings.filterEdits) {
-            not().like(TimelineEventEntityFields.ROOT.CONTENT, TimelineEventFilter.Content.EDIT)
-            not().like(TimelineEventEntityFields.ROOT.CONTENT, TimelineEventFilter.Content.RESPONSE)
-        }
-        if (settings.filterRedacted) {
-            not().like(TimelineEventEntityFields.ROOT.UNSIGNED_DATA, TimelineEventFilter.Unsigned.REDACTED)
-        }
-        return this
     }
 
     private fun List<TimelineEvent>.filterEventsWithSettings(): List<TimelineEvent> {
         return filter {
-            val filterType = !settings.filterTypes || settings.allowedTypes.contains(it.root.type)
+            val filterType = !settings.filters.filterTypes || settings.filters.allowedTypes.contains(it.root.type)
             if (!filterType) return@filter false
 
-            val filterEdits = if (settings.filterEdits && it.root.type == EventType.MESSAGE) {
+            val filterEdits = if (settings.filters.filterEdits && it.root.type == EventType.MESSAGE) {
                 val messageContent = it.root.content.toModel<MessageContent>()
-                messageContent?.relatesTo?.type != RelationType.REPLACE
+                messageContent?.relatesTo?.type != RelationType.REPLACE && messageContent?.relatesTo?.type != RelationType.RESPONSE
             } else {
                 true
             }
             if (!filterEdits) return@filter false
 
-            val filterRedacted = !settings.filterRedacted || it.root.isRedacted()
+            val filterRedacted = settings.filters.filterRedacted && it.root.isRedacted()
-
+            !filterRedacted
-            filterRedacted
         }
     }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/DefaultTimelineService.kt

@@ -22,6 +22,9 @@ import androidx.lifecycle.Transformations
 import com.squareup.inject.assisted.Assisted
 import com.squareup.inject.assisted.AssistedInject
 import com.zhuinden.monarchy.Monarchy
+import io.realm.Sort
+import io.realm.kotlin.where
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.session.events.model.isImageMessage
 import org.matrix.android.sdk.api.session.events.model.isVideoMessage
 import org.matrix.android.sdk.api.session.room.timeline.Timeline
@@ -30,7 +33,7 @@ import org.matrix.android.sdk.api.session.room.timeline.TimelineService
 import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
 import org.matrix.android.sdk.api.util.Optional
 import org.matrix.android.sdk.api.util.toOptional
-import org.matrix.android.sdk.internal.crypto.store.db.doWithRealm
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.mapper.ReadReceiptsSummaryMapper
 import org.matrix.android.sdk.internal.database.mapper.TimelineEventMapper
 import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
@@ -38,13 +41,10 @@ import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.task.TaskExecutor
-import org.matrix.android.sdk.internal.util.fetchCopyMap
-import io.realm.Sort
-import io.realm.kotlin.where
-import org.greenrobot.eventbus.EventBus
 
 internal class DefaultTimelineService @AssistedInject constructor(@Assisted private val roomId: String,
                                                                   @SessionDatabase private val monarchy: Monarchy,
+                                                                  private val realmSessionProvider: RealmSessionProvider,
                                                                   private val eventBus: EventBus,
                                                                   private val taskExecutor: TaskExecutor,
                                                                   private val contextOfEventTask: GetContextOfEventTask,
@@ -73,17 +73,17 @@ internal class DefaultTimelineService @AssistedInject constructor(@Assisted priv
                 hiddenReadReceipts = TimelineHiddenReadReceipts(readReceiptsSummaryMapper, roomId, settings),
                 eventBus = eventBus,
                 eventDecryptor = eventDecryptor,
-                fetchTokenAndPaginateTask = fetchTokenAndPaginateTask
+                fetchTokenAndPaginateTask = fetchTokenAndPaginateTask,
+                realmSessionProvider = realmSessionProvider
         )
     }
 
     override fun getTimeLineEvent(eventId: String): TimelineEvent? {
-        return monarchy
+        return realmSessionProvider.withRealm { realm ->
-                .fetchCopyMap({
+            TimelineEventEntity.where(realm, roomId = roomId, eventId = eventId).findFirst()?.let {
-                    TimelineEventEntity.where(it, roomId = roomId, eventId = eventId).findFirst()
+                timelineEventMapper.map(it)
-                }, { entity, _ ->
+            }
-                    timelineEventMapper.map(entity)
+        }
-                })
     }
 
     override fun getTimeLineEventLive(eventId: String): LiveData<Optional<TimelineEvent>> {
@@ -98,7 +98,7 @@ internal class DefaultTimelineService @AssistedInject constructor(@Assisted priv
 
     override fun getAttachmentMessages(): List<TimelineEvent> {
         // TODO pretty bad query.. maybe we should denormalize clear type in base?
-        return doWithRealm(monarchy.realmConfiguration) { realm ->
+        return realmSessionProvider.withRealm { realm ->
             realm.where<TimelineEventEntity>()
                     .equalTo(TimelineEventEntityFields.ROOM_ID, roomId)
                     .sort(TimelineEventEntityFields.DISPLAY_INDEX, Sort.ASCENDING)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineHiddenReadReceipts.kt

@@ -18,6 +18,10 @@
 package org.matrix.android.sdk.internal.session.room.timeline
 
 import android.util.SparseArray
+import io.realm.OrderedRealmCollectionChangeListener
+import io.realm.Realm
+import io.realm.RealmQuery
+import io.realm.RealmResults
 import org.matrix.android.sdk.api.session.room.model.ReadReceipt
 import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
 import org.matrix.android.sdk.internal.database.mapper.ReadReceiptsSummaryMapper
@@ -27,10 +31,6 @@ import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
 import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
 import org.matrix.android.sdk.internal.database.query.TimelineEventFilter
 import org.matrix.android.sdk.internal.database.query.whereInRoom
-import io.realm.OrderedRealmCollectionChangeListener
-import io.realm.Realm
-import io.realm.RealmQuery
-import io.realm.RealmResults
 
 /**
  * This class is responsible for handling the read receipts for hidden events (check [TimelineSettings] to see filtering).
@@ -151,23 +151,24 @@ internal class TimelineHiddenReadReceipts constructor(private val readReceiptsSu
     private fun RealmQuery<ReadReceiptsSummaryEntity>.filterReceiptsWithSettings(): RealmQuery<ReadReceiptsSummaryEntity> {
         beginGroup()
         var needOr = false
-        if (settings.filterTypes) {
+        if (settings.filters.filterTypes) {
-            not().`in`("${ReadReceiptsSummaryEntityFields.TIMELINE_EVENT}.${TimelineEventEntityFields.ROOT.TYPE}", settings.allowedTypes.toTypedArray())
+            val allowedTypes = settings.filters.allowedTypes.toTypedArray()
+            not().`in`("${ReadReceiptsSummaryEntityFields.TIMELINE_EVENT}.${TimelineEventEntityFields.ROOT.TYPE}", allowedTypes)
             needOr = true
         }
-        if (settings.filterUseless) {
+        if (settings.filters.filterUseless) {
             if (needOr) or()
             equalTo("${ReadReceiptsSummaryEntityFields.TIMELINE_EVENT}.${TimelineEventEntityFields.ROOT.IS_USELESS}", true)
             needOr = true
         }
-        if (settings.filterEdits) {
+        if (settings.filters.filterEdits) {
             if (needOr) or()
             like("${ReadReceiptsSummaryEntityFields.TIMELINE_EVENT}.${TimelineEventEntityFields.ROOT.CONTENT}", TimelineEventFilter.Content.EDIT)
             or()
             like("${ReadReceiptsSummaryEntityFields.TIMELINE_EVENT}.${TimelineEventEntityFields.ROOT.CONTENT}", TimelineEventFilter.Content.RESPONSE)
             needOr = true
         }
-        if (settings.filterRedacted) {
+        if (settings.filters.filterRedacted) {
             if (needOr) or()
             like("${ReadReceiptsSummaryEntityFields.TIMELINE_EVENT}.${TimelineEventEntityFields.ROOT.UNSIGNED_DATA}", TimelineEventFilter.Unsigned.REDACTED)
         }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TokenChunkEventPersistor.kt

@@ -18,6 +18,7 @@
 package org.matrix.android.sdk.internal.session.room.timeline
 
 import com.zhuinden.monarchy.Monarchy
+import io.realm.Realm
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.room.model.RoomMemberContent
@@ -32,19 +33,16 @@ import org.matrix.android.sdk.internal.database.model.ChunkEntity
 import org.matrix.android.sdk.internal.database.model.EventInsertType
 import org.matrix.android.sdk.internal.database.model.RoomEntity
 import org.matrix.android.sdk.internal.database.model.RoomSummaryEntity
-import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
 import org.matrix.android.sdk.internal.database.query.copyToRealmOrIgnore
 import org.matrix.android.sdk.internal.database.query.create
 import org.matrix.android.sdk.internal.database.query.find
 import org.matrix.android.sdk.internal.database.query.findAllIncludingEvents
 import org.matrix.android.sdk.internal.database.query.findLastForwardChunkOfRoom
 import org.matrix.android.sdk.internal.database.query.getOrCreate
-import org.matrix.android.sdk.internal.database.query.latestEvent
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.di.SessionDatabase
-import org.matrix.android.sdk.internal.session.room.summary.RoomSummaryUpdater
+import org.matrix.android.sdk.internal.session.room.summary.RoomSummaryEventsHelper
 import org.matrix.android.sdk.internal.util.awaitTransaction
-import io.realm.Realm
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -177,12 +175,7 @@ internal class TokenChunkEventPersistor @Inject constructor(@SessionDatabase pri
                 currentChunk.isLastForward = true
                 currentLastForwardChunk?.deleteOnCascade()
                 RoomSummaryEntity.where(realm, roomId).findFirst()?.apply {
-                    latestPreviewableEvent = TimelineEventEntity.latestEvent(
+                    latestPreviewableEvent = RoomSummaryEventsHelper.getLatestPreviewableEvent(realm, roomId)
-                            realm,
-                            roomId,
-                            includesSending = true,
-                            filterTypes = RoomSummaryUpdater.PREVIEWABLE_TYPES
-                    )
                 }
             }
         } else {
@@ -249,13 +242,7 @@ internal class TokenChunkEventPersistor @Inject constructor(@SessionDatabase pri
         val shouldUpdateSummary = roomSummaryEntity.latestPreviewableEvent == null
                 || (chunksToDelete.isNotEmpty() && currentChunk.isLastForward && direction == PaginationDirection.FORWARDS)
         if (shouldUpdateSummary) {
-            val latestPreviewableEvent = TimelineEventEntity.latestEvent(
+            roomSummaryEntity.latestPreviewableEvent = RoomSummaryEventsHelper.getLatestPreviewableEvent(realm, roomId)
-                    realm,
-                    roomId,
-                    includesSending = true,
-                    filterTypes = RoomSummaryUpdater.PREVIEWABLE_TYPES
-            )
-            roomSummaryEntity.latestPreviewableEvent = latestPreviewableEvent
         }
         if (currentChunk.isValid) {
             RoomEntity.where(realm, roomId).findFirst()?.addOrUpdate(currentChunk)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt

@@ -131,7 +131,7 @@ internal class SyncResponseHandler @Inject constructor(@SessionDatabase private
 
     /**
      * At the moment we don't get any group data through the sync, so we poll where every hour.
-       You can also force to refetch group data using [Group] API.
+     * You can also force to refetch group data using [Group] API.
      */
     private fun scheduleGroupDataFetchingIfNeeded(groupsSyncResponse: GroupsSyncResponse) {
         val groupIds = ArrayList<String>()

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt

@@ -32,7 +32,7 @@ import javax.inject.Inject
 
 internal interface SyncTask : Task<SyncTask.Params, Unit> {
 
-    data class Params(var timeout: Long = 30_000L)
+    data class Params(var timeout: Long = 6_000L)
 }
 
 internal class DefaultSyncTask @Inject constructor(

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/job/SyncService.kt

@@ -19,7 +19,14 @@ package org.matrix.android.sdk.internal.session.sync.job
 import android.app.Service
 import android.content.Intent
 import android.os.IBinder
+import android.os.PowerManager
+import androidx.core.content.getSystemService
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.cancelChildren
+import kotlinx.coroutines.launch
 import org.matrix.android.sdk.api.Matrix
+import org.matrix.android.sdk.api.failure.Failure
 import org.matrix.android.sdk.api.failure.isTokenError
 import org.matrix.android.sdk.api.session.Session
 import org.matrix.android.sdk.api.session.sync.SyncState
@@ -28,10 +35,6 @@ import org.matrix.android.sdk.internal.session.sync.SyncTask
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.util.BackgroundDetectionObserver
 import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.cancelChildren
-import kotlinx.coroutines.launch
 import timber.log.Timber
 import java.util.concurrent.atomic.AtomicBoolean
 
@@ -46,6 +49,11 @@ abstract class SyncService : Service() {
     private var sessionId: String? = null
     private var mIsSelfDestroyed: Boolean = false
 
+    private var syncTimeoutSeconds: Int = 6
+    private var syncDelaySeconds: Int = 60
+    private var periodic: Boolean = false
+    private var preventReschedule: Boolean = false
+
     private var isInitialSync: Boolean = false
     private lateinit var session: Session
     private lateinit var syncTask: SyncTask
@@ -59,27 +67,60 @@ abstract class SyncService : Service() {
     private val serviceScope = CoroutineScope(SupervisorJob())
 
     override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
-        Timber.i("onStartCommand $intent")
+        Timber.i("## Sync: onStartCommand [$this] $intent with action: ${intent?.action}")
-        val isInit = initialize(intent)
+
-        if (isInit) {
+        // We should start we have to ensure we fulfill contract to show notification
-            onStart(isInitialSync)
+        // for foreground service (as per design for this service)
-            doSyncIfNotAlreadyRunning()
+        // TODO can we check if it's really in foreground
-        } else {
+        onStart(isInitialSync)
-            // We should start and stop as we have to ensure to call Service.startForeground()
+        when (intent?.action) {
-            onStart(isInitialSync)
+            ACTION_STOP -> {
-            stopMe()
+                Timber.i("## Sync: stop command received")
+                // If it was periodic we ensure that it will not reschedule itself
+                preventReschedule = true
+                // we don't want to cancel initial syncs, let it finish
+                if (!isInitialSync) {
+                    stopMe()
+                }
+            }
+            else        -> {
+                val isInit = initialize(intent)
+                if (isInit) {
+                    periodic = intent?.getBooleanExtra(EXTRA_PERIODIC, false) ?: false
+                    val onNetworkBack = intent?.getBooleanExtra(EXTRA_NETWORK_BACK_RESTART, false) ?: false
+                    Timber.d("## Sync: command received, periodic: $periodic  networkBack: $onNetworkBack")
+                    if (onNetworkBack && !backgroundDetectionObserver.isInBackground) {
+                        // the restart after network occurs while the app is in foreground
+                        // so just stop. It will be restarted when entering background
+                        preventReschedule = true
+                        stopMe()
+                    } else {
+                        // default is syncing
+                        doSyncIfNotAlreadyRunning()
+                    }
+                } else {
+                    Timber.d("## Sync: Failed to initialize service")
+                    stopMe()
+                }
+            }
         }
-        // No intent just start the service, an alarm will should call with intent
+
-        return START_STICKY
+        // It's ok to be not sticky because we will explicitly start it again on the next alarm?
+        return START_NOT_STICKY
     }
 
     override fun onDestroy() {
-        Timber.i("## onDestroy() : $this")
+        Timber.i("## Sync: onDestroy() [$this] periodic:$periodic preventReschedule:$preventReschedule")
         if (!mIsSelfDestroyed) {
-            Timber.w("## Destroy by the system : $this")
+            Timber.d("## Sync: Destroy by the system : $this")
         }
-        serviceScope.coroutineContext.cancelChildren()
         isRunning.set(false)
+        // Cancelling the context will trigger the catch close the doSync try
+        serviceScope.coroutineContext.cancelChildren()
+        if (!preventReschedule && periodic && sessionId != null && backgroundDetectionObserver.isInBackground) {
+            Timber.d("## Sync: Reschedule service in $syncDelaySeconds sec")
+            onRescheduleAsked(sessionId ?: "", false, syncTimeoutSeconds, syncDelaySeconds)
+        }
         super.onDestroy()
     }
 
@@ -90,9 +131,15 @@ abstract class SyncService : Service() {
 
     private fun doSyncIfNotAlreadyRunning() {
         if (isRunning.get()) {
-            Timber.i("Received a start while was already syncing... ignore")
+            Timber.i("## Sync: Received a start while was already syncing... ignore")
         } else {
             isRunning.set(true)
+            // Acquire a lock to give enough time for the sync :/
+            getSystemService<PowerManager>()?.run {
+                newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "riotx:fdroidSynclock").apply {
+                    acquire((syncTimeoutSeconds * 1000L + 10_000L))
+                }
+            }
             serviceScope.launch(coroutineDispatchers.io) {
                 doSync()
             }
@@ -100,9 +147,10 @@ abstract class SyncService : Service() {
     }
 
     private suspend fun doSync() {
-        Timber.v("Execute sync request with timeout 0")
+        Timber.v("## Sync: Execute sync request with timeout $syncTimeoutSeconds seconds")
-        val params = SyncTask.Params(TIME_OUT)
+        val params = SyncTask.Params(syncTimeoutSeconds * 1000L)
         try {
+            // never do that in foreground, let the syncThread work
             syncTask.execute(params)
             // Start sync if we were doing an initial sync and the syncThread is not launched yet
             if (isInitialSync && session.getSyncState() == SyncState.Idle) {
@@ -111,28 +159,34 @@ abstract class SyncService : Service() {
             }
             stopMe()
         } catch (throwable: Throwable) {
-            Timber.e(throwable)
+            Timber.e(throwable, "## Sync: sync service did fail ${isRunning.get()}")
             if (throwable.isTokenError()) {
-                stopMe()
+                // no need to retry
-            } else {
+                preventReschedule = true
-                Timber.v("Should be rescheduled to avoid wasting resources")
-                sessionId?.also {
-                    onRescheduleAsked(it, isInitialSync, delay = 10_000L)
-                }
-                stopMe()
             }
+            if (throwable is Failure.NetworkConnection) {
+                // Network is off, no need to reschedule endless alarms :/
+                preventReschedule = true
+                // Instead start a work to restart background sync when network is back
+                onNetworkError(sessionId ?: "", isInitialSync, syncTimeoutSeconds, syncDelaySeconds)
+            }
+            // JobCancellation could be caught here when onDestroy cancels the coroutine context
+            if (isRunning.get()) stopMe()
         }
     }
 
     private fun initialize(intent: Intent?): Boolean {
         if (intent == null) {
+            Timber.d("## Sync: initialize intent is null")
             return false
         }
         val matrix = Matrix.getInstance(applicationContext)
         val safeSessionId = intent.getStringExtra(EXTRA_SESSION_ID) ?: return false
+        syncTimeoutSeconds = intent.getIntExtra(EXTRA_TIMEOUT_SECONDS, 6)
+        syncDelaySeconds = intent.getIntExtra(EXTRA_DELAY_SECONDS, 60)
         try {
             val sessionComponent = matrix.sessionManager.getSessionComponent(safeSessionId)
-                    ?: throw IllegalStateException("You should have a session to make it work")
+                    ?: throw IllegalStateException("## Sync: You should have a session to make it work")
             session = sessionComponent.session()
             sessionId = safeSessionId
             syncTask = sessionComponent.syncTask()
@@ -143,14 +197,16 @@ abstract class SyncService : Service() {
             backgroundDetectionObserver = matrix.backgroundDetectionObserver
             return true
         } catch (exception: Exception) {
-            Timber.e(exception, "An exception occurred during initialisation")
+            Timber.e(exception, "## Sync: An exception occurred during initialisation")
             return false
         }
     }
 
     abstract fun onStart(isInitialSync: Boolean)
 
-    abstract fun onRescheduleAsked(sessionId: String, isInitialSync: Boolean, delay: Long)
+    abstract fun onRescheduleAsked(sessionId: String, isInitialSync: Boolean, timeout: Int, delay: Int)
+
+    abstract fun onNetworkError(sessionId: String, isInitialSync: Boolean, timeout: Int, delay: Int)
 
     override fun onBind(intent: Intent?): IBinder? {
         return null
@@ -158,6 +214,11 @@ abstract class SyncService : Service() {
 
     companion object {
         const val EXTRA_SESSION_ID = "EXTRA_SESSION_ID"
-        private const val TIME_OUT = 0L
+        const val EXTRA_TIMEOUT_SECONDS = "EXTRA_TIMEOUT_SECONDS"
+        const val EXTRA_DELAY_SECONDS = "EXTRA_DELAY_SECONDS"
+        const val EXTRA_PERIODIC = "EXTRA_PERIODIC"
+        const val EXTRA_NETWORK_BACK_RESTART = "EXTRA_NETWORK_BACK_RESTART"
+
+        const val ACTION_STOP = "ACTION_STOP"
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/job/SyncWorker.kt

@@ -18,23 +18,24 @@ package org.matrix.android.sdk.internal.session.sync.job
 
 import android.content.Context
 import androidx.work.BackoffPolicy
-import androidx.work.CoroutineWorker
 import androidx.work.ExistingWorkPolicy
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
 import org.matrix.android.sdk.api.failure.isTokenError
 import org.matrix.android.sdk.internal.di.WorkManagerProvider
 import org.matrix.android.sdk.internal.network.NetworkConnectivityChecker
+import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.session.sync.SyncTask
 import org.matrix.android.sdk.internal.task.TaskExecutor
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.getSessionComponent
 import timber.log.Timber
 import java.util.concurrent.TimeUnit
 import javax.inject.Inject
 
-private const val DEFAULT_LONG_POOL_TIMEOUT = 0L
+private const val DEFAULT_LONG_POOL_TIMEOUT = 6L
+private const val DEFAULT_DELAY_TIMEOUT = 30_000L
 
 /**
  * Possible previous worker: None
@@ -42,44 +43,59 @@ private const val DEFAULT_LONG_POOL_TIMEOUT = 0L
  */
 internal class SyncWorker(context: Context,
                           workerParameters: WorkerParameters
-) : CoroutineWorker(context, workerParameters) {
+) : SessionSafeCoroutineWorker<SyncWorker.Params>(context, workerParameters, Params::class.java) {
 
     @JsonClass(generateAdapter = true)
     internal data class Params(
             override val sessionId: String,
             val timeout: Long = DEFAULT_LONG_POOL_TIMEOUT,
-            val automaticallyRetry: Boolean = false,
+            val delay: Long = DEFAULT_DELAY_TIMEOUT,
+            val periodic: Boolean = false,
             override val lastFailureMessage: String? = null
     ) : SessionWorkerParams
 
     @Inject lateinit var syncTask: SyncTask
     @Inject lateinit var taskExecutor: TaskExecutor
     @Inject lateinit var networkConnectivityChecker: NetworkConnectivityChecker
+    @Inject lateinit var workManagerProvider: WorkManagerProvider
 
-    override suspend fun doWork(): Result {
+    override fun injectWith(injector: SessionComponent) {
+        injector.inject(this)
+    }
+
+    override suspend fun doSafeWork(params: Params): Result {
         Timber.i("Sync work starting")
-        val params = WorkerParamsFactory.fromData<Params>(inputData)
-                ?: return Result.success()
-                        .also { Timber.e("Unable to parse work parameters") }
 
-        val sessionComponent = getSessionComponent(params.sessionId) ?: return Result.success()
-        sessionComponent.inject(this)
         return runCatching {
             doSync(params.timeout)
         }.fold(
-                { Result.success() },
+                {
+                    Result.success().also {
+                        if (params.periodic) {
+                            // we want to schedule another one after delay
+                            automaticallyBackgroundSync(workManagerProvider, params.sessionId, params.timeout, params.delay)
+                        }
+                    }
+                },
                 { failure ->
-                    if (failure.isTokenError() || !params.automaticallyRetry) {
+                    if (failure.isTokenError()) {
                         Result.failure()
                     } else {
+                        // If the worker was stopped (when going back in foreground), a JobCancellation exception is sent
+                        // but in this case the result is ignored, as the work is considered stopped,
+                        // so don't worry of the retry here for this case
                         Result.retry()
                     }
                 }
         )
     }
 
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
+
     private suspend fun doSync(timeout: Long) {
-        val taskParams = SyncTask.Params(timeout)
+        val taskParams = SyncTask.Params(timeout * 1000)
         syncTask.execute(taskParams)
     }
 
@@ -87,25 +103,27 @@ internal class SyncWorker(context: Context,
         private const val BG_SYNC_WORK_NAME = "BG_SYNCP"
 
         fun requireBackgroundSync(workManagerProvider: WorkManagerProvider, sessionId: String, serverTimeout: Long = 0) {
-            val data = WorkerParamsFactory.toData(Params(sessionId, serverTimeout, false))
+            val data = WorkerParamsFactory.toData(Params(sessionId, serverTimeout, 0L, false))
             val workRequest = workManagerProvider.matrixOneTimeWorkRequestBuilder<SyncWorker>()
                     .setConstraints(WorkManagerProvider.workConstraints)
                     .setBackoffCriteria(BackoffPolicy.LINEAR, 1_000, TimeUnit.MILLISECONDS)
                     .setInputData(data)
                     .build()
             workManagerProvider.workManager
-                    .enqueueUniqueWork(BG_SYNC_WORK_NAME, ExistingWorkPolicy.REPLACE, workRequest)
+                    .enqueueUniqueWork(BG_SYNC_WORK_NAME, ExistingWorkPolicy.APPEND_OR_REPLACE, workRequest)
         }
 
-        fun automaticallyBackgroundSync(workManagerProvider: WorkManagerProvider, sessionId: String, serverTimeout: Long = 0, delay: Long = 30_000) {
+        fun automaticallyBackgroundSync(workManagerProvider: WorkManagerProvider, sessionId: String, serverTimeout: Long = 0, delayInSeconds: Long = 30) {
-            val data = WorkerParamsFactory.toData(Params(sessionId, serverTimeout, true))
+            val data = WorkerParamsFactory.toData(Params(sessionId, serverTimeout, delayInSeconds, true))
             val workRequest = workManagerProvider.matrixOneTimeWorkRequestBuilder<SyncWorker>()
                     .setConstraints(WorkManagerProvider.workConstraints)
                     .setInputData(data)
-                    .setBackoffCriteria(BackoffPolicy.LINEAR, delay, TimeUnit.MILLISECONDS)
+                    .setBackoffCriteria(BackoffPolicy.LINEAR, 1_000, TimeUnit.MILLISECONDS)
+                    .setInitialDelay(delayInSeconds, TimeUnit.SECONDS)
                     .build()
+
             workManagerProvider.workManager
-                    .enqueueUniqueWork(BG_SYNC_WORK_NAME, ExistingWorkPolicy.REPLACE, workRequest)
+                    .enqueueUniqueWork(BG_SYNC_WORK_NAME, ExistingWorkPolicy.APPEND_OR_REPLACE, workRequest)
         }
 
         fun stopAnyBackgroundSync(workManagerProvider: WorkManagerProvider) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/user/UserDataSource.kt

@@ -23,9 +23,11 @@ import androidx.paging.DataSource
 import androidx.paging.LivePagedListBuilder
 import androidx.paging.PagedList
 import com.zhuinden.monarchy.Monarchy
+import io.realm.Case
 import org.matrix.android.sdk.api.session.user.model.User
 import org.matrix.android.sdk.api.util.Optional
 import org.matrix.android.sdk.api.util.toOptional
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.mapper.asDomain
 import org.matrix.android.sdk.internal.database.model.IgnoredUserEntity
 import org.matrix.android.sdk.internal.database.model.IgnoredUserEntityFields
@@ -33,11 +35,10 @@ import org.matrix.android.sdk.internal.database.model.UserEntity
 import org.matrix.android.sdk.internal.database.model.UserEntityFields
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.di.SessionDatabase
-import org.matrix.android.sdk.internal.util.fetchCopied
-import io.realm.Case
 import javax.inject.Inject
 
-internal class UserDataSource @Inject constructor(@SessionDatabase private val monarchy: Monarchy) {
+internal class UserDataSource @Inject constructor(@SessionDatabase private val monarchy: Monarchy,
+                                                  private val realmSessionProvider: RealmSessionProvider) {
 
     private val realmDataSourceFactory: Monarchy.RealmDataSourceFactory<UserEntity> by lazy {
         monarchy.createDataSourceFactory { realm ->
@@ -58,10 +59,10 @@ internal class UserDataSource @Inject constructor(@SessionDatabase private val m
     }
 
     fun getUser(userId: String): User? {
-        val userEntity = monarchy.fetchCopied { UserEntity.where(it, userId).findFirst() }
+        return realmSessionProvider.withRealm {
-                ?: return null
+            val userEntity = UserEntity.where(it, userId).findFirst()
-
+            userEntity?.asDomain()
-        return userEntity.asDomain()
+        }
     }
 
     fun getUserLive(userId: String): LiveData<Optional<User>> {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/user/accountdata/AccountDataDataSource.kt

@@ -20,18 +20,20 @@ package org.matrix.android.sdk.internal.session.user.accountdata
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.Transformations
 import com.zhuinden.monarchy.Monarchy
+import io.realm.Realm
+import io.realm.RealmQuery
+import org.matrix.android.sdk.api.session.accountdata.UserAccountDataEvent
 import org.matrix.android.sdk.api.util.Optional
 import org.matrix.android.sdk.api.util.toOptional
+import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.mapper.AccountDataMapper
 import org.matrix.android.sdk.internal.database.model.UserAccountDataEntity
 import org.matrix.android.sdk.internal.database.model.UserAccountDataEntityFields
 import org.matrix.android.sdk.internal.di.SessionDatabase
-import org.matrix.android.sdk.api.session.accountdata.UserAccountDataEvent
-import io.realm.Realm
-import io.realm.RealmQuery
 import javax.inject.Inject
 
 internal class AccountDataDataSource @Inject constructor(@SessionDatabase private val monarchy: Monarchy,
+                                                         private val realmSessionProvider: RealmSessionProvider,
                                                          private val accountDataMapper: AccountDataMapper) {
 
     fun getAccountDataEvent(type: String): UserAccountDataEvent? {
@@ -45,10 +47,9 @@ internal class AccountDataDataSource @Inject constructor(@SessionDatabase privat
     }
 
     fun getAccountDataEvents(types: Set<String>): List<UserAccountDataEvent> {
-        return monarchy.fetchAllMappedSync(
+        return realmSessionProvider.withRealm {
-                { accountDataEventsQuery(it, types) },
+            accountDataEventsQuery(it, types).findAll().map(accountDataMapper::map)
-                accountDataMapper::map
+        }
-        )
     }
 
     fun getLiveAccountDataEvents(types: Set<String>): LiveData<List<UserAccountDataEvent>> {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/widgets/WidgetManager.kt

@@ -202,6 +202,6 @@ internal class WidgetManager @Inject constructor(private val integrationManager:
                 stateKey = QueryStringValue.NoCondition
         )
         val powerLevelsContent = powerLevelsEvent?.content?.toModel<PowerLevelsContent>() ?: return false
-        return PowerLevelsHelper(powerLevelsContent).isUserAllowedToSend(userId, true, null)
+        return PowerLevelsHelper(powerLevelsContent).isUserAllowedToSend(userId, true, EventType.STATE_ROOM_WIDGET_LEGACY)
     }
 }