From dc19652c2b10cfcbf3d71fdec42de0a176f19f5a Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 May 2020 11:46:21 +0200 Subject: [PATCH 01/83] WIP refact WIP TMP WIP --- matrix-sdk-android/build.gradle | 4 + .../matrix/android/api/session/Session.kt | 6 + .../android/api/session/call/CallService.kt | 57 + .../android/api/session/call/CallsListener.kt | 47 + .../android/api/session/call/EglUtils.kt | 55 + .../api/session/call/PeerSignalingClient.kt | 66 + .../session/call/RoomConnectionParameter.kt | 40 + .../android/api/session/call/TurnServer.kt | 28 + .../android/api/session/call/VoipApi.kt | 28 + .../api/session/events/model/EventType.kt | 1 - .../room/model/call/CallInviteContent.kt | 35 +- .../android/internal/crypto/CryptoModule.kt | 5 + .../internal/crypto/tasks/SendEventTask.kt | 79 + .../internal/session/DefaultSession.kt | 6 +- .../android/internal/session/SessionModule.kt | 16 +- .../session/call/CallEventObserver.kt | 67 + .../session/call/CallEventsObserverTask.kt | 88 ++ .../session/call/DefaultCallService.kt | 206 +++ .../internal/session/room/RoomModule.kt | 6 + .../session/room/send/DefaultSendService.kt | 21 +- .../session/room/send/RoomEventSender.kt | 72 + vector/build.gradle | 3 + .../riotx/features/debug/DebugMenuActivity.kt | 4 +- vector/src/main/AndroidManifest.xml | 16 + .../java/im/vector/riotx/VectorApplication.kt | 4 + .../vector/riotx/core/di/ScreenComponent.kt | 2 + .../vector/riotx/core/di/VectorComponent.kt | 3 + .../riotx/features/call/CallConnection.kt | 42 + .../riotx/features/call/CallFragment.java | 133 ++ .../features/call/PeerConnectionClient.java | 1374 +++++++++++++++++ .../call/PeerConnectionObserverAdapter.kt | 70 + .../riotx/features/call/SdpObserverAdapter.kt | 39 + .../riotx/features/call/VectorCallActivity.kt | 434 ++++++ .../features/call/VectorCallViewModel.kt | 124 ++ .../features/call/VectorConnectionService.kt | 52 + .../call/WebRtcPeerConnectionManager.kt | 372 +++++ .../home/room/detail/RoomDetailAction.kt | 1 + .../home/room/detail/RoomDetailFragment.kt | 12 + .../home/room/detail/RoomDetailViewModel.kt | 2 + .../timeline/format/NoticeEventFormatter.kt | 2 +- vector/src/main/res/drawable/ic_phone.xml | 14 + vector/src/main/res/layout/activity_call.xml | 33 + vector/src/main/res/layout/fragment_call.xml | 63 + vector/src/main/res/menu/menu_timeline.xml | 8 + 44 files changed, 3710 insertions(+), 30 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallFragment.java create mode 100644 vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java create mode 100644 vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt create mode 100644 vector/src/main/res/drawable/ic_phone.xml create mode 100644 vector/src/main/res/layout/activity_call.xml create mode 100644 vector/src/main/res/layout/fragment_call.xml diff --git a/matrix-sdk-android/build.gradle b/matrix-sdk-android/build.gradle index e4251a733c..b44cb595d0 100644 --- a/matrix-sdk-android/build.gradle +++ b/matrix-sdk-android/build.gradle @@ -162,6 +162,10 @@ dependencies { // Phone number https://github.com/google/libphonenumber implementation 'com.googlecode.libphonenumber:libphonenumber:8.10.23' + // Web RTC + // TODO meant for development purposes only + implementation 'org.webrtc:google-webrtc:1.0.+' + debugImplementation 'com.airbnb.okreplay:okreplay:1.5.0' releaseImplementation 'com.airbnb.okreplay:noop:1.5.0' androidTestImplementation 'com.airbnb.okreplay:espresso:1.5.0' diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt index 5736e78a30..ab1588f8c5 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt @@ -24,6 +24,7 @@ import im.vector.matrix.android.api.pushrules.PushRuleService import im.vector.matrix.android.api.session.account.AccountService import im.vector.matrix.android.api.session.accountdata.AccountDataService import im.vector.matrix.android.api.session.cache.CacheService +import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.content.ContentUploadStateTracker import im.vector.matrix.android.api.session.content.ContentUrlResolver import im.vector.matrix.android.api.session.crypto.CryptoService @@ -165,6 +166,11 @@ interface Session : */ fun integrationManagerService(): IntegrationManagerService + /** + * Returns the cryptoService associated with the session + */ + fun callService(): CallService + /** * Add a listener to the session. * @param listener the listener to add. diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt new file mode 100644 index 0000000000..5e3f331148 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import im.vector.matrix.android.api.MatrixCallback +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription + +interface CallService { + + + fun getTurnServer(callback: MatrixCallback) + + fun isCallSupportedInRoom(roomId: String) : Boolean + + + /** + * Send offer SDP to the other participant. + */ + fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) + + /** + * Send answer SDP to the other participant. + */ + fun sendAnswerSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) + + /** + * Send Ice candidate to the other participant. + */ + fun sendLocalIceCandidates(callId: String, roomId: String, candidates: List) + + /** + * Send removed ICE candidates to the other participant. + */ + fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) + + + fun addCallListener(listener: CallsListener) + + fun removeCallListener(listener: CallsListener) + + +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt new file mode 100644 index 0000000000..556555369a --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent + +interface CallsListener { +// /** +// * Called when there is an incoming call within the room. +// * @param peerSignalingClient the incoming call +// */ +// fun onIncomingCall(peerSignalingClient: PeerSignalingClient) +// +// /** +// * An outgoing call is started. +// * +// * @param peerSignalingClient the outgoing call +// */ +// fun onOutgoingCall(peerSignalingClient: PeerSignalingClient) +// +// /** +// * Called when a called has been hung up +// * +// * @param peerSignalingClient the incoming call +// */ +// fun onCallHangUp(peerSignalingClient: PeerSignalingClient) + + fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) + + fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) + +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt new file mode 100644 index 0000000000..bd1d95ae6f --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package im.vector.matrix.android.api.session.call + +import org.webrtc.EglBase +import timber.log.Timber + +/** + * The root [EglBase] instance shared by the entire application for + * the sake of reducing the utilization of system resources (such as EGL + * contexts) + * by performing a runtime check. + */ +object EglUtils { + + // TODO how do we release that? + + /** + * Lazily creates and returns the one and only [EglBase] which will + * serve as the root for all contexts that are needed. + */ + @get:Synchronized var rootEglBase: EglBase? = null + get() { + if (field == null) { + val configAttributes = EglBase.CONFIG_PLAIN + try { + field = EglBase.createEgl14(configAttributes) + ?: EglBase.createEgl10(configAttributes) // Fall back to EglBase10. + } catch (ex: Throwable) { + Timber.e(ex, "Failed to create EglBase") + } + } + return field + } + private set + + val rootEglBaseContext: EglBase.Context? + get() { + val eglBase = rootEglBase + return eglBase?.eglBaseContext + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt new file mode 100644 index 0000000000..9a948adbb8 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt @@ -0,0 +1,66 @@ +///* +// * Copyright (c) 2020 New Vector Ltd +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package im.vector.matrix.android.api.session.call +// +//import im.vector.matrix.android.api.MatrixCallback +//import org.webrtc.IceCandidate +//import org.webrtc.SessionDescription +// +//interface PeerSignalingClient { +// +// val callID: String +// +// fun addListener(listener: SignalingListener) +// +// /** +// * Send offer SDP to the other participant. +// */ +// fun sendOfferSdp(sdp: SessionDescription, callback: MatrixCallback) +// +// /** +// * Send answer SDP to the other participant. +// */ +// fun sendAnswerSdp(sdp: SessionDescription, callback: MatrixCallback) +// +// /** +// * Send Ice candidate to the other participant. +// */ +// fun sendLocalIceCandidates(candidates: List) +// +// /** +// * Send removed ICE candidates to the other participant. +// */ +// fun sendLocalIceCandidateRemovals(candidates: List) +// +// +// interface SignalingListener { +// /** +// * Callback fired once remote SDP is received. +// */ +// fun onRemoteDescription(sdp: SessionDescription) +// +// /** +// * Callback fired once remote Ice candidate is received. +// */ +// fun onRemoteIceCandidate(candidate: IceCandidate) +// +// /** +// * Callback fired once remote Ice candidate removals are received. +// */ +// fun onRemoteIceCandidatesRemoved(candidates: List) +// } +//} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt new file mode 100644 index 0000000000..09769efc55 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import org.webrtc.IceCandidate +import org.webrtc.PeerConnection.IceServer +import org.webrtc.SessionDescription + +/** + * Struct holding the connection parameters of an AppRTC room. + */ +data class RoomConnectionParameters( + val callId: String, + val matrixRoomId: String +) + +/** + * Struct holding the signaling parameters of an AppRTC room. + */ +data class SignalingParameters( + val iceServers: List, + val initiator: Boolean, + val clientId: String, + val offerSdp: SessionDescription, + val iceCandidates: List +) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt new file mode 100644 index 0000000000..fddd1c0c6a --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +@JsonClass(generateAdapter = true) +data class TurnServer( + @Json(name = "username") val username: String?, + @Json(name = "password") val password: String?, + @Json(name = "uris") val uris: List?, + @Json(name = "ttl") val ttl: Int? +) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt new file mode 100644 index 0000000000..e324822617 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import im.vector.matrix.android.internal.network.NetworkConstants +import retrofit2.Call +import retrofit2.http.GET + +internal interface VoipApi { + + @GET(NetworkConstants.URI_API_PREFIX_PATH_R0 + "voip/turnServer") + fun getTurnServer(): Call + +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt index a33b9e70df..801a9bb7d0 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt @@ -58,7 +58,6 @@ object EventType { const val STATE_ROOM_ENCRYPTION = "m.room.encryption" // Call Events - const val CALL_INVITE = "m.call.invite" const val CALL_CANDIDATES = "m.call.candidates" const val CALL_ANSWER = "m.call.answer" diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt index 29305d1420..44c3cfbf0b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt @@ -21,21 +21,42 @@ import com.squareup.moshi.JsonClass @JsonClass(generateAdapter = true) data class CallInviteContent( - @Json(name = "call_id") val callId: String, - @Json(name = "version") val version: Int, - @Json(name = "lifetime") val lifetime: Int, - @Json(name = "offer") val offer: Offer + + /** + * A unique identifier for the call. + */ + @Json(name = "call_id") val callId: String?, + /** + * The session description object + */ + @Json(name = "version") val version: Int?, + /** + * The version of the VoIP specification this message adheres to. This specification is version 0. + */ + @Json(name = "lifetime") val lifetime: Int?, + /** + * The time in milliseconds that the invite is valid for. + * Once the invite age exceeds this value, clients should discard it. + * They should also no longer show the call as awaiting an answer in the UI. + */ + @Json(name = "offer") val offer: Offer? ) { @JsonClass(generateAdapter = true) data class Offer( - @Json(name = "type") val type: String, - @Json(name = "sdp") val sdp: String + /** + * The type of session description (offer, answer) + */ + @Json(name = "type") val type: String?, + /** + * The SDP text of the session description. + */ + @Json(name = "sdp") val sdp: String? ) { companion object { const val SDP_VIDEO = "m=video" } } - fun isVideo(): Boolean = offer.sdp.contains(Offer.SDP_VIDEO) + fun isVideo(): Boolean = offer?.sdp?.contains(Offer.SDP_VIDEO) == true } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt index 1efdffdb06..44092f4ae4 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt @@ -67,6 +67,7 @@ import im.vector.matrix.android.internal.crypto.tasks.DefaultEncryptEventTask import im.vector.matrix.android.internal.crypto.tasks.DefaultGetDeviceInfoTask import im.vector.matrix.android.internal.crypto.tasks.DefaultGetDevicesTask import im.vector.matrix.android.internal.crypto.tasks.DefaultInitializeCrossSigningTask +import im.vector.matrix.android.internal.crypto.tasks.DefaultSendEventTask import im.vector.matrix.android.internal.crypto.tasks.DefaultSendToDeviceTask import im.vector.matrix.android.internal.crypto.tasks.DefaultSendVerificationMessageTask import im.vector.matrix.android.internal.crypto.tasks.DefaultSetDeviceNameTask @@ -80,6 +81,7 @@ import im.vector.matrix.android.internal.crypto.tasks.EncryptEventTask import im.vector.matrix.android.internal.crypto.tasks.GetDeviceInfoTask import im.vector.matrix.android.internal.crypto.tasks.GetDevicesTask import im.vector.matrix.android.internal.crypto.tasks.InitializeCrossSigningTask +import im.vector.matrix.android.internal.crypto.tasks.SendEventTask import im.vector.matrix.android.internal.crypto.tasks.SendToDeviceTask import im.vector.matrix.android.internal.crypto.tasks.SendVerificationMessageTask import im.vector.matrix.android.internal.crypto.tasks.SetDeviceNameTask @@ -251,4 +253,7 @@ internal abstract class CryptoModule { @Binds abstract fun bindInitializeCrossSigningTask(task: DefaultInitializeCrossSigningTask): InitializeCrossSigningTask + + @Binds + abstract fun bindSendEventTask(task: DefaultSendEventTask): SendEventTask } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt new file mode 100644 index 0000000000..637db1790e --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package im.vector.matrix.android.internal.crypto.tasks + +import im.vector.matrix.android.api.session.crypto.CryptoService +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.room.send.SendState +import im.vector.matrix.android.internal.network.executeRequest +import im.vector.matrix.android.internal.session.room.RoomAPI +import im.vector.matrix.android.internal.session.room.send.LocalEchoUpdater +import im.vector.matrix.android.internal.session.room.send.SendResponse +import im.vector.matrix.android.internal.task.Task +import org.greenrobot.eventbus.EventBus +import javax.inject.Inject + +internal interface SendEventTask : Task { + data class Params( + val event: Event, + val cryptoService: CryptoService? + ) +} + +internal class DefaultSendEventTask @Inject constructor( + private val localEchoUpdater: LocalEchoUpdater, + private val encryptEventTask: DefaultEncryptEventTask, + private val roomAPI: RoomAPI, + private val eventBus: EventBus) : SendEventTask { + + override suspend fun execute(params: SendEventTask.Params): String { + val event = handleEncryption(params) + val localId = event.eventId!! + + try { + localEchoUpdater.updateSendState(localId, SendState.SENDING) + val executeRequest = executeRequest(eventBus) { + apiCall = roomAPI.send( + localId, + roomId = event.roomId ?: "", + content = event.content, + eventType = event.type + ) + } + localEchoUpdater.updateSendState(localId, SendState.SENT) + return executeRequest.eventId + } catch (e: Throwable) { + localEchoUpdater.updateSendState(localId, SendState.UNDELIVERED) + throw e + } + } + + private suspend fun handleEncryption(params: SendEventTask.Params): Event { + if (params.cryptoService?.isRoomEncrypted(params.event.roomId ?: "") == true) { + try { + return encryptEventTask.execute(EncryptEventTask.Params( + params.event.roomId ?: "", + params.event, + listOf("m.relates_to"), + params.cryptoService + )) + } catch (throwable: Throwable) { + // We said it's ok to send verification request in clear + } + } + return params.event + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt index d62eb7b505..8b3affd82f 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt @@ -28,6 +28,7 @@ import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.account.AccountService import im.vector.matrix.android.api.session.accountdata.AccountDataService import im.vector.matrix.android.api.session.cache.CacheService +import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.content.ContentUploadStateTracker import im.vector.matrix.android.api.session.content.ContentUrlResolver import im.vector.matrix.android.api.session.crypto.CryptoService @@ -110,7 +111,8 @@ internal class DefaultSession @Inject constructor( private val integrationManagerService: IntegrationManagerService, private val taskExecutor: TaskExecutor, private val widgetDependenciesHolder: WidgetDependenciesHolder, - private val shieldTrustUpdater: ShieldTrustUpdater) + private val shieldTrustUpdater: ShieldTrustUpdater, + private val callService: Lazy) : Session, RoomService by roomService.get(), RoomDirectoryService by roomDirectoryService.get(), @@ -245,6 +247,8 @@ internal class DefaultSession @Inject constructor( override fun integrationManagerService() = integrationManagerService + override fun callService(): CallService = callService.get() + override fun addListener(listener: Session.Listener) { sessionListeners.addListener(listener) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt index c7afcc1d47..c670609411 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt @@ -33,6 +33,7 @@ import im.vector.matrix.android.api.crypto.MXCryptoConfig import im.vector.matrix.android.api.session.InitialSyncProgressService import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.accountdata.AccountDataService +import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.homeserver.HomeServerCapabilitiesService import im.vector.matrix.android.api.session.securestorage.SecureStorageService import im.vector.matrix.android.api.session.securestorage.SharedSecretStorageService @@ -56,9 +57,9 @@ import im.vector.matrix.android.internal.network.NetworkCallbackStrategy import im.vector.matrix.android.internal.network.NetworkConnectivityChecker import im.vector.matrix.android.internal.network.PreferredNetworkCallbackStrategy import im.vector.matrix.android.internal.network.RetrofitFactory -import im.vector.matrix.android.internal.network.httpclient.addAccessTokenInterceptor -import im.vector.matrix.android.internal.network.token.AccessTokenProvider -import im.vector.matrix.android.internal.network.token.HomeserverAccessTokenProvider +import im.vector.matrix.android.internal.network.interceptors.CurlLoggingInterceptor +import im.vector.matrix.android.internal.session.call.CallEventObserver +import im.vector.matrix.android.internal.session.call.DefaultCallService import im.vector.matrix.android.internal.session.group.GroupSummaryUpdater import im.vector.matrix.android.internal.session.homeserver.DefaultHomeServerCapabilitiesService import im.vector.matrix.android.internal.session.room.EventRelationsAggregationUpdater @@ -245,7 +246,11 @@ internal abstract class SessionModule { @Binds @IntoSet - abstract fun bindRoomTombstoneEventLiveObserver(observer: RoomTombstoneEventLiveObserver): LiveEntityObserver + abstract fun bindCallEventObserver(callEventObserver: CallEventObserver): LiveEntityObserver + + @Binds + @IntoSet + abstract fun bindRoomTombstoneEventLiveObserver(roomTombstoneEventLiveObserver: RoomTombstoneEventLiveObserver): LiveEntityObserver @Binds @IntoSet @@ -269,4 +274,7 @@ internal abstract class SessionModule { @Binds abstract fun bindSharedSecretStorageService(service: DefaultSharedSecretStorageService): SharedSecretStorageService + + @Binds + abstract fun bindCallService(service:DefaultCallService): CallService } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt new file mode 100644 index 0000000000..61e6087737 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import com.zhuinden.monarchy.Monarchy +import im.vector.matrix.android.api.session.events.model.EventType +import im.vector.matrix.android.internal.database.RealmLiveEntityObserver +import im.vector.matrix.android.internal.database.mapper.asDomain +import im.vector.matrix.android.internal.database.model.EventEntity +import im.vector.matrix.android.internal.database.query.whereTypes +import im.vector.matrix.android.internal.di.SessionDatabase +import im.vector.matrix.android.internal.di.UserId +import im.vector.matrix.android.internal.session.room.EventRelationsAggregationTask +import io.realm.OrderedCollectionChangeSet +import io.realm.RealmConfiguration +import io.realm.RealmResults +import kotlinx.coroutines.launch +import timber.log.Timber +import javax.inject.Inject + +internal class CallEventObserver @Inject constructor( + @SessionDatabase realmConfiguration: RealmConfiguration, + @UserId private val userId: String, + private val task: CallEventsObserverTask) : + RealmLiveEntityObserver(realmConfiguration) { + + override val query = Monarchy.Query { + EventEntity.whereTypes(it, listOf( + EventType.CALL_ANSWER, + EventType.CALL_CANDIDATES, + EventType.CALL_INVITE, + EventType.CALL_HANGUP, + EventType.ENCRYPTED) + ) + } + + override fun onChange(results: RealmResults, changeSet: OrderedCollectionChangeSet) { + Timber.v("EventRelationsAggregationUpdater called with ${changeSet.insertions.size} insertions") + + val insertedDomains = changeSet.insertions + .asSequence() + .mapNotNull { results[it]?.asDomain() } + .toList() + + val params = CallEventsObserverTask.Params( + insertedDomains, + userId + ) + observerScope.launch { + task.execute(params) + } + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt new file mode 100644 index 0000000000..2aeabc09f4 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import com.zhuinden.monarchy.Monarchy +import im.vector.matrix.android.api.session.crypto.CryptoService +import im.vector.matrix.android.api.session.crypto.MXCryptoError +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.events.model.EventType +import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult +import im.vector.matrix.android.internal.task.Task +import im.vector.matrix.android.internal.util.awaitTransaction +import io.realm.Realm +import timber.log.Timber +import javax.inject.Inject + +internal interface CallEventsObserverTask : Task { + + data class Params( + val events: List, + val userId: String + ) +} + +internal class DefaultCallEventsObserverTask @Inject constructor( + private val monarchy: Monarchy, + private val cryptoService: CryptoService, + private val callService: DefaultCallService) : CallEventsObserverTask { + + override suspend fun execute(params: CallEventsObserverTask.Params) { + val events = params.events + val userId = params.userId + monarchy.awaitTransaction { realm -> + Timber.v(">>> DefaultCallEventsObserverTask[${params.hashCode()}] called with ${events.size} events") + update(realm, events, userId) + Timber.v("<<< DefaultCallEventsObserverTask[${params.hashCode()}] finished") + } + } + + private fun update(realm: Realm, events: List, userId: String) { + events.forEach { event -> + event.roomId ?: return@forEach Unit.also { + Timber.w("Event with no room id ${event.eventId}") + } + decryptIfNeeded(event) + when (event.getClearType()) { + EventType.CALL_INVITE, + EventType.CALL_CANDIDATES, + EventType.CALL_HANGUP, + EventType.CALL_ANSWER -> { + callService.onCallEvent(event) + } + } + } + Timber.v("$realm : $userId") + } + + private fun decryptIfNeeded(event: Event) { + if (event.isEncrypted() && event.mxDecryptionResult == null) { + try { + val result = cryptoService.decryptEvent(event, event.roomId ?: "") + event.mxDecryptionResult = OlmDecryptionResult( + payload = result.clearEvent, + senderKey = result.senderCurve25519Key, + keysClaimed = result.claimedEd25519Key?.let { k -> mapOf("ed25519" to k) }, + forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain + ) + } catch (e: MXCryptoError) { + Timber.v("Call service: Failed to decrypt event") + // TODO -> we should keep track of this and retry, or aggregation will be broken + } + } + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt new file mode 100644 index 0000000000..ab30a61ebf --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import im.vector.matrix.android.api.MatrixCallback +import im.vector.matrix.android.api.extensions.tryThis +import im.vector.matrix.android.api.session.call.CallService +import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.call.TurnServer +import im.vector.matrix.android.api.session.events.model.Content +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.events.model.EventType +import im.vector.matrix.android.api.session.events.model.LocalEcho +import im.vector.matrix.android.api.session.events.model.UnsignedData +import im.vector.matrix.android.api.session.events.model.toContent +import im.vector.matrix.android.api.session.events.model.toModel +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.matrix.android.internal.di.UserId +import im.vector.matrix.android.internal.session.SessionScope +import im.vector.matrix.android.internal.session.room.send.LocalEchoEventFactory +import im.vector.matrix.android.internal.session.room.send.RoomEventSender +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription +import javax.inject.Inject + +@SessionScope +internal class DefaultCallService @Inject constructor( + @UserId + private val userId: String, + private val localEchoEventFactory: LocalEchoEventFactory, + private val roomEventSender: RoomEventSender +) : CallService { + + private val callListeners = ArrayList() + + override fun getTurnServer(callback: MatrixCallback) { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + + override fun isCallSupportedInRoom(roomId: String): Boolean { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + + override fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { + val eventContent = CallInviteContent( + callId = callId, + version = 0, + lifetime = CALL_TIMEOUT_MS, + offer = CallInviteContent.Offer( + type = sdp.type.canonicalForm(), + sdp = sdp.description + ) + ) + + createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) +// sendEventTask +// .configureWith( +// SendEventTask.Params(event = event, cryptoService = cryptoService) +// ) { +// this.callback = callback +// }.executeBy(taskExecutor) + } + } + + override fun sendAnswerSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { + val eventContent = CallAnswerContent( + callId = callId, + version = 0, + answer = CallAnswerContent.Answer( + type = sdp.type.canonicalForm(), + sdp = sdp.description + ) + ) + + createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) +// sendEventTask +// .configureWith( +// SendEventTask.Params(event = event, cryptoService = cryptoService) +// ) { +// this.callback = callback +// }.executeBy(taskExecutor) + } + } + + override fun sendLocalIceCandidates(callId: String, roomId: String, candidates: List) { + val eventContent = CallCandidatesContent( + callId = callId, + version = 0, + candidates = candidates.map { + CallCandidatesContent.Candidate( + sdpMid = it.sdpMid, + sdpMLineIndex = it.sdpMLineIndex.toString(), + candidate = it.sdp + ) + } + ) + createEventAndLocalEcho(type = EventType.CALL_CANDIDATES, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) +// sendEventTask +// .configureWith( +// SendEventTask.Params(event = event, cryptoService = cryptoService) +// ) { +// this.callback = callback +// }.executeBy(taskExecutor) + } + } + + override fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) { + } + + override fun addCallListener(listener: CallsListener) { + if (!callListeners.contains(listener)) callListeners.add(listener) + } + + override fun removeCallListener(listener: CallsListener) { + callListeners.remove(listener) + } + + fun onCallEvent(event: Event) { + when (event.getClearType()) { + EventType.CALL_ANSWER -> { + event.getClearContent().toModel()?.let { + onCallAnswer(it) + } + } + EventType.CALL_INVITE -> { + event.getClearContent().toModel()?.let { + onCallInvite(event.roomId ?: "", it) + } + } + } + } + + private fun onCallAnswer(answer: CallAnswerContent) { + callListeners.forEach { + tryThis { + it.onCallAnswerReceived(answer) + } + } + } + + private fun onCallInvite(roomId: String, answer: CallInviteContent) { + callListeners.forEach { + tryThis { + it.onCallInviteReceived(roomId, answer) + } + } + } + + private fun createEventAndLocalEcho(localId: String = LocalEcho.createLocalEchoId(), type: String, roomId: String, content: Content): Event { + return Event( + roomId = roomId, + originServerTs = System.currentTimeMillis(), + senderId = userId, + eventId = localId, + type = type, + content = content, + unsignedData = UnsignedData(age = null, transactionId = localId) + ).also { + localEchoEventFactory.createLocalEcho(it) + } + } + + companion object { + const val CALL_TIMEOUT_MS = 120_000 + } + +// internal class PeerSignalingClientFactory @Inject constructor( +// @UserId private val userId: String, +// private val localEchoEventFactory: LocalEchoEventFactory, +// private val sendEventTask: SendEventTask, +// private val taskExecutor: TaskExecutor, +// private val cryptoService: CryptoService +// ) { +// +// fun create(roomId: String, callId: String): PeerSignalingClient { +// return RoomPeerSignalingClient( +// callID = callId, +// roomId = roomId, +// userId = userId, +// localEchoEventFactory = localEchoEventFactory, +// sendEventTask = sendEventTask, +// taskExecutor = taskExecutor, +// cryptoService = cryptoService +// ) +// } +// } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt index 0572a37506..e2580831e6 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt @@ -24,6 +24,8 @@ import im.vector.matrix.android.api.session.room.RoomDirectoryService import im.vector.matrix.android.api.session.room.RoomService import im.vector.matrix.android.internal.session.DefaultFileService import im.vector.matrix.android.internal.session.SessionScope +import im.vector.matrix.android.internal.session.call.CallEventsObserverTask +import im.vector.matrix.android.internal.session.call.DefaultCallEventsObserverTask import im.vector.matrix.android.internal.session.room.alias.DefaultGetRoomIdByAliasTask import im.vector.matrix.android.internal.session.room.alias.GetRoomIdByAliasTask import im.vector.matrix.android.internal.session.room.create.CreateRoomTask @@ -201,4 +203,8 @@ internal abstract class RoomModule { @Binds abstract fun bindDeleteTagFromRoomTask(task: DefaultDeleteTagFromRoomTask): DeleteTagFromRoomTask + + // TODO is this in the correct module? + @Binds + abstract fun bindCallEventsObserverTask(task: DefaultCallEventsObserverTask): CallEventsObserverTask } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt index d60e652e12..b4593bc71b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt @@ -58,7 +58,8 @@ internal class DefaultSendService @AssistedInject constructor( private val localEchoEventFactory: LocalEchoEventFactory, private val cryptoService: CryptoService, private val taskExecutor: TaskExecutor, - private val localEchoRepository: LocalEchoRepository + private val localEchoRepository: LocalEchoRepository, + private val roomEventSender: RoomEventSender ) : SendService { @AssistedInject.Factory @@ -111,20 +112,6 @@ internal class DefaultSendService @AssistedInject constructor( .let { sendEvent(it) } } - private fun sendEvent(event: Event): Cancelable { - // Encrypted room handling - return if (cryptoService.isRoomEncrypted(roomId)) { - Timber.v("Send event in encrypted room") - val encryptWork = createEncryptEventWork(event, true) - // Note that event will be replaced by the result of the previous work - val sendWork = createSendEventWork(event, false) - timelineSendEventWorkCommon.postSequentialWorks(roomId, encryptWork, sendWork) - } else { - val sendWork = createSendEventWork(event, true) - timelineSendEventWorkCommon.postWork(roomId, sendWork) - } - } - override fun sendMedias(attachments: List, compressBeforeSending: Boolean, roomIds: Set): Cancelable { @@ -269,6 +256,10 @@ internal class DefaultSendService @AssistedInject constructor( return cancelableBag } + private fun sendEvent(event: Event): Cancelable { + return roomEventSender.sendEvent(event) + } + private fun createLocalEcho(event: Event) { localEchoEventFactory.createLocalEcho(event) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt new file mode 100644 index 0000000000..4d43067ceb --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.room.send + +import androidx.work.BackoffPolicy +import androidx.work.OneTimeWorkRequest +import im.vector.matrix.android.api.session.crypto.CryptoService +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.util.Cancelable +import im.vector.matrix.android.internal.di.SessionId +import im.vector.matrix.android.internal.di.WorkManagerProvider +import im.vector.matrix.android.internal.session.room.timeline.TimelineSendEventWorkCommon +import im.vector.matrix.android.internal.worker.WorkerParamsFactory +import im.vector.matrix.android.internal.worker.startChain +import timber.log.Timber +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +internal class RoomEventSender @Inject constructor( + private val workManagerProvider: WorkManagerProvider, + private val timelineSendEventWorkCommon: TimelineSendEventWorkCommon, + @SessionId private val sessionId: String, + private val cryptoService: CryptoService +) { + fun sendEvent(event: Event): Cancelable { + // Encrypted room handling + return if (cryptoService.isRoomEncrypted(event.roomId ?: "")) { + Timber.v("Send event in encrypted room") + val encryptWork = createEncryptEventWork(event, true) + // Note that event will be replaced by the result of the previous work + val sendWork = createSendEventWork(event, false) + timelineSendEventWorkCommon.postSequentialWorks(event.roomId ?: "", encryptWork, sendWork) + } else { + val sendWork = createSendEventWork(event, true) + timelineSendEventWorkCommon.postWork(event.roomId ?: "", sendWork) + } + } + + private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest { + // Same parameter + val params = EncryptEventWorker.Params(sessionId, event) + val sendWorkData = WorkerParamsFactory.toData(params) + + return workManagerProvider.matrixOneTimeWorkRequestBuilder() + .setConstraints(WorkManagerProvider.workConstraints) + .setInputData(sendWorkData) + .startChain(startChain) + .setBackoffCriteria(BackoffPolicy.LINEAR, WorkManagerProvider.BACKOFF_DELAY, TimeUnit.MILLISECONDS) + .build() + } + + private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest { + val sendContentWorkerParams = SendEventWorker.Params(sessionId, event) + val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams) + + return timelineSendEventWorkCommon.createWork(sendWorkData, startChain) + } +} diff --git a/vector/build.gradle b/vector/build.gradle index 5830b38559..f253501177 100644 --- a/vector/build.gradle +++ b/vector/build.gradle @@ -390,6 +390,9 @@ dependencies { implementation 'com.github.BillCarsonFr:JsonViewer:0.5' + // TODO meant for development purposes only + implementation 'org.webrtc:google-webrtc:1.0.+' + // QR-code // Stick to 3.3.3 because of https://github.com/zxing/zxing/issues/1170 implementation 'com.google.zxing:core:3.3.3' diff --git a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt index a197a6f93e..16db7b0c38 100644 --- a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt +++ b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt @@ -35,6 +35,7 @@ import im.vector.riotx.core.utils.PERMISSION_REQUEST_CODE_LAUNCH_CAMERA import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions import im.vector.riotx.core.utils.toast +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.debug.sas.DebugSasEmojiActivity import im.vector.riotx.features.qrcode.QrCodeScannerActivity import kotlinx.android.synthetic.debug.activity_debug_menu.* @@ -183,7 +184,8 @@ class DebugMenuActivity : VectorBaseActivity() { @OnClick(R.id.debug_scan_qr_code) fun scanQRCode() { if (checkPermissions(PERMISSIONS_FOR_TAKING_PHOTO, this, PERMISSION_REQUEST_CODE_LAUNCH_CAMERA)) { - doScanQRCode() + //doScanQRCode() + startActivity(VectorCallActivity.newIntent(this, "!cyIJhOLwWgmmqreHLD:matrix.org")) } } diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 833267483a..1cd522337e 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -8,8 +8,16 @@ + + + + + + + + @@ -186,6 +195,13 @@ android:name=".core.services.VectorSyncService" android:exported="false" /> + + + + + + diff --git a/vector/src/main/java/im/vector/riotx/VectorApplication.kt b/vector/src/main/java/im/vector/riotx/VectorApplication.kt index 2bceb38b75..a8e597dadb 100644 --- a/vector/src/main/java/im/vector/riotx/VectorApplication.kt +++ b/vector/src/main/java/im/vector/riotx/VectorApplication.kt @@ -43,6 +43,7 @@ import im.vector.riotx.core.di.HasVectorInjector import im.vector.riotx.core.di.VectorComponent import im.vector.riotx.core.extensions.configureAndStart import im.vector.riotx.core.rx.RxConfig +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.configuration.VectorConfiguration import im.vector.riotx.features.lifecycle.VectorActivityLifecycleCallbacks import im.vector.riotx.features.notifications.NotificationDrawerManager @@ -80,6 +81,8 @@ class VectorApplication : Application(), HasVectorInjector, MatrixConfiguration. @Inject lateinit var appStateHandler: AppStateHandler @Inject lateinit var rxConfig: RxConfig @Inject lateinit var popupAlertManager: PopupAlertManager + @Inject lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager + lateinit var vectorComponent: VectorComponent private var fontThreadHandler: Handler? = null @@ -122,6 +125,7 @@ class VectorApplication : Application(), HasVectorInjector, MatrixConfiguration. val lastAuthenticatedSession = authenticationService.getLastAuthenticatedSession()!! activeSessionHolder.setActiveSession(lastAuthenticatedSession) lastAuthenticatedSession.configureAndStart(applicationContext, pushRuleTriggerListener, sessionListener) + lastAuthenticatedSession.callService().addCallListener(webRtcPeerConnectionManager) } ProcessLifecycleOwner.get().lifecycle.addObserver(object : LifecycleObserver { @OnLifecycleEvent(Lifecycle.Event.ON_RESUME) diff --git a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt index f8ed0b01c4..ce00162e5c 100644 --- a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt +++ b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt @@ -24,6 +24,7 @@ import dagger.Component import im.vector.riotx.core.error.ErrorFormatter import im.vector.riotx.core.preference.UserAvatarPreference import im.vector.riotx.features.MainActivity +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.createdirect.CreateDirectRoomActivity import im.vector.riotx.features.crypto.keysbackup.settings.KeysBackupManageActivity import im.vector.riotx.features.crypto.quads.SharedSecureStorageActivity @@ -130,6 +131,7 @@ interface ScreenComponent { fun inject(activity: InviteUsersToRoomActivity) fun inject(activity: ReviewTermsActivity) fun inject(activity: WidgetActivity) + fun inject(activity: VectorCallActivity) /* ========================================================================================== * BottomSheets diff --git a/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt b/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt index 5c5052cf2b..14f3019666 100644 --- a/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt +++ b/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt @@ -31,6 +31,7 @@ import im.vector.riotx.core.error.ErrorFormatter import im.vector.riotx.core.pushers.PushersManager import im.vector.riotx.core.utils.AssetReader import im.vector.riotx.core.utils.DimensionConverter +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.configuration.VectorConfiguration import im.vector.riotx.features.crypto.keysrequest.KeyRequestHandler import im.vector.riotx.features.crypto.verification.IncomingVerificationRequestHandler @@ -134,6 +135,8 @@ interface VectorComponent { fun reAuthHelper(): ReAuthHelper + fun webRtcPeerConnectionManager(): WebRtcPeerConnectionManager + @Component.Factory interface Factory { fun create(@BindsInstance context: Context): VectorComponent diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt new file mode 100644 index 0000000000..5f1cd81383 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.content.Context +import android.os.Build +import android.telecom.Connection +import androidx.annotation.RequiresApi + +@RequiresApi(Build.VERSION_CODES.M) class CallConnection( + private val context: Context, + private val roomId: String, + private val callId: String +) : Connection() { + + /** + * The telecom subsystem calls this method when you add a new incoming call and your app should show its incoming call UI. + */ + override fun onShowIncomingCallUi() { + VectorCallActivity.newIntent(context, roomId).let { + context.startActivity(it) + } + } + + override fun onAnswer() { + super.onAnswer() + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java b/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java new file mode 100644 index 0000000000..750c7b6416 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java @@ -0,0 +1,133 @@ +///* +// * Copyright (c) 2020 New Vector Ltd +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package im.vector.riotx.features.call; +// +//import android.os.Bundle; +//import android.view.LayoutInflater; +//import android.view.View; +//import android.view.ViewGroup; +//import android.widget.ImageButton; +//import android.widget.SeekBar; +//import android.widget.TextView; +//import org.webrtc.RendererCommon.ScalingType; +// +//import androidx.fragment.app.Fragment; +// +//import im.vector.riotx.R; +// +///** +// * Fragment for call control. +// */ +//public class CallFragment extends Fragment { +// private TextView contactView; +// private ImageButton cameraSwitchButton; +// private ImageButton videoScalingButton; +// private ImageButton toggleMuteButton; +// private TextView captureFormatText; +// private SeekBar captureFormatSlider; +// private OnCallEvents callEvents; +// private ScalingType scalingType; +// private boolean videoCallEnabled = true; +// /** +// * Call control interface for container activity. +// */ +// public interface OnCallEvents { +// void onCallHangUp(); +// void onCameraSwitch(); +// void onVideoScalingSwitch(ScalingType scalingType); +// void onCaptureFormatChange(int width, int height, int framerate); +// boolean onToggleMic(); +// } +// @Override +// public View onCreateView( +// LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { +// View controlView = inflater.inflate(R.layout.fragment_call, container, false); +// // Create UI controls. +// contactView = controlView.findViewById(R.id.contact_name_call); +// ImageButton disconnectButton = controlView.findViewById(R.id.button_call_disconnect); +// cameraSwitchButton = controlView.findViewById(R.id.button_call_switch_camera); +// videoScalingButton = controlView.findViewById(R.id.button_call_scaling_mode); +// toggleMuteButton = controlView.findViewById(R.id.button_call_toggle_mic); +// captureFormatText = controlView.findViewById(R.id.capture_format_text_call); +// captureFormatSlider = controlView.findViewById(R.id.capture_format_slider_call); +// // Add buttons click events. +// disconnectButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// callEvents.onCallHangUp(); +// } +// }); +// cameraSwitchButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// callEvents.onCameraSwitch(); +// } +// }); +// videoScalingButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// if (scalingType == ScalingType.SCALE_ASPECT_FILL) { +// videoScalingButton.setBackgroundResource(R.drawable.ic_action_full_screen); +// scalingType = ScalingType.SCALE_ASPECT_FIT; +// } else { +// videoScalingButton.setBackgroundResource(R.drawable.ic_action_return_from_full_screen); +// scalingType = ScalingType.SCALE_ASPECT_FILL; +// } +// callEvents.onVideoScalingSwitch(scalingType); +// } +// }); +// scalingType = ScalingType.SCALE_ASPECT_FILL; +// toggleMuteButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// boolean enabled = callEvents.onToggleMic(); +// toggleMuteButton.setAlpha(enabled ? 1.0f : 0.3f); +// } +// }); +// return controlView; +// } +// @Override +// public void onStart() { +// super.onStart(); +// boolean captureSliderEnabled = false; +// Bundle args = getArguments(); +// if (args != null) { +// String contactName = args.getString(CallActivity.EXTRA_ROOMID); +// contactView.setText(contactName); +// videoCallEnabled = args.getBoolean(CallActivity.EXTRA_VIDEO_CALL, true); +// captureSliderEnabled = videoCallEnabled +// && args.getBoolean(CallActivity.EXTRA_VIDEO_CAPTUREQUALITYSLIDER_ENABLED, false); +// } +// if (!videoCallEnabled) { +// cameraSwitchButton.setVisibility(View.INVISIBLE); +// } +// if (captureSliderEnabled) { +// captureFormatSlider.setOnSeekBarChangeListener( +// new CaptureQualityController(captureFormatText, callEvents)); +// } else { +// captureFormatText.setVisibility(View.GONE); +// captureFormatSlider.setVisibility(View.GONE); +// } +// } +// // TODO(sakal): Replace with onAttach(Context) once we only support API level 23+. +// @SuppressWarnings("deprecation") +// @Override +// public void onAttach(Activity activity) { +// super.onAttach(activity); +// callEvents = (OnCallEvents) activity; +// } +//} diff --git a/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java new file mode 100644 index 0000000000..ee2594fd1a --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java @@ -0,0 +1,1374 @@ +///* +// * Copyright (c) 2020 New Vector Ltd +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package im.vector.riotx.features.call; +// +//import android.content.Context; +//import android.os.Environment; +//import android.os.ParcelFileDescriptor; +//import android.util.Log; +// +//import org.appspot.apprtc.AppRTCClient.SignalingParameters; +//import org.webrtc.AudioSource; +//import org.webrtc.AudioTrack; +//import org.webrtc.CameraVideoCapturer; +//import org.webrtc.DataChannel; +//import org.webrtc.EglBase; +//import org.webrtc.IceCandidate; +//import org.webrtc.Logging; +//import org.webrtc.MediaConstraints; +//import org.webrtc.MediaStream; +//import org.webrtc.PeerConnection; +//import org.webrtc.PeerConnection.IceConnectionState; +//import org.webrtc.PeerConnectionFactory; +//import org.webrtc.RtpParameters; +//import org.webrtc.RtpReceiver; +//import org.webrtc.RtpSender; +//import org.webrtc.SdpObserver; +//import org.webrtc.SessionDescription; +//import org.webrtc.StatsObserver; +//import org.webrtc.StatsReport; +//import org.webrtc.VideoCapturer; +//import org.webrtc.VideoRenderer; +//import org.webrtc.VideoSink; +//import org.webrtc.VideoSource; +//import org.webrtc.VideoTrack; +//import org.webrtc.voiceengine.WebRtcAudioManager; +//import org.webrtc.voiceengine.WebRtcAudioRecord; +//import org.webrtc.voiceengine.WebRtcAudioRecord.AudioRecordStartErrorCode; +//import org.webrtc.voiceengine.WebRtcAudioRecord.WebRtcAudioRecordErrorCallback; +//import org.webrtc.voiceengine.WebRtcAudioTrack; +//import org.webrtc.voiceengine.WebRtcAudioTrack.WebRtcAudioTrackErrorCallback; +//import org.webrtc.voiceengine.WebRtcAudioUtils; +// +//import java.io.File; +//import java.io.IOException; +//import java.nio.ByteBuffer; +//import java.util.ArrayList; +//import java.util.Arrays; +//import java.util.Collections; +//import java.util.EnumSet; +//import java.util.Iterator; +//import java.util.LinkedList; +//import java.util.List; +//import java.util.Timer; +//import java.util.TimerTask; +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +//import java.util.regex.Matcher; +//import java.util.regex.Pattern; +// +///** +// * Peer connection client implementation. +// * +// *

All public methods are routed to local looper thread. +// * All PeerConnectionEvents callbacks are invoked from the same looper thread. +// * This class is a singleton. +// */ +//public class PeerConnectionClient { +// public static final String VIDEO_TRACK_ID = "ARDAMSv0"; +// public static final String AUDIO_TRACK_ID = "ARDAMSa0"; +// public static final String VIDEO_TRACK_TYPE = "video"; +// private static final String TAG = "PCRTCClient"; +// private static final String VIDEO_CODEC_VP8 = "VP8"; +// private static final String VIDEO_CODEC_VP9 = "VP9"; +// private static final String VIDEO_CODEC_H264 = "H264"; +// private static final String VIDEO_CODEC_H264_BASELINE = "H264 Baseline"; +// private static final String VIDEO_CODEC_H264_HIGH = "H264 High"; +// private static final String AUDIO_CODEC_OPUS = "opus"; +// private static final String AUDIO_CODEC_ISAC = "ISAC"; +// private static final String VIDEO_CODEC_PARAM_START_BITRATE = "x-google-start-bitrate"; +// private static final String VIDEO_FLEXFEC_FIELDTRIAL = +// "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/"; +// private static final String VIDEO_VP8_INTEL_HW_ENCODER_FIELDTRIAL = "WebRTC-IntelVP8/Enabled/"; +// private static final String VIDEO_H264_HIGH_PROFILE_FIELDTRIAL = +// "WebRTC-H264HighProfile/Enabled/"; +// private static final String DISABLE_WEBRTC_AGC_FIELDTRIAL = +// "WebRTC-Audio-MinimizeResamplingOnMobile/Enabled/"; +// private static final String AUDIO_CODEC_PARAM_BITRATE = "maxaveragebitrate"; +// private static final String AUDIO_ECHO_CANCELLATION_CONSTRAINT = "googEchoCancellation"; +// private static final String AUDIO_AUTO_GAIN_CONTROL_CONSTRAINT = "googAutoGainControl"; +// private static final String AUDIO_HIGH_PASS_FILTER_CONSTRAINT = "googHighpassFilter"; +// private static final String AUDIO_NOISE_SUPPRESSION_CONSTRAINT = "googNoiseSuppression"; +// private static final String AUDIO_LEVEL_CONTROL_CONSTRAINT = "levelControl"; +// private static final String DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT = "DtlsSrtpKeyAgreement"; +// private static final int HD_VIDEO_WIDTH = 1280; +// private static final int HD_VIDEO_HEIGHT = 720; +// private static final int BPS_IN_KBPS = 1000; +// +// private static final PeerConnectionClient instance = new PeerConnectionClient(); +// private final PCObserver pcObserver = new PCObserver(); +// private final SDPObserver sdpObserver = new SDPObserver(); +// private final ExecutorService executor; +// +// private PeerConnectionFactory factory; +// private PeerConnection peerConnection; +// PeerConnectionFactory.Options options = null; +// private AudioSource audioSource; +// private VideoSource videoSource; +// private boolean videoCallEnabled; +// private boolean preferIsac; +// private String preferredVideoCodec; +// private boolean videoCapturerStopped; +// private boolean isError; +// private Timer statsTimer; +// private VideoSink.Callbacks localRender; +// private List remoteRenders; +// private SignalingParameters signalingParameters; +// private MediaConstraints pcConstraints; +// private int videoWidth; +// private int videoHeight; +// private int videoFps; +// private MediaConstraints audioConstraints; +// private ParcelFileDescriptor aecDumpFileDescriptor; +// private MediaConstraints sdpMediaConstraints; +// private PeerConnectionParameters peerConnectionParameters; +// // Queued remote ICE candidates are consumed only after both local and +// // remote descriptions are set. Similarly local ICE candidates are sent to +// // remote peer after both local and remote description are set. +// private LinkedList queuedRemoteCandidates; +// private PeerConnectionEvents events; +// private boolean isInitiator; +// private SessionDescription localSdp; // either offer or answer SDP +// private MediaStream mediaStream; +// private VideoCapturer videoCapturer; +// // enableVideo is set to true if video should be rendered and sent. +// private boolean renderVideo; +// private VideoTrack localVideoTrack; +// private VideoTrack remoteVideoTrack; +// private RtpSender localVideoSender; +// // enableAudio is set to true if audio should be sent. +// private boolean enableAudio; +// private AudioTrack localAudioTrack; +// private DataChannel dataChannel; +// private boolean dataChannelEnabled; +// +// /** +// * Peer connection parameters. +// */ +// public static class DataChannelParameters { +// public final boolean ordered; +// public final int maxRetransmitTimeMs; +// public final int maxRetransmits; +// public final String protocol; +// public final boolean negotiated; +// public final int id; +// +// public DataChannelParameters(boolean ordered, int maxRetransmitTimeMs, int maxRetransmits, +// String protocol, boolean negotiated, int id) { +// this.ordered = ordered; +// this.maxRetransmitTimeMs = maxRetransmitTimeMs; +// this.maxRetransmits = maxRetransmits; +// this.protocol = protocol; +// this.negotiated = negotiated; +// this.id = id; +// } +// } +// +// /** +// * Peer connection parameters. +// */ +// public static class PeerConnectionParameters { +// public final boolean videoCallEnabled; +// public final boolean loopback; +// public final boolean tracing; +// public final int videoWidth; +// public final int videoHeight; +// public final int videoFps; +// public final int videoMaxBitrate; +// public final String videoCodec; +// public final boolean videoCodecHwAcceleration; +// public final boolean videoFlexfecEnabled; +// public final int audioStartBitrate; +// public final String audioCodec; +// public final boolean noAudioProcessing; +// public final boolean aecDump; +// public final boolean useOpenSLES; +// public final boolean disableBuiltInAEC; +// public final boolean disableBuiltInAGC; +// public final boolean disableBuiltInNS; +// public final boolean enableLevelControl; +// public final boolean disableWebRtcAGCAndHPF; +// private final DataChannelParameters dataChannelParameters; +// +// public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing, +// int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec, +// boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate, +// String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES, +// boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS, +// boolean enableLevelControl, boolean disableWebRtcAGCAndHPF) { +// this(videoCallEnabled, loopback, tracing, videoWidth, videoHeight, videoFps, videoMaxBitrate, +// videoCodec, videoCodecHwAcceleration, videoFlexfecEnabled, audioStartBitrate, audioCodec, +// noAudioProcessing, aecDump, useOpenSLES, disableBuiltInAEC, disableBuiltInAGC, +// disableBuiltInNS, enableLevelControl, disableWebRtcAGCAndHPF, null); +// } +// +// public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing, +// int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec, +// boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate, +// String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES, +// boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS, +// boolean enableLevelControl, boolean disableWebRtcAGCAndHPF, +// DataChannelParameters dataChannelParameters) { +// this.videoCallEnabled = videoCallEnabled; +// this.loopback = loopback; +// this.tracing = tracing; +// this.videoWidth = videoWidth; +// this.videoHeight = videoHeight; +// this.videoFps = videoFps; +// this.videoMaxBitrate = videoMaxBitrate; +// this.videoCodec = videoCodec; +// this.videoFlexfecEnabled = videoFlexfecEnabled; +// this.videoCodecHwAcceleration = videoCodecHwAcceleration; +// this.audioStartBitrate = audioStartBitrate; +// this.audioCodec = audioCodec; +// this.noAudioProcessing = noAudioProcessing; +// this.aecDump = aecDump; +// this.useOpenSLES = useOpenSLES; +// this.disableBuiltInAEC = disableBuiltInAEC; +// this.disableBuiltInAGC = disableBuiltInAGC; +// this.disableBuiltInNS = disableBuiltInNS; +// this.enableLevelControl = enableLevelControl; +// this.disableWebRtcAGCAndHPF = disableWebRtcAGCAndHPF; +// this.dataChannelParameters = dataChannelParameters; +// } +// } +// +// /** +// * Peer connection events. +// */ +// public interface PeerConnectionEvents { +// /** +// * Callback fired once local SDP is created and set. +// */ +// void onLocalDescription(final SessionDescription sdp); +// +// /** +// * Callback fired once local Ice candidate is generated. +// */ +// void onIceCandidate(final IceCandidate candidate); +// +// /** +// * Callback fired once local ICE candidates are removed. +// */ +// void onIceCandidatesRemoved(final IceCandidate[] candidates); +// +// /** +// * Callback fired once connection is established (IceConnectionState is +// * CONNECTED). +// */ +// void onIceConnected(); +// +// /** +// * Callback fired once connection is closed (IceConnectionState is +// * DISCONNECTED). +// */ +// void onIceDisconnected(); +// +// /** +// * Callback fired once peer connection is closed. +// */ +// void onPeerConnectionClosed(); +// +// /** +// * Callback fired once peer connection statistics is ready. +// */ +// void onPeerConnectionStatsReady(final StatsReport[] reports); +// +// /** +// * Callback fired once peer connection error happened. +// */ +// void onPeerConnectionError(final String description); +// } +// +// private PeerConnectionClient() { +// // Executor thread is started once in private ctor and is used for all +// // peer connection API calls to ensure new peer connection factory is +// // created on the same thread as previously destroyed factory. +// executor = Executors.newSingleThreadExecutor(); +// } +// +// public static PeerConnectionClient getInstance() { +// return instance; +// } +// +// public void setPeerConnectionFactoryOptions(PeerConnectionFactory.Options options) { +// this.options = options; +// } +// +// public void createPeerConnectionFactory(final Context context, +// final PeerConnectionParameters peerConnectionParameters, final PeerConnectionEvents events) { +// this.peerConnectionParameters = peerConnectionParameters; +// this.events = events; +// videoCallEnabled = peerConnectionParameters.videoCallEnabled; +// dataChannelEnabled = peerConnectionParameters.dataChannelParameters != null; +// // Reset variables to initial states. +// factory = null; +// peerConnection = null; +// preferIsac = false; +// videoCapturerStopped = false; +// isError = false; +// queuedRemoteCandidates = null; +// localSdp = null; // either offer or answer SDP +// mediaStream = null; +// videoCapturer = null; +// renderVideo = true; +// localVideoTrack = null; +// remoteVideoTrack = null; +// localVideoSender = null; +// enableAudio = true; +// localAudioTrack = null; +// statsTimer = new Timer(); +// +// executor.execute(new Runnable() { +// @Override +// public void run() { +// createPeerConnectionFactoryInternal(context); +// } +// }); +// } +// +// public void createPeerConnection(final EglBase.Context renderEGLContext, +// final VideoRenderer.Callbacks localRender, final VideoRenderer.Callbacks remoteRender, +// final VideoCapturer videoCapturer, final SignalingParameters signalingParameters) { +// createPeerConnection(renderEGLContext, localRender, Collections.singletonList(remoteRender), +// videoCapturer, signalingParameters); +// } +// +// public void createPeerConnection(final EglBase.Context renderEGLContext, +// final VideoRenderer.Callbacks localRender, final List remoteRenders, +// final VideoCapturer videoCapturer, final SignalingParameters signalingParameters) { +// if (peerConnectionParameters == null) { +// Log.e(TAG, "Creating peer connection without initializing factory."); +// return; +// } +// this.localRender = localRender; +// this.remoteRenders = remoteRenders; +// this.videoCapturer = videoCapturer; +// this.signalingParameters = signalingParameters; +// executor.execute(new Runnable() { +// @Override +// public void run() { +// try { +// createMediaConstraintsInternal(); +// createPeerConnectionInternal(renderEGLContext); +// } catch (Exception e) { +// reportError("Failed to create peer connection: " + e.getMessage()); +// throw e; +// } +// } +// }); +// } +// +// public void close() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// closeInternal(); +// } +// }); +// } +// +// public boolean isVideoCallEnabled() { +// return videoCallEnabled; +// } +// +// private void createPeerConnectionFactoryInternal(Context context) { +// PeerConnectionFactory.initializeInternalTracer(); +// if (peerConnectionParameters.tracing) { +// PeerConnectionFactory.startInternalTracingCapture( +// Environment.getExternalStorageDirectory().getAbsolutePath() + File.separator +// + "webrtc-trace.txt"); +// } +// Log.d(TAG, +// "Create peer connection factory. Use video: " + peerConnectionParameters.videoCallEnabled); +// isError = false; +// +// // Initialize field trials. +// String fieldTrials = ""; +// if (peerConnectionParameters.videoFlexfecEnabled) { +// fieldTrials += VIDEO_FLEXFEC_FIELDTRIAL; +// Log.d(TAG, "Enable FlexFEC field trial."); +// } +// fieldTrials += VIDEO_VP8_INTEL_HW_ENCODER_FIELDTRIAL; +// if (peerConnectionParameters.disableWebRtcAGCAndHPF) { +// fieldTrials += DISABLE_WEBRTC_AGC_FIELDTRIAL; +// Log.d(TAG, "Disable WebRTC AGC field trial."); +// } +// +// // Check preferred video codec. +// preferredVideoCodec = VIDEO_CODEC_VP8; +// if (videoCallEnabled && peerConnectionParameters.videoCodec != null) { +// switch (peerConnectionParameters.videoCodec) { +// case VIDEO_CODEC_VP8: +// preferredVideoCodec = VIDEO_CODEC_VP8; +// break; +// case VIDEO_CODEC_VP9: +// preferredVideoCodec = VIDEO_CODEC_VP9; +// break; +// case VIDEO_CODEC_H264_BASELINE: +// preferredVideoCodec = VIDEO_CODEC_H264; +// break; +// case VIDEO_CODEC_H264_HIGH: +// // TODO(magjed): Strip High from SDP when selecting Baseline instead of using field trial. +// fieldTrials += VIDEO_H264_HIGH_PROFILE_FIELDTRIAL; +// preferredVideoCodec = VIDEO_CODEC_H264; +// break; +// default: +// preferredVideoCodec = VIDEO_CODEC_VP8; +// } +// } +// Log.d(TAG, "Preferred video codec: " + preferredVideoCodec); +// PeerConnectionFactory.initializeFieldTrials(fieldTrials); +// Log.d(TAG, "Field trials: " + fieldTrials); +// +// // Check if ISAC is used by default. +// preferIsac = peerConnectionParameters.audioCodec != null +// && peerConnectionParameters.audioCodec.equals(AUDIO_CODEC_ISAC); +// +// // Enable/disable OpenSL ES playback. +// if (!peerConnectionParameters.useOpenSLES) { +// Log.d(TAG, "Disable OpenSL ES audio even if device supports it"); +// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true /* enable */); +// } else { +// Log.d(TAG, "Allow OpenSL ES audio if device supports it"); +// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(false); +// } +// +// if (peerConnectionParameters.disableBuiltInAEC) { +// Log.d(TAG, "Disable built-in AEC even if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true); +// } else { +// Log.d(TAG, "Enable built-in AEC if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(false); +// } +// +// if (peerConnectionParameters.disableBuiltInAGC) { +// Log.d(TAG, "Disable built-in AGC even if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(true); +// } else { +// Log.d(TAG, "Enable built-in AGC if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(false); +// } +// +// if (peerConnectionParameters.disableBuiltInNS) { +// Log.d(TAG, "Disable built-in NS even if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true); +// } else { +// Log.d(TAG, "Enable built-in NS if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(false); +// } +// +// // Set audio record error callbacks. +// WebRtcAudioRecord.setErrorCallback(new WebRtcAudioRecordErrorCallback() { +// @Override +// public void onWebRtcAudioRecordInitError(String errorMessage) { +// Log.e(TAG, "onWebRtcAudioRecordInitError: " + errorMessage); +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioRecordStartError( +// AudioRecordStartErrorCode errorCode, String errorMessage) { +// Log.e(TAG, "onWebRtcAudioRecordStartError: " + errorCode + ". " + errorMessage); +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioRecordError(String errorMessage) { +// Log.e(TAG, "onWebRtcAudioRecordError: " + errorMessage); +// reportError(errorMessage); +// } +// }); +// +// WebRtcAudioTrack.setErrorCallback(new WebRtcAudioTrackErrorCallback() { +// @Override +// public void onWebRtcAudioTrackInitError(String errorMessage) { +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioTrackStartError(String errorMessage) { +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioTrackError(String errorMessage) { +// reportError(errorMessage); +// } +// }); +// +// // Create peer connection factory. +// PeerConnectionFactory.initializeAndroidGlobals( +// context, peerConnectionParameters.videoCodecHwAcceleration); +// if (options != null) { +// Log.d(TAG, "Factory networkIgnoreMask option: " + options.networkIgnoreMask); +// } +// factory = new PeerConnectionFactory(options); +// Log.d(TAG, "Peer connection factory created."); +// } +// +// private void createMediaConstraintsInternal() { +// // Create peer connection constraints. +// pcConstraints = new MediaConstraints(); +// // Enable DTLS for normal calls and disable for loopback calls. +// if (peerConnectionParameters.loopback) { +// pcConstraints.optional.add( +// new MediaConstraints.KeyValuePair(DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT, "false")); +// } else { +// pcConstraints.optional.add( +// new MediaConstraints.KeyValuePair(DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT, "true")); +// } +// +// // Check if there is a camera on device and disable video call if not. +// if (videoCapturer == null) { +// Log.w(TAG, "No camera on device. Switch to audio only call."); +// videoCallEnabled = false; +// } +// // Create video constraints if video call is enabled. +// if (videoCallEnabled) { +// videoWidth = peerConnectionParameters.videoWidth; +// videoHeight = peerConnectionParameters.videoHeight; +// videoFps = peerConnectionParameters.videoFps; +// +// // If video resolution is not specified, default to HD. +// if (videoWidth == 0 || videoHeight == 0) { +// videoWidth = HD_VIDEO_WIDTH; +// videoHeight = HD_VIDEO_HEIGHT; +// } +// +// // If fps is not specified, default to 30. +// if (videoFps == 0) { +// videoFps = 30; +// } +// Logging.d(TAG, "Capturing format: " + videoWidth + "x" + videoHeight + "@" + videoFps); +// } +// +// // Create audio constraints. +// audioConstraints = new MediaConstraints(); +// // added for audio performance measurements +// if (peerConnectionParameters.noAudioProcessing) { +// Log.d(TAG, "Disabling audio processing"); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_ECHO_CANCELLATION_CONSTRAINT, "false")); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_AUTO_GAIN_CONTROL_CONSTRAINT, "false")); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_HIGH_PASS_FILTER_CONSTRAINT, "false")); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_NOISE_SUPPRESSION_CONSTRAINT, "false")); +// } +// if (peerConnectionParameters.enableLevelControl) { +// Log.d(TAG, "Enabling level control."); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_LEVEL_CONTROL_CONSTRAINT, "true")); +// } +// // Create SDP constraints. +// sdpMediaConstraints = new MediaConstraints(); +// sdpMediaConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); +// if (videoCallEnabled || peerConnectionParameters.loopback) { +// sdpMediaConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); +// } else { +// sdpMediaConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "false")); +// } +// } +// +// private void createPeerConnectionInternal(EglBase.Context renderEGLContext) { +// if (factory == null || isError) { +// Log.e(TAG, "Peerconnection factory is not created"); +// return; +// } +// Log.d(TAG, "Create peer connection."); +// +// Log.d(TAG, "PCConstraints: " + pcConstraints.toString()); +// queuedRemoteCandidates = new LinkedList(); +// +// if (videoCallEnabled) { +// Log.d(TAG, "EGLContext: " + renderEGLContext); +// factory.setVideoHwAccelerationOptions(renderEGLContext, renderEGLContext); +// } +// +// PeerConnection.RTCConfiguration rtcConfig = +// new PeerConnection.RTCConfiguration(signalingParameters.iceServers); +// // TCP candidates are only useful when connecting to a server that supports +// // ICE-TCP. +// rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; +// rtcConfig.bundlePolicy = PeerConnection.BundlePolicy.MAXBUNDLE; +// rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.REQUIRE; +// rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; +// // Use ECDSA encryption. +// rtcConfig.keyType = PeerConnection.KeyType.ECDSA; +// +// peerConnection = factory.createPeerConnection(rtcConfig, pcConstraints, pcObserver); +// +// if (dataChannelEnabled) { +// DataChannel.Init init = new DataChannel.Init(); +// init.ordered = peerConnectionParameters.dataChannelParameters.ordered; +// init.negotiated = peerConnectionParameters.dataChannelParameters.negotiated; +// init.maxRetransmits = peerConnectionParameters.dataChannelParameters.maxRetransmits; +// init.maxRetransmitTimeMs = peerConnectionParameters.dataChannelParameters.maxRetransmitTimeMs; +// init.id = peerConnectionParameters.dataChannelParameters.id; +// init.protocol = peerConnectionParameters.dataChannelParameters.protocol; +// dataChannel = peerConnection.createDataChannel("ApprtcDemo data", init); +// } +// isInitiator = false; +// +// // Set default WebRTC tracing and INFO libjingle logging. +// // NOTE: this _must_ happen while |factory| is alive! +// Logging.enableTracing("logcat:", EnumSet.of(Logging.TraceLevel.TRACE_DEFAULT)); +// Logging.enableLogToDebugOutput(Logging.Severity.LS_INFO); +// +// mediaStream = factory.createLocalMediaStream("ARDAMS"); +// if (videoCallEnabled) { +// mediaStream.addTrack(createVideoTrack(videoCapturer)); +// } +// +// mediaStream.addTrack(createAudioTrack()); +// peerConnection.addStream(mediaStream); +// if (videoCallEnabled) { +// findVideoSender(); +// } +// +// if (peerConnectionParameters.aecDump) { +// try { +// aecDumpFileDescriptor = +// ParcelFileDescriptor.open(new File(Environment.getExternalStorageDirectory().getPath() +// + File.separator + "Download/audio.aecdump"), +// ParcelFileDescriptor.MODE_READ_WRITE | ParcelFileDescriptor.MODE_CREATE +// | ParcelFileDescriptor.MODE_TRUNCATE); +// factory.startAecDump(aecDumpFileDescriptor.getFd(), -1); +// } catch (IOException e) { +// Log.e(TAG, "Can not open aecdump file", e); +// } +// } +// +// Log.d(TAG, "Peer connection created."); +// } +// +// private void closeInternal() { +// if (factory != null && peerConnectionParameters.aecDump) { +// factory.stopAecDump(); +// } +// Log.d(TAG, "Closing peer connection."); +// statsTimer.cancel(); +// if (dataChannel != null) { +// dataChannel.dispose(); +// dataChannel = null; +// } +// if (peerConnection != null) { +// peerConnection.dispose(); +// peerConnection = null; +// } +// Log.d(TAG, "Closing audio source."); +// if (audioSource != null) { +// audioSource.dispose(); +// audioSource = null; +// } +// Log.d(TAG, "Stopping capture."); +// if (videoCapturer != null) { +// try { +// videoCapturer.stopCapture(); +// } catch (InterruptedException e) { +// throw new RuntimeException(e); +// } +// videoCapturerStopped = true; +// videoCapturer.dispose(); +// videoCapturer = null; +// } +// Log.d(TAG, "Closing video source."); +// if (videoSource != null) { +// videoSource.dispose(); +// videoSource = null; +// } +// localRender = null; +// remoteRenders = null; +// Log.d(TAG, "Closing peer connection factory."); +// if (factory != null) { +// factory.dispose(); +// factory = null; +// } +// options = null; +// Log.d(TAG, "Closing peer connection done."); +// events.onPeerConnectionClosed(); +// PeerConnectionFactory.stopInternalTracingCapture(); +// PeerConnectionFactory.shutdownInternalTracer(); +// events = null; +// } +// +// public boolean isHDVideo() { +// if (!videoCallEnabled) { +// return false; +// } +// +// return videoWidth * videoHeight >= 1280 * 720; +// } +// +// private void getStats() { +// if (peerConnection == null || isError) { +// return; +// } +// boolean success = peerConnection.getStats(new StatsObserver() { +// @Override +// public void onComplete(final StatsReport[] reports) { +// events.onPeerConnectionStatsReady(reports); +// } +// }, null); +// if (!success) { +// Log.e(TAG, "getStats() returns false!"); +// } +// } +// +// public void enableStatsEvents(boolean enable, int periodMs) { +// if (enable) { +// try { +// statsTimer.schedule(new TimerTask() { +// @Override +// public void run() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// getStats(); +// } +// }); +// } +// }, 0, periodMs); +// } catch (Exception e) { +// Log.e(TAG, "Can not schedule statistics timer", e); +// } +// } else { +// statsTimer.cancel(); +// } +// } +// +// public void setAudioEnabled(final boolean enable) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// enableAudio = enable; +// if (localAudioTrack != null) { +// localAudioTrack.setEnabled(enableAudio); +// } +// } +// }); +// } +// +// public void setVideoEnabled(final boolean enable) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// renderVideo = enable; +// if (localVideoTrack != null) { +// localVideoTrack.setEnabled(renderVideo); +// } +// if (remoteVideoTrack != null) { +// remoteVideoTrack.setEnabled(renderVideo); +// } +// } +// }); +// } +// +// public void createOffer() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// Log.d(TAG, "PC Create OFFER"); +// isInitiator = true; +// peerConnection.createOffer(sdpObserver, sdpMediaConstraints); +// } +// } +// }); +// } +// +// public void createAnswer() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// Log.d(TAG, "PC create ANSWER"); +// isInitiator = false; +// peerConnection.createAnswer(sdpObserver, sdpMediaConstraints); +// } +// } +// }); +// } +// +// public void addRemoteIceCandidate(final IceCandidate candidate) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// if (queuedRemoteCandidates != null) { +// queuedRemoteCandidates.add(candidate); +// } else { +// peerConnection.addIceCandidate(candidate); +// } +// } +// } +// }); +// } +// +// public void removeRemoteIceCandidates(final IceCandidate[] candidates) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// // Drain the queued remote candidates if there is any so that +// // they are processed in the proper order. +// drainCandidates(); +// peerConnection.removeIceCandidates(candidates); +// } +// }); +// } +// +// public void setRemoteDescription(final SessionDescription sdp) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// String sdpDescription = sdp.description; +// if (preferIsac) { +// sdpDescription = preferCodec(sdpDescription, AUDIO_CODEC_ISAC, true); +// } +// if (videoCallEnabled) { +// sdpDescription = preferCodec(sdpDescription, preferredVideoCodec, false); +// } +// if (peerConnectionParameters.audioStartBitrate > 0) { +// sdpDescription = setStartBitrate( +// AUDIO_CODEC_OPUS, false, sdpDescription, peerConnectionParameters.audioStartBitrate); +// } +// Log.d(TAG, "Set remote SDP."); +// SessionDescription sdpRemote = new SessionDescription(sdp.type, sdpDescription); +// peerConnection.setRemoteDescription(sdpObserver, sdpRemote); +// } +// }); +// } +// +// public void stopVideoSource() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (videoCapturer != null && !videoCapturerStopped) { +// Log.d(TAG, "Stop video source."); +// try { +// videoCapturer.stopCapture(); +// } catch (InterruptedException e) { +// } +// videoCapturerStopped = true; +// } +// } +// }); +// } +// +// public void startVideoSource() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (videoCapturer != null && videoCapturerStopped) { +// Log.d(TAG, "Restart video source."); +// videoCapturer.startCapture(videoWidth, videoHeight, videoFps); +// videoCapturerStopped = false; +// } +// } +// }); +// } +// +// public void setVideoMaxBitrate(final Integer maxBitrateKbps) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || localVideoSender == null || isError) { +// return; +// } +// Log.d(TAG, "Requested max video bitrate: " + maxBitrateKbps); +// if (localVideoSender == null) { +// Log.w(TAG, "Sender is not ready."); +// return; +// } +// +// RtpParameters parameters = localVideoSender.getParameters(); +// if (parameters.encodings.size() == 0) { +// Log.w(TAG, "RtpParameters are not ready."); +// return; +// } +// +// for (RtpParameters.Encoding encoding : parameters.encodings) { +// // Null value means no limit. +// encoding.maxBitrateBps = maxBitrateKbps == null ? null : maxBitrateKbps * BPS_IN_KBPS; +// } +// if (!localVideoSender.setParameters(parameters)) { +// Log.e(TAG, "RtpSender.setParameters failed."); +// } +// Log.d(TAG, "Configured max video bitrate to: " + maxBitrateKbps); +// } +// }); +// } +// +// private void reportError(final String errorMessage) { +// Log.e(TAG, "Peerconnection error: " + errorMessage); +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (!isError) { +// events.onPeerConnectionError(errorMessage); +// isError = true; +// } +// } +// }); +// } +// +// private AudioTrack createAudioTrack() { +// audioSource = factory.createAudioSource(audioConstraints); +// localAudioTrack = factory.createAudioTrack(AUDIO_TRACK_ID, audioSource); +// localAudioTrack.setEnabled(enableAudio); +// return localAudioTrack; +// } +//æ +// private VideoTrack createVideoTrack(VideoCapturer capturer) { +// videoSource = factory.createVideoSource(capturer); +// capturer.startCapture(videoWidth, videoHeight, videoFps); +// +// localVideoTrack = factory.createVideoTrack(VIDEO_TRACK_ID, videoSource); +// localVideoTrack.setEnabled(renderVideo); +// localVideoTrack.addRenderer(new VideoRenderer(localRender)); +// return localVideoTrack; +// } +// +// private void findVideoSender() { +// for (RtpSender sender : peerConnection.getSenders()) { +// if (sender.track() != null) { +// String trackType = sender.track().kind(); +// if (trackType.equals(VIDEO_TRACK_TYPE)) { +// Log.d(TAG, "Found video sender."); +// localVideoSender = sender; +// } +// } +// } +// } +// +// private static String setStartBitrate( +// String codec, boolean isVideoCodec, String sdpDescription, int bitrateKbps) { +// String[] lines = sdpDescription.split("\r\n"); +// int rtpmapLineIndex = -1; +// boolean sdpFormatUpdated = false; +// String codecRtpMap = null; +// // Search for codec rtpmap in format +// // a=rtpmap: / [/] +// String regex = "^a=rtpmap:(\\d+) " + codec + "(/\\d+)+[\r]?$"; +// Pattern codecPattern = Pattern.compile(regex); +// for (int i = 0; i < lines.length; i++) { +// Matcher codecMatcher = codecPattern.matcher(lines[i]); +// if (codecMatcher.matches()) { +// codecRtpMap = codecMatcher.group(1); +// rtpmapLineIndex = i; +// break; +// } +// } +// if (codecRtpMap == null) { +// Log.w(TAG, "No rtpmap for " + codec + " codec"); +// return sdpDescription; +// } +// Log.d(TAG, "Found " + codec + " rtpmap " + codecRtpMap + " at " + lines[rtpmapLineIndex]); +// +// // Check if a=fmtp string already exist in remote SDP for this codec and +// // update it with new bitrate parameter. +// regex = "^a=fmtp:" + codecRtpMap + " \\w+=\\d+.*[\r]?$"; +// codecPattern = Pattern.compile(regex); +// for (int i = 0; i < lines.length; i++) { +// Matcher codecMatcher = codecPattern.matcher(lines[i]); +// if (codecMatcher.matches()) { +// Log.d(TAG, "Found " + codec + " " + lines[i]); +// if (isVideoCodec) { +// lines[i] += "; " + VIDEO_CODEC_PARAM_START_BITRATE + "=" + bitrateKbps; +// } else { +// lines[i] += "; " + AUDIO_CODEC_PARAM_BITRATE + "=" + (bitrateKbps * 1000); +// } +// Log.d(TAG, "Update remote SDP line: " + lines[i]); +// sdpFormatUpdated = true; +// break; +// } +// } +// +// StringBuilder newSdpDescription = new StringBuilder(); +// for (int i = 0; i < lines.length; i++) { +// newSdpDescription.append(lines[i]).append("\r\n"); +// // Append new a=fmtp line if no such line exist for a codec. +// if (!sdpFormatUpdated && i == rtpmapLineIndex) { +// String bitrateSet; +// if (isVideoCodec) { +// bitrateSet = +// "a=fmtp:" + codecRtpMap + " " + VIDEO_CODEC_PARAM_START_BITRATE + "=" + bitrateKbps; +// } else { +// bitrateSet = "a=fmtp:" + codecRtpMap + " " + AUDIO_CODEC_PARAM_BITRATE + "=" +// + (bitrateKbps * 1000); +// } +// Log.d(TAG, "Add remote SDP line: " + bitrateSet); +// newSdpDescription.append(bitrateSet).append("\r\n"); +// } +// } +// return newSdpDescription.toString(); +// } +// +// /** +// * Returns the line number containing "m=audio|video", or -1 if no such line exists. +// */ +// private static int findMediaDescriptionLine(boolean isAudio, String[] sdpLines) { +// final String mediaDescription = isAudio ? "m=audio " : "m=video "; +// for (int i = 0; i < sdpLines.length; ++i) { +// if (sdpLines[i].startsWith(mediaDescription)) { +// return i; +// } +// } +// return -1; +// } +// +// private static String joinString( +// Iterable s, String delimiter, boolean delimiterAtEnd) { +// Iterator iter = s.iterator(); +// if (!iter.hasNext()) { +// return ""; +// } +// StringBuilder buffer = new StringBuilder(iter.next()); +// while (iter.hasNext()) { +// buffer.append(delimiter).append(iter.next()); +// } +// if (delimiterAtEnd) { +// buffer.append(delimiter); +// } +// return buffer.toString(); +// } +// +// private static String movePayloadTypesToFront(List preferredPayloadTypes, String mLine) { +// // The format of the media description line should be: m= ... +// final List origLineParts = Arrays.asList(mLine.split(" ")); +// if (origLineParts.size() <= 3) { +// Log.e(TAG, "Wrong SDP media description format: " + mLine); +// return null; +// } +// final List header = origLineParts.subList(0, 3); +// final List unpreferredPayloadTypes = +// new ArrayList(origLineParts.subList(3, origLineParts.size())); +// unpreferredPayloadTypes.removeAll(preferredPayloadTypes); +// // Reconstruct the line with |preferredPayloadTypes| moved to the beginning of the payload +// // types. +// final List newLineParts = new ArrayList(); +// newLineParts.addAll(header); +// newLineParts.addAll(preferredPayloadTypes); +// newLineParts.addAll(unpreferredPayloadTypes); +// return joinString(newLineParts, " ", false /* delimiterAtEnd */); +// } +// +// private static String preferCodec(String sdpDescription, String codec, boolean isAudio) { +// final String[] lines = sdpDescription.split("\r\n"); +// final int mLineIndex = findMediaDescriptionLine(isAudio, lines); +// if (mLineIndex == -1) { +// Log.w(TAG, "No mediaDescription line, so can't prefer " + codec); +// return sdpDescription; +// } +// // A list with all the payload types with name |codec|. The payload types are integers in the +// // range 96-127, but they are stored as strings here. +// final List codecPayloadTypes = new ArrayList(); +// // a=rtpmap: / [/] +// final Pattern codecPattern = Pattern.compile("^a=rtpmap:(\\d+) " + codec + "(/\\d+)+[\r]?$"); +// for (int i = 0; i < lines.length; ++i) { +// Matcher codecMatcher = codecPattern.matcher(lines[i]); +// if (codecMatcher.matches()) { +// codecPayloadTypes.add(codecMatcher.group(1)); +// } +// } +// if (codecPayloadTypes.isEmpty()) { +// Log.w(TAG, "No payload types with name " + codec); +// return sdpDescription; +// } +// +// final String newMLine = movePayloadTypesToFront(codecPayloadTypes, lines[mLineIndex]); +// if (newMLine == null) { +// return sdpDescription; +// } +// Log.d(TAG, "Change media description from: " + lines[mLineIndex] + " to " + newMLine); +// lines[mLineIndex] = newMLine; +// return joinString(Arrays.asList(lines), "\r\n", true /* delimiterAtEnd */); +// } +// +// private void drainCandidates() { +// if (queuedRemoteCandidates != null) { +// Log.d(TAG, "Add " + queuedRemoteCandidates.size() + " remote candidates"); +// for (IceCandidate candidate : queuedRemoteCandidates) { +// peerConnection.addIceCandidate(candidate); +// } +// queuedRemoteCandidates = null; +// } +// } +// +// private void switchCameraInternal() { +// if (videoCapturer instanceof CameraVideoCapturer) { +// if (!videoCallEnabled || isError || videoCapturer == null) { +// Log.e(TAG, "Failed to switch camera. Video: " + videoCallEnabled + ". Error : " + isError); +// return; // No video is sent or only one camera is available or error happened. +// } +// Log.d(TAG, "Switch camera"); +// CameraVideoCapturer cameraVideoCapturer = (CameraVideoCapturer) videoCapturer; +// cameraVideoCapturer.switchCamera(null); +// } else { +// Log.d(TAG, "Will not switch camera, video caputurer is not a camera"); +// } +// } +// +// public void switchCamera() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// switchCameraInternal(); +// } +// }); +// } +// +// public void changeCaptureFormat(final int width, final int height, final int framerate) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// changeCaptureFormatInternal(width, height, framerate); +// } +// }); +// } +// +// private void changeCaptureFormatInternal(int width, int height, int framerate) { +// if (!videoCallEnabled || isError || videoCapturer == null) { +// Log.e(TAG, +// "Failed to change capture format. Video: " + videoCallEnabled + ". Error : " + isError); +// return; +// } +// Log.d(TAG, "changeCaptureFormat: " + width + "x" + height + "@" + framerate); +// videoSource.adaptOutputFormat(width, height, framerate); +// } +// +// // Implementation detail: observe ICE & stream changes and react accordingly. +// private class PCObserver implements PeerConnection.Observer { +// @Override +// public void onIceCandidate(final IceCandidate candidate) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// events.onIceCandidate(candidate); +// } +// }); +// } +// +// @Override +// public void onIceCandidatesRemoved(final IceCandidate[] candidates) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// events.onIceCandidatesRemoved(candidates); +// } +// }); +// } +// +// @Override +// public void onSignalingChange(PeerConnection.SignalingState newState) { +// Log.d(TAG, "SignalingState: " + newState); +// } +// +// @Override +// public void onIceConnectionChange(final PeerConnection.IceConnectionState newState) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// Log.d(TAG, "IceConnectionState: " + newState); +// if (newState == IceConnectionState.CONNECTED) { +// events.onIceConnected(); +// } else if (newState == IceConnectionState.DISCONNECTED) { +// events.onIceDisconnected(); +// } else if (newState == IceConnectionState.FAILED) { +// reportError("ICE connection failed."); +// } +// } +// }); +// } +// +// @Override +// public void onIceGatheringChange(PeerConnection.IceGatheringState newState) { +// Log.d(TAG, "IceGatheringState: " + newState); +// } +// +// @Override +// public void onIceConnectionReceivingChange(boolean receiving) { +// Log.d(TAG, "IceConnectionReceiving changed to " + receiving); +// } +// +// @Override +// public void onAddStream(final MediaStream stream) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// if (stream.audioTracks.size() > 1 || stream.videoTracks.size() > 1) { +// reportError("Weird-looking stream: " + stream); +// return; +// } +// if (stream.videoTracks.size() == 1) { +// remoteVideoTrack = stream.videoTracks.get(0); +// remoteVideoTrack.setEnabled(renderVideo); +// for (VideoRenderer.Callbacks remoteRender : remoteRenders) { +// remoteVideoTrack.addRenderer(new VideoRenderer(remoteRender)); +// } +// } +// } +// }); +// } +// +// @Override +// public void onRemoveStream(final MediaStream stream) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// remoteVideoTrack = null; +// } +// }); +// } +// +// @Override +// public void onDataChannel(final DataChannel dc) { +// Log.d(TAG, "New Data channel " + dc.label()); +// +// if (!dataChannelEnabled) +// return; +// +// dc.registerObserver(new DataChannel.Observer() { +// public void onBufferedAmountChange(long previousAmount) { +// Log.d(TAG, "Data channel buffered amount changed: " + dc.label() + ": " + dc.state()); +// } +// +// @Override +// public void onStateChange() { +// Log.d(TAG, "Data channel state changed: " + dc.label() + ": " + dc.state()); +// } +// +// @Override +// public void onMessage(final DataChannel.Buffer buffer) { +// if (buffer.binary) { +// Log.d(TAG, "Received binary msg over " + dc); +// return; +// } +// ByteBuffer data = buffer.data; +// final byte[] bytes = new byte[data.capacity()]; +// data.get(bytes); +// String strData = new String(bytes); +// Log.d(TAG, "Got msg: " + strData + " over " + dc); +// } +// }); +// } +// +// @Override +// public void onRenegotiationNeeded() { +// // No need to do anything; AppRTC follows a pre-agreed-upon +// // signaling/negotiation protocol. +// } +// +// @Override +// public void onAddTrack(final RtpReceiver receiver, final MediaStream[] mediaStreams) { +// } +// } +// +// // Implementation detail: handle offer creation/signaling and answer setting, +// // as well as adding remote ICE candidates once the answer SDP is set. +// private class SDPObserver implements SdpObserver { +// @Override +// public void onCreateSuccess(final SessionDescription origSdp) { +// if (localSdp != null) { +// reportError("Multiple SDP create."); +// return; +// } +// String sdpDescription = origSdp.description; +// if (preferIsac) { +// sdpDescription = preferCodec(sdpDescription, AUDIO_CODEC_ISAC, true); +// } +// if (videoCallEnabled) { +// sdpDescription = preferCodec(sdpDescription, preferredVideoCodec, false); +// } +// final SessionDescription sdp = new SessionDescription(origSdp.type, sdpDescription); +// localSdp = sdp; +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// Log.d(TAG, "Set local SDP from " + sdp.type); +// peerConnection.setLocalDescription(sdpObserver, sdp); +// } +// } +// }); +// } +// +// @Override +// public void onSetSuccess() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// if (isInitiator) { +// // For offering peer connection we first create offer and set +// // local SDP, then after receiving answer set remote SDP. +// if (peerConnection.getRemoteDescription() == null) { +// // We've just set our local SDP so time to send it. +// Log.d(TAG, "Local SDP set succesfully"); +// events.onLocalDescription(localSdp); +// } else { +// // We've just set remote description, so drain remote +// // and send local ICE candidates. +// Log.d(TAG, "Remote SDP set succesfully"); +// drainCandidates(); +// } +// } else { +// // For answering peer connection we set remote SDP and then +// // create answer and set local SDP. +// if (peerConnection.getLocalDescription() != null) { +// // We've just set our local SDP so time to send it, drain +// // remote and send local ICE candidates. +// Log.d(TAG, "Local SDP set succesfully"); +// events.onLocalDescription(localSdp); +// drainCandidates(); +// } else { +// // We've just set remote SDP - do nothing for now - +// // answer will be created soon. +// Log.d(TAG, "Remote SDP set succesfully"); +// } +// } +// } +// }); +// } +// +// @Override +// public void onCreateFailure(final String error) { +// reportError("createSDP error: " + error); +// } +// +// @Override +// public void onSetFailure(final String error) { +// reportError("setSDP error: " + error); +// } +// } +//} diff --git a/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt new file mode 100644 index 0000000000..ffc90d47fa --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import org.webrtc.DataChannel +import org.webrtc.IceCandidate +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.RtpReceiver +import timber.log.Timber + +abstract class PeerConnectionObserverAdapter : PeerConnection.Observer { + override fun onIceCandidate(p0: IceCandidate?) { + Timber.v("## VOIP onIceCandidate $p0") + } + + override fun onDataChannel(p0: DataChannel?) { + Timber.v("## VOIP onDataChannel $p0") + } + + override fun onIceConnectionReceivingChange(p0: Boolean) { + Timber.v("## VOIP onIceConnectionReceivingChange $p0") + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + } + + override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + } + + override fun onAddStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onAddStream $mediaStream") + } + + override fun onSignalingChange(p0: PeerConnection.SignalingState?) { + Timber.v("## VOIP onSignalingChange $p0") + } + + override fun onIceCandidatesRemoved(p0: Array?) { + Timber.v("## VOIP onIceCandidatesRemoved $p0") + } + + override fun onRemoveStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onRemoveStream $mediaStream") + } + + override fun onRenegotiationNeeded() { + Timber.v("## VOIP onRenegotiationNeeded") + } + + override fun onAddTrack(p0: RtpReceiver?, p1: Array?) { + Timber.v("## VOIP onAddTrack $p0 / out: $p1") + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt b/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt new file mode 100644 index 0000000000..0e15c97052 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import timber.log.Timber + +abstract class SdpObserverAdapter : SdpObserver { + override fun onSetFailure(p0: String?) { + Timber.e("## SdpObserver: onSetFailure $p0") + } + + override fun onSetSuccess() { + Timber.v("## SdpObserver: onSetSuccess") + } + + override fun onCreateSuccess(p0: SessionDescription?) { + Timber.e("## SdpObserver: onSetFailure $p0") + } + + override fun onCreateFailure(p0: String?) { + Timber.e("## SdpObserver: onSetFailure $p0") + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt new file mode 100644 index 0000000000..c8387daa20 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -0,0 +1,434 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.app.KeyguardManager +import android.content.Context +import android.content.Intent +import android.os.Build +import android.os.Bundle +import android.os.Parcelable +import android.view.View +import android.view.Window +import android.view.WindowManager +import butterknife.BindView +import com.airbnb.mvrx.MvRx +import com.airbnb.mvrx.viewModel +import im.vector.matrix.android.api.session.call.EglUtils +import im.vector.riotx.R +import im.vector.riotx.core.di.ScreenComponent +import im.vector.riotx.core.platform.VectorBaseActivity +import im.vector.riotx.core.utils.PERMISSIONS_FOR_VIDEO_IP_CALL +import im.vector.riotx.core.utils.allGranted +import im.vector.riotx.core.utils.checkPermissions +import io.reactivex.android.schedulers.AndroidSchedulers +import io.reactivex.subjects.PublishSubject +import kotlinx.android.parcel.Parcelize +import org.webrtc.Camera1Enumerator +import org.webrtc.Camera2Enumerator +import org.webrtc.EglBase +import org.webrtc.IceCandidate +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.RendererCommon +import org.webrtc.SessionDescription +import org.webrtc.SurfaceViewRenderer +import org.webrtc.VideoTrack +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +@Parcelize +data class CallArgs( +// val callId: String? = null, + val roomId: String +) : Parcelable + +class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Listener { + + override fun getLayoutRes() = R.layout.activity_call + + override fun injectWith(injector: ScreenComponent) { + super.injectWith(injector) + injector.inject(this) + } + + private val callViewModel: VectorCallViewModel by viewModel() + + @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager + + @Inject lateinit var viewModelFactory: VectorCallViewModel.Factory + + @BindView(R.id.pip_video_view) + lateinit var pipRenderer: SurfaceViewRenderer + + @BindView(R.id.fullscreen_video_view) + lateinit var fullscreenRenderer: SurfaceViewRenderer + + private var rootEglBase: EglBase? = null + +// private var peerConnectionFactory: PeerConnectionFactory? = null + + //private var peerConnection: PeerConnection? = null + +// private var remoteVideoTrack: VideoTrack? = null + + private val iceCandidateSource: PublishSubject = PublishSubject.create() + + override fun doBeforeSetContentView() { + // Set window styles for fullscreen-window size. Needs to be done before adding content. + requestWindowFeature(Window.FEATURE_NO_TITLE) + window.addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN or WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + setTurnScreenOn(true) + setShowWhenLocked(true) + getSystemService(KeyguardManager::class.java)?.requestDismissKeyguard(this, null) + } else { + @Suppress("DEPRECATION") + window.addFlags( + WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON + or WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED + or WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD + ) + } + + window.decorView.systemUiVisibility = + View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY + setContentView(R.layout.activity_call) + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + rootEglBase = EglUtils.rootEglBase ?: return Unit.also { + finish() + } + + callViewModel.viewEvents + .observe() + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { + handleViewEvents(it) + } + .disposeOnDestroy() +// +// if (isFirstCreation()) { +// +// } + + if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, CAPTURE_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { + start() + } + peerConnectionManager.listener = this + } + + override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { + if (requestCode == CAPTURE_PERMISSION_REQUEST_CODE && allGranted(grantResults)) { + start() + } else { + // TODO display something + finish() + } + } + + private fun start(): Boolean { + // Init Picture in Picture renderer + pipRenderer.init(rootEglBase!!.eglBaseContext, null) + pipRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) + + // Init Full Screen renderer + fullscreenRenderer.init(rootEglBase!!.eglBaseContext, null) + fullscreenRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) + + + pipRenderer.setZOrderMediaOverlay(true); + pipRenderer.setEnableHardwareScaler(true /* enabled */); + fullscreenRenderer.setEnableHardwareScaler(true /* enabled */); + // Start with local feed in fullscreen and swap it to the pip when the call is connected. + //setSwappedFeeds(true /* isSwappedFeeds */); + + if (isFirstCreation()) { + peerConnectionManager.createPeerConnectionFactory() + + val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) + val frontCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isFrontFacing(it) } + ?: cameraIterator.deviceNames?.first() + ?: return true + val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + + val iceServers = ArrayList().apply { + listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { + add( + PeerConnection.IceServer.builder(it) + .setUsername("xxxxx") + .setPassword("xxxxx") + .createIceServer() + ) + } + } + + peerConnectionManager.createPeerConnection(videoCapturer, iceServers) + peerConnectionManager.startCall() + } +// PeerConnectionFactory.initialize(PeerConnectionFactory +// .InitializationOptions.builder(applicationContext) +// .createInitializationOptions() +// ) + +// val options = PeerConnectionFactory.Options() +// val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( +// rootEglBase!!.eglBaseContext, /* enableIntelVp8Encoder */ +// true, /* enableH264HighProfile */ +// true) +// val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(rootEglBase!!.eglBaseContext) +// +// peerConnectionFactory = PeerConnectionFactory.builder() +// .setOptions(options) +// .setVideoEncoderFactory(defaultVideoEncoderFactory) +// .setVideoDecoderFactory(defaultVideoDecoderFactory) +// .createPeerConnectionFactory() + +// val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) +// val frontCamera = cameraIterator.deviceNames +// ?.firstOrNull { cameraIterator.isFrontFacing(it) } +// ?: cameraIterator.deviceNames?.first() +// ?: return true +// val videoCapturer = cameraIterator.createCapturer(frontCamera, null) +// +// // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object +// val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) +// +// val videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) +// videoCapturer.initialize(surfaceTextureHelper, this, videoSource!!.capturerObserver) +// videoCapturer.startCapture(1280, 720, 30) +// +// +// val localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) +// +// // create a local audio track +// val audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) +// val audioTrack = peerConnectionFactory?.createAudioTrack("ARDAMSa0", audioSource) + + pipRenderer.setMirror(true) +// localVideoTrack?.addSink(pipRenderer) + + /* + { + "username": "1586847781:@valere35:matrix.org", + "password": "ZzbqbqfT9O2G3WpCpesdts2lyns=", + "ttl": 86400.0, + "uris": ["turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp"] + } + */ + +// val iceServers = ArrayList().apply { +// listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { +// add( +// PeerConnection.IceServer.builder(it) +// .setUsername("1586847781:@valere35:matrix.org") +// .setPassword("ZzbqbqfT9O2G3WpCpesdts2lyns=") +// .createIceServer() +// ) +// } +// } +// +// val iceCandidateSource: PublishSubject = PublishSubject.create() +// +// iceCandidateSource +// .buffer(400, TimeUnit.MILLISECONDS) +// .subscribe { +// // omit empty :/ +// if (it.isNotEmpty()) { +// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) +// } +// } +// .disposeOnDestroy() +// +// peerConnection = peerConnectionFactory?.createPeerConnection( +// iceServers, +// object : PeerConnectionObserverAdapter() { +// override fun onIceCandidate(p0: IceCandidate?) { +// p0?.let { +// iceCandidateSource.onNext(it) +// } +// } +// +// override fun onAddStream(mediaStream: MediaStream?) { +// runOnUiThread { +// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> +// remoteVideoTrack = videoTrack +// remoteVideoTrack?.setEnabled(true) +// remoteVideoTrack?.addSink(fullscreenRenderer) +// } +// } +// } +// +// override fun onRemoveStream(mediaStream: MediaStream?) { +// remoteVideoTrack = null +// } +// +// override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { +// if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { +// // TODO prompt something? +// finish() +// } +// } +// } +// ) +// +// val localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? +// localMediaStream?.addTrack(localVideoTrack) +// localMediaStream?.addTrack(audioTrack) +// +// val constraints = MediaConstraints() +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) +// +// peerConnection?.addStream(localMediaStream) +// +// peerConnection?.createOffer(object : SdpObserver { +// override fun onSetFailure(p0: String?) { +// Timber.v("## VOIP onSetFailure $p0") +// } +// +// override fun onSetSuccess() { +// Timber.v("## VOIP onSetSuccess") +// } +// +// override fun onCreateSuccess(sessionDescription: SessionDescription) { +// Timber.v("## VOIP onCreateSuccess $sessionDescription") +// peerConnection?.setLocalDescription(object : SdpObserverAdapter() { +// override fun onSetSuccess() { +// callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) +// } +// }, sessionDescription) +// } +// +// override fun onCreateFailure(p0: String?) { +// Timber.v("## VOIP onCreateFailure $p0") +// } +// }, constraints) + iceCandidateSource + .buffer(400, TimeUnit.MILLISECONDS) + .subscribe { + // omit empty :/ + if (it.isNotEmpty()) { + callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) + } + } + .disposeOnDestroy() + + peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) + return false + } + + override fun onDestroy() { + peerConnectionManager.detachRenderers() + peerConnectionManager.listener = this + super.onDestroy() + } + + private fun handleViewEvents(event: VectorCallViewEvents?) { + when (event) { + is VectorCallViewEvents.CallAnswered -> { + val sdp = SessionDescription(SessionDescription.Type.ANSWER, event.content.answer.sdp) + peerConnectionManager.answerReceived("", sdp) +// peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) + } + } + } + +// @TargetApi(17) +// private fun getDisplayMetrics(): DisplayMetrics? { +// val displayMetrics = DisplayMetrics() +// val windowManager = application.getSystemService(Context.WINDOW_SERVICE) as WindowManager +// windowManager.defaultDisplay.getRealMetrics(displayMetrics) +// return displayMetrics +// } + +// @TargetApi(21) +// private fun startScreenCapture() { +// val mediaProjectionManager: MediaProjectionManager = application.getSystemService( +// Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager +// startActivityForResult( +// mediaProjectionManager.createScreenCaptureIntent(), CAPTURE_PERMISSION_REQUEST_CODE) +// } +// +// override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { +// if (requestCode != CAPTURE_PERMISSION_REQUEST_CODE) { +// super.onActivityResult(requestCode, resultCode, data) +// } +//// mediaProjectionPermissionResultCode = resultCode; +//// mediaProjectionPermissionResultData = data; +//// startCall(); +// } + + companion object { + + private const val CAPTURE_PERMISSION_REQUEST_CODE = 1 + +// private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { +// // add all existing audio filters to avoid having echos +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) +// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) +// } + + fun newIntent(context: Context, signalingRoomId: String): Intent { + return Intent(context, VectorCallActivity::class.java).apply { + putExtra(MvRx.KEY_ARG, CallArgs(roomId = signalingRoomId)) + } + } + } + + override fun addLocalIceCandidate(candidates: IceCandidate) { + iceCandidateSource.onNext(candidates) + } + + override fun addRemoteVideoTrack(videoTrack: VideoTrack) { + runOnUiThread { + videoTrack.setEnabled(true) + videoTrack.addSink(fullscreenRenderer) + } + } + + override fun addLocalVideoTrack(videoTrack: VideoTrack) { + runOnUiThread { + videoTrack.addSink(pipRenderer) + } + } + + override fun removeRemoteVideoStream(mediaStream: MediaStream) { + } + + override fun onDisconnect() { + } + + override fun sendOffer(sessionDescription: SessionDescription) { + callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt new file mode 100644 index 0000000000..a995e3197e --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import androidx.lifecycle.viewModelScope +import com.airbnb.mvrx.MvRxState +import com.airbnb.mvrx.MvRxViewModelFactory +import com.airbnb.mvrx.ViewModelContext +import com.squareup.inject.assisted.Assisted +import com.squareup.inject.assisted.AssistedInject +import im.vector.matrix.android.api.session.Session +import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.matrix.android.internal.util.awaitCallback +import im.vector.riotx.core.extensions.exhaustive +import im.vector.riotx.core.platform.VectorViewEvents +import im.vector.riotx.core.platform.VectorViewModel +import im.vector.riotx.core.platform.VectorViewModelAction +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription +import java.util.UUID + +data class VectorCallViewState( + val callId: String? = null, + val roomId: String = "" +) : MvRxState + +sealed class VectorCallViewActions : VectorViewModelAction { + + data class SendOffer(val sdp: SessionDescription) : VectorCallViewActions() + data class AddLocalIceCandidate(val iceCandidates: List) : VectorCallViewActions() +} + +sealed class VectorCallViewEvents : VectorViewEvents { + + data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() +} + +class VectorCallViewModel @AssistedInject constructor( + @Assisted initialState: VectorCallViewState, + val session: Session +) : VectorViewModel(initialState) { + + private val callServiceListener: CallsListener = object : CallsListener { + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + withState { state -> + if (callAnswerContent.callId == state.callId) { + _viewEvents.post(VectorCallViewEvents.CallAnswered(callAnswerContent)) + } + } + } + + override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { + + } + } + + init { + session.callService().addCallListener(callServiceListener) + } + + override fun onCleared() { + session.callService().removeCallListener(callServiceListener) + super.onCleared() + } + + override fun handle(action: VectorCallViewActions) = withState { state -> + when (action) { + is VectorCallViewActions.SendOffer -> { + viewModelScope.launch(Dispatchers.IO) { + awaitCallback { + val callId = state.callId ?: UUID.randomUUID().toString().also { + setState { + copy(callId = it) + } + } + session.callService().sendOfferSdp(callId, state.roomId, action.sdp, it) + } + } + } + is VectorCallViewActions.AddLocalIceCandidate -> { + viewModelScope.launch { + session.callService().sendLocalIceCandidates(state.callId ?: "", state.roomId, action.iceCandidates) + } + } + }.exhaustive + } + + @AssistedInject.Factory + interface Factory { + fun create(initialState: VectorCallViewState): VectorCallViewModel + } + + companion object : MvRxViewModelFactory { + + @JvmStatic + override fun create(viewModelContext: ViewModelContext, state: VectorCallViewState): VectorCallViewModel? { + val callActivity: VectorCallActivity = viewModelContext.activity() + return callActivity.viewModelFactory.create(state) + } + + override fun initialState(viewModelContext: ViewModelContext): VectorCallViewState? { + val args: CallArgs = viewModelContext.args() + return VectorCallViewState(roomId = args.roomId) + } + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt new file mode 100644 index 0000000000..633f2482cf --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.os.Build +import android.telecom.Connection +import android.telecom.ConnectionRequest +import android.telecom.ConnectionService +import android.telecom.PhoneAccountHandle +import androidx.annotation.RequiresApi + +/** + * No active calls in other apps + * + *To answer incoming calls when there are no active calls in other apps, follow these steps: + * + * 

+ *     * Your app receives a new incoming call using its usual mechanisms.
+ *          - Use the addNewIncomingCall(PhoneAccountHandle, Bundle) method to inform the telecom subsystem about the new incoming call.
+ *          - The telecom subsystem binds to your app's ConnectionService implementation and requests a new instance of the
+ *            Connection class representing the new incoming call using the onCreateIncomingConnection(PhoneAccountHandle, ConnectionRequest) method.
+ *          - The telecom subsystem informs your app that it should show its incoming call user interface using the onShowIncomingCallUi() method.
+ *          - Your app shows its incoming UI using a notification with an associated full-screen intent. For more information, see onShowIncomingCallUi().
+ *          - Call the setActive() method if the user accepts the incoming call, or setDisconnected(DisconnectCause) specifying REJECTED as
+ *            the parameter followed by a call to the destroy() method if the user rejects the incoming call.
+ *
+ */ +@RequiresApi(Build.VERSION_CODES.M) class VectorConnectionService : ConnectionService() { + + /** + * The telecom subsystem calls this method in response to your app calling placeCall(Uri, Bundle) to create a new outgoing call + */ + override fun onCreateOutgoingConnection(connectionManagerPhoneAccount: PhoneAccountHandle?, request: ConnectionRequest?): Connection? { + val callId = request?.address?.encodedQuery ?: return null + val roomId = request.extras.getString("MX_CALL_ROOM_ID") ?: return null + return CallConnection(applicationContext, roomId, callId) + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt new file mode 100644 index 0000000000..7833ac1787 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -0,0 +1,372 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.content.ComponentName +import android.content.Context +import android.graphics.drawable.Icon +import android.os.Build +import android.os.Bundle +import android.telecom.PhoneAccount +import android.telecom.PhoneAccountHandle +import android.telecom.TelecomManager +import androidx.core.content.ContextCompat +import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.call.EglUtils +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.riotx.BuildConfig +import im.vector.riotx.R +import org.webrtc.AudioSource +import org.webrtc.AudioTrack +import org.webrtc.DefaultVideoDecoderFactory +import org.webrtc.DefaultVideoEncoderFactory +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.SurfaceTextureHelper +import org.webrtc.SurfaceViewRenderer +import org.webrtc.VideoCapturer +import org.webrtc.VideoSource +import org.webrtc.VideoTrack +import timber.log.Timber +import java.lang.ref.WeakReference +import java.util.concurrent.Executors +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Manage peerConnectionFactory & Peer connections outside of activity lifecycle to resist configuration changes + * Use app context + */ +@Singleton +class WebRtcPeerConnectionManager @Inject constructor( + private val context: Context +) : CallsListener { + + interface Listener { + fun addLocalIceCandidate(candidates: IceCandidate) + fun addRemoteVideoTrack(videoTrack: VideoTrack) + fun addLocalVideoTrack(videoTrack: VideoTrack) + fun removeRemoteVideoStream(mediaStream: MediaStream) + fun onDisconnect() + fun sendOffer(sessionDescription: SessionDescription) + } + + var phoneAccountHandle: PhoneAccountHandle? = null + + init { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + val componentName = ComponentName(BuildConfig.APPLICATION_ID, VectorConnectionService::class.java.name) + val appName = context.getString(R.string.app_name) + phoneAccountHandle = PhoneAccountHandle(componentName, appName) + val phoneAccount = PhoneAccount.Builder(phoneAccountHandle, appName) + .setIcon(Icon.createWithResource(context, R.drawable.riotx_logo)) + .build() + ContextCompat.getSystemService(context, TelecomManager::class.java) + ?.registerPhoneAccount(phoneAccount) + } else { + // ignore? + } + } + + var listener: Listener? = null + + // *Comments copied from webrtc demo app* + // Executor thread is started once and is used for all + // peer connection API calls to ensure new peer connection factory is + // created on the same thread as previously destroyed factory. + private val executor = Executors.newSingleThreadExecutor(); + + private val rootEglBase by lazy { EglUtils.rootEglBase } + + private var peerConnectionFactory: PeerConnectionFactory? = null + + private var peerConnection: PeerConnection? = null + + private var remoteVideoTrack: VideoTrack? = null + private var localVideoTrack: VideoTrack? = null + + private var videoSource: VideoSource? = null + private var audioSource: AudioSource? = null + private var audioTrack: AudioTrack? = null + + private var videoCapturer: VideoCapturer? = null + + var localSurfaceRenderer: WeakReference? = null + var remoteSurfaceRenderer: WeakReference? = null + + fun createPeerConnectionFactory() { + executor.execute { + if (peerConnectionFactory == null) { + Timber.v("## VOIP createPeerConnectionFactory") + val eglBaseContext = rootEglBase?.eglBaseContext ?: return@execute Unit.also { + Timber.e("## VOIP No EGL BASE") + } + + Timber.v("## VOIP PeerConnectionFactory.initialize") + PeerConnectionFactory.initialize(PeerConnectionFactory + .InitializationOptions.builder(context.applicationContext) + .createInitializationOptions() + ) + + val options = PeerConnectionFactory.Options() + val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( + eglBaseContext, + /* enableIntelVp8Encoder */ + true, + /* enableH264HighProfile */ + true) + val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) + + + Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") + peerConnectionFactory = PeerConnectionFactory.builder() + .setOptions(options) + .setVideoEncoderFactory(defaultVideoEncoderFactory) + .setVideoDecoderFactory(defaultVideoDecoderFactory) + .createPeerConnectionFactory() + } + } + } + + fun createPeerConnection(videoCapturer: VideoCapturer, iceServers: List) { + executor.execute { + Timber.v("## VOIP PeerConnectionFactory.createPeerConnection ${peerConnectionFactory}...") + // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + + videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) + Timber.v("## VOIP Local video source created") + videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) + videoCapturer.startCapture(1280, 720, 30) + + localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { + Timber.v("## VOIP Local video track created") + listener?.addLocalVideoTrack(it) +// localSurfaceRenderer?.get()?.let { surface -> +//// it.addSink(surface) +//// } + } + + // create a local audio track + Timber.v("## VOIP create local audio track") + audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + +// pipRenderer.setMirror(true) +// localVideoTrack?.addSink(pipRenderer) +// + +// val iceCandidateSource: PublishSubject = PublishSubject.create() +// +// iceCandidateSource +// .buffer(400, TimeUnit.MILLISECONDS) +// .subscribe { +// // omit empty :/ +// if (it.isNotEmpty()) { +// listener.addLocalIceCandidate() +// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) +// } +// } +// .disposeOnDestroy() + + Timber.v("## VOIP creating peer connection... ") + peerConnection = peerConnectionFactory?.createPeerConnection( + iceServers, + object : PeerConnectionObserverAdapter() { + override fun onIceCandidate(p0: IceCandidate?) { + Timber.v("## VOIP onIceCandidate local $p0") + p0?.let { + // iceCandidateSource.onNext(it) + listener?.addLocalIceCandidate(it) + } + } + + override fun onAddStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onAddStream remote $mediaStream") + mediaStream?.videoTracks?.firstOrNull()?.let { + listener?.addRemoteVideoTrack(it) + remoteVideoTrack = it +// remoteSurfaceRenderer?.get()?.let { surface -> +// it.setEnabled(true) +// it.addSink(surface) +// } + } +// runOnUiThread { +// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> +// remoteVideoTrack = videoTrack +// remoteVideoTrack?.setEnabled(true) +// remoteVideoTrack?.addSink(fullscreenRenderer) +// } +// } + } + + override fun onRemoveStream(mediaStream: MediaStream?) { + mediaStream?.let { + listener?.removeRemoteVideoStream(it) + } + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + remoteVideoTrack = null + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { + listener?.onDisconnect() + } + } + } + ) + + val localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? + localMediaStream?.addTrack(localVideoTrack) + localMediaStream?.addTrack(audioTrack) + +// val constraints = MediaConstraints() +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) + + Timber.v("## VOIP add local stream to peer connection") + peerConnection?.addStream(localMediaStream) + } + } + + fun answerReceived(callId: String, answerSdp: SessionDescription) { + executor.execute { + Timber.v("## answerReceived $callId") + peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, answerSdp) + } + } + + fun startCall() { + executor.execute { + val constraints = MediaConstraints() + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) + + Timber.v("## VOIP creating offer...") + peerConnection?.createOffer(object : SdpObserver { + override fun onSetFailure(p0: String?) { + Timber.v("## VOIP onSetFailure $p0") + } + + override fun onSetSuccess() { + Timber.v("## VOIP onSetSuccess") + } + + override fun onCreateSuccess(sessionDescription: SessionDescription) { + Timber.v("## VOIP onCreateSuccess $sessionDescription") + peerConnection?.setLocalDescription(object : SdpObserverAdapter() { + override fun onSetSuccess() { + listener?.sendOffer(sessionDescription) + //callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } + }, sessionDescription) + } + + override fun onCreateFailure(p0: String?) { + Timber.v("## VOIP onCreateFailure $p0") + } + }, constraints) + } + } + + fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { + localVideoTrack?.addSink(localViewRenderer) + remoteVideoTrack?.let { + it.setEnabled(true) + it.addSink(remoteViewRenderer) + } + localSurfaceRenderer = WeakReference(localViewRenderer) + remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + } + + fun detachRenderers() { + localSurfaceRenderer?.get()?.let { + localVideoTrack?.removeSink(it) + } + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + localSurfaceRenderer = null + remoteSurfaceRenderer = null + } + + fun close() { + executor.execute { + peerConnectionFactory?.stopAecDump() + peerConnectionFactory = null + audioSource?.dispose() + videoSource?.dispose() + peerConnection?.dispose() + peerConnection = null + videoCapturer?.dispose() + } + } + + companion object { + + private const val AUDIO_TRACK_ID = "ARDAMSa0" + + private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { + // add all existing audio filters to avoid having echos + mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) + mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) + } + } + + override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + ContextCompat.getSystemService(context, TelecomManager::class.java)?.let { telecomManager -> + phoneAccountHandle?.let { phoneAccountHandle -> + telecomManager.addNewIncomingCall( + phoneAccountHandle, + Bundle().apply { + putString("MX_CALL_ROOM_ID", signalingRoomId) + putString("MX_CALL_CALL_ID", callInviteContent.callId) + } + ) + } + + } + } + } + + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + } +} + + diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt index fba4f9e79e..896b29009e 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt @@ -68,6 +68,7 @@ sealed class RoomDetailAction : VectorViewModelAction { object ClearSendQueue : RoomDetailAction() object ResendAll : RoomDetailAction() + object StartCall : RoomDetailAction() data class AcceptVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() data class DeclineVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index bf386d15a2..2b743ed940 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -127,6 +127,7 @@ import im.vector.riotx.features.attachments.ContactAttachment import im.vector.riotx.features.attachments.preview.AttachmentsPreviewActivity import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs import im.vector.riotx.features.attachments.toGroupedContentAttachmentData +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.command.Command import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity import im.vector.riotx.features.crypto.util.toImageRes @@ -479,6 +480,17 @@ class RoomDetailFragment @Inject constructor( } else -> super.onOptionsItemSelected(item) } + if (item.itemId == R.id.resend_all) { + roomDetailViewModel.handle(RoomDetailAction.ResendAll) + return true + } + if (item.itemId == R.id.voip_call) { + VectorCallActivity.newIntent(requireContext(), roomDetailArgs.roomId).let { + startActivity(it) + } + return true + } + return super.onOptionsItemSelected(item) } private fun displayDisabledIntegrationDialog() { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index 1785151af5..c25bd6c8da 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -66,6 +66,7 @@ import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.resources.StringProvider import im.vector.riotx.core.resources.UserPreferencesProvider import im.vector.riotx.core.utils.subscribeLogError +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.command.CommandParser import im.vector.riotx.features.command.ParsedCommand import im.vector.riotx.features.crypto.verification.SupportedVerificationMethodsProvider @@ -372,6 +373,7 @@ class RoomDetailViewModel @AssistedInject constructor( R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 R.id.open_matrix_apps -> true + R.id.voip_call -> room.roomSummary()?.isDirect == true && room.roomSummary()?.joinedMembersCount == 2 else -> false } diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt index 33079a8f33..c1ec2d1cac 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt @@ -239,7 +239,7 @@ class NoticeEventFormatter @Inject constructor(private val sessionHolder: Active return when (type) { EventType.CALL_INVITE -> { val content = event.getClearContent().toModel() ?: return null - val isVideoCall = content.offer.sdp == CallInviteContent.Offer.SDP_VIDEO + val isVideoCall = content.offer?.sdp == CallInviteContent.Offer.SDP_VIDEO return if (isVideoCall) { if (event.isSentByCurrentUser()) { sp.getString(R.string.notice_placed_video_call_by_you) diff --git a/vector/src/main/res/drawable/ic_phone.xml b/vector/src/main/res/drawable/ic_phone.xml new file mode 100644 index 0000000000..430c438577 --- /dev/null +++ b/vector/src/main/res/drawable/ic_phone.xml @@ -0,0 +1,14 @@ + + + diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml new file mode 100644 index 0000000000..f1a2a3075f --- /dev/null +++ b/vector/src/main/res/layout/activity_call.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/layout/fragment_call.xml b/vector/src/main/res/layout/fragment_call.xml new file mode 100644 index 0000000000..2ab342d68a --- /dev/null +++ b/vector/src/main/res/layout/fragment_call.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index a4adb203f2..8e271a0285 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -8,6 +8,14 @@ android:title="@string/room_add_matrix_apps" app:showAsAction="never" /> + + Date: Wed, 13 May 2020 22:25:40 +0300 Subject: [PATCH 02/83] Finish CallActivity when m.call.hangup received. --- .../android/api/session/call/CallsListener.kt | 3 +++ .../session/call/DefaultCallService.kt | 14 ++++++++++++++ .../riotx/features/call/VectorCallActivity.kt | 4 ++++ .../riotx/features/call/VectorCallViewModel.kt | 10 ++++++++++ .../call/WebRtcPeerConnectionManager.kt | 18 +++++++++++++----- 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index 556555369a..d48841f1bb 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -17,6 +17,7 @@ package im.vector.matrix.android.api.session.call import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent interface CallsListener { @@ -44,4 +45,6 @@ interface CallsListener { fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) + fun onCallHangupReceived(callHangupContent: CallHangupContent) + } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index ab30a61ebf..61f72e5a4c 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -30,6 +30,7 @@ import im.vector.matrix.android.api.session.events.model.toContent import im.vector.matrix.android.api.session.events.model.toModel import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent +import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.matrix.android.internal.di.UserId import im.vector.matrix.android.internal.session.SessionScope @@ -146,6 +147,19 @@ internal class DefaultCallService @Inject constructor( onCallInvite(event.roomId ?: "", it) } } + EventType.CALL_HANGUP -> { + event.getClearContent().toModel()?.let { + onCallHangup(it) + } + } + } + } + + private fun onCallHangup(hangup: CallHangupContent) { + callListeners.forEach { + tryThis { + it.onCallHangupReceived(hangup) + } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index c8387daa20..f1d50fd7b3 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -348,6 +348,9 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis peerConnectionManager.answerReceived("", sdp) // peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) } + is VectorCallViewEvents.CallHangup -> { + finish() + } } } @@ -423,6 +426,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } override fun removeRemoteVideoStream(mediaStream: MediaStream) { + } override fun onDisconnect() { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index a995e3197e..95a3677f61 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -25,6 +25,7 @@ import com.squareup.inject.assisted.AssistedInject import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.matrix.android.internal.util.awaitCallback import im.vector.riotx.core.extensions.exhaustive @@ -51,6 +52,7 @@ sealed class VectorCallViewActions : VectorViewModelAction { sealed class VectorCallViewEvents : VectorViewEvents { data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() + data class CallHangup(val content: CallHangupContent) : VectorCallViewEvents() } class VectorCallViewModel @AssistedInject constructor( @@ -70,6 +72,14 @@ class VectorCallViewModel @AssistedInject constructor( override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { } + + override fun onCallHangupReceived(callHangupContent: CallHangupContent) { + withState { state -> + if (callHangupContent.callId == state.callId) { + _viewEvents.post(VectorCallViewEvents.CallHangup(callHangupContent)) + } + } + } } init { diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 7833ac1787..e58404d8fb 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -28,6 +28,7 @@ import androidx.core.content.ContextCompat import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.riotx.BuildConfig import im.vector.riotx.R @@ -72,6 +73,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } var phoneAccountHandle: PhoneAccountHandle? = null + var localMediaStream: MediaStream? = null init { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { @@ -239,7 +241,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } ) - val localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? + localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? localMediaStream?.addTrack(localVideoTrack) localMediaStream?.addTrack(audioTrack) @@ -315,13 +317,15 @@ class WebRtcPeerConnectionManager @Inject constructor( fun close() { executor.execute { - peerConnectionFactory?.stopAecDump() - peerConnectionFactory = null + // Do not dispose peer connection (https://bugs.chromium.org/p/webrtc/issues/detail?id=7543) + peerConnection?.close() + peerConnection?.removeStream(localMediaStream) + peerConnection = null audioSource?.dispose() videoSource?.dispose() - peerConnection?.dispose() - peerConnection = null videoCapturer?.dispose() + peerConnectionFactory?.stopAecDump() + peerConnectionFactory = null } } @@ -367,6 +371,10 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { } + + override fun onCallHangupReceived(callHangupContent: CallHangupContent) { + close() + } } From 4a4edcf82a1d4dfc168c278a91e2f5761bee9491 Mon Sep 17 00:00:00 2001 From: onurays Date: Tue, 19 May 2020 10:57:17 +0300 Subject: [PATCH 03/83] Experimental implementation of Telecom API. --- .../android/api/session/call/CallService.kt | 5 - .../android/api/session/call/CallsListener.kt | 1 - .../api/session/call/PeerSignalingClient.kt | 14 +-- .../android/api/session/call/VoipApi.kt | 1 - .../session/call/CallEventObserver.kt | 1 - .../session/call/DefaultCallService.kt | 4 +- .../riotx/features/debug/DebugMenuActivity.kt | 2 +- .../vector/riotx/core/services/CallService.kt | 15 +++ .../riotx/features/call/CallConnection.kt | 93 ++++++++++++++++++- .../riotx/features/call/VectorCallActivity.kt | 18 ++-- .../features/call/VectorCallViewModel.kt | 1 - .../features/call/VectorConnectionService.kt | 39 ++++++++ .../call/WebRtcPeerConnectionManager.kt | 23 +++-- .../home/room/detail/RoomDetailViewModel.kt | 1 - 14 files changed, 176 insertions(+), 42 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt index 5e3f331148..bb5a81907e 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -22,12 +22,10 @@ import org.webrtc.SessionDescription interface CallService { - fun getTurnServer(callback: MatrixCallback) fun isCallSupportedInRoom(roomId: String) : Boolean - /** * Send offer SDP to the other participant. */ @@ -48,10 +46,7 @@ interface CallService { */ fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) - fun addCallListener(listener: CallsListener) fun removeCallListener(listener: CallsListener) - - } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index d48841f1bb..ff8ddb8de9 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -46,5 +46,4 @@ interface CallsListener { fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) fun onCallHangupReceived(callHangupContent: CallHangupContent) - } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt index 9a948adbb8..ab99f8875a 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt @@ -1,4 +1,4 @@ -///* +// /* // * Copyright (c) 2020 New Vector Ltd // * // * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,13 +14,13 @@ // * limitations under the License. // */ // -//package im.vector.matrix.android.api.session.call +// package im.vector.matrix.android.api.session.call // -//import im.vector.matrix.android.api.MatrixCallback -//import org.webrtc.IceCandidate -//import org.webrtc.SessionDescription +// import im.vector.matrix.android.api.MatrixCallback +// import org.webrtc.IceCandidate +// import org.webrtc.SessionDescription // -//interface PeerSignalingClient { +// interface PeerSignalingClient { // // val callID: String // @@ -63,4 +63,4 @@ // */ // fun onRemoteIceCandidatesRemoved(candidates: List) // } -//} +// } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt index e324822617..dc92af8023 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt @@ -24,5 +24,4 @@ internal interface VoipApi { @GET(NetworkConstants.URI_API_PREFIX_PATH_R0 + "voip/turnServer") fun getTurnServer(): Call - } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt index 61e6087737..ad70cb245d 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt @@ -24,7 +24,6 @@ import im.vector.matrix.android.internal.database.model.EventEntity import im.vector.matrix.android.internal.database.query.whereTypes import im.vector.matrix.android.internal.di.SessionDatabase import im.vector.matrix.android.internal.di.UserId -import im.vector.matrix.android.internal.session.room.EventRelationsAggregationTask import io.realm.OrderedCollectionChangeSet import io.realm.RealmConfiguration import io.realm.RealmResults diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index 61f72e5a4c..0617a657fc 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -51,11 +51,11 @@ internal class DefaultCallService @Inject constructor( private val callListeners = ArrayList() override fun getTurnServer(callback: MatrixCallback) { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + TODO("not implemented") // To change body of created functions use File | Settings | File Templates. } override fun isCallSupportedInRoom(roomId: String): Boolean { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + TODO("not implemented") // To change body of created functions use File | Settings | File Templates. } override fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { diff --git a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt index 16db7b0c38..8b4a777454 100644 --- a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt +++ b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt @@ -184,7 +184,7 @@ class DebugMenuActivity : VectorBaseActivity() { @OnClick(R.id.debug_scan_qr_code) fun scanQRCode() { if (checkPermissions(PERMISSIONS_FOR_TAKING_PHOTO, this, PERMISSION_REQUEST_CODE_LAUNCH_CAMERA)) { - //doScanQRCode() + // doScanQRCode() startActivity(VectorCallActivity.newIntent(this, "!cyIJhOLwWgmmqreHLD:matrix.org")) } } diff --git a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt index 30ab62d5b2..fba433727a 100644 --- a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt @@ -1,3 +1,4 @@ + /* * Copyright 2019 New Vector Ltd * @@ -20,8 +21,10 @@ package im.vector.riotx.core.services import android.content.Context import android.content.Intent +import android.os.Binder import androidx.core.content.ContextCompat import im.vector.riotx.core.extensions.vectorComponent +import im.vector.riotx.features.call.CallConnection import im.vector.riotx.features.notifications.NotificationUtils import timber.log.Timber @@ -30,6 +33,8 @@ import timber.log.Timber */ class CallService : VectorService() { + private val connections = mutableMapOf() + /** * call in progress (foreground notification) */ @@ -154,6 +159,10 @@ class CallService : VectorService() { myStopSelf() } + fun addConnection(callConnection: CallConnection) { + connections[callConnection.callId] = callConnection + } + companion object { private const val NOTIFICATION_ID = 6480 @@ -214,4 +223,10 @@ class CallService : VectorService() { ContextCompat.startForegroundService(context, intent) } } + + inner class CallServiceBinder : Binder() { + fun getCallService(): CallService { + return this@CallService + } + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt index 5f1cd81383..9523b4ebdd 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt @@ -19,24 +19,113 @@ package im.vector.riotx.features.call import android.content.Context import android.os.Build import android.telecom.Connection +import android.telecom.DisconnectCause import androidx.annotation.RequiresApi +import org.webrtc.Camera1Enumerator +import org.webrtc.Camera2Enumerator +import org.webrtc.IceCandidate +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.SessionDescription +import org.webrtc.VideoTrack +import timber.log.Timber +import javax.inject.Inject @RequiresApi(Build.VERSION_CODES.M) class CallConnection( private val context: Context, private val roomId: String, - private val callId: String -) : Connection() { + val callId: String +) : Connection(), WebRtcPeerConnectionManager.Listener { + + @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager + @Inject lateinit var callViewModel: VectorCallViewModel + + init { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + connectionProperties = PROPERTY_SELF_MANAGED + } + } /** * The telecom subsystem calls this method when you add a new incoming call and your app should show its incoming call UI. */ override fun onShowIncomingCallUi() { + super.onShowIncomingCallUi() + Timber.i("onShowIncomingCallUi") + /* VectorCallActivity.newIntent(context, roomId).let { context.startActivity(it) } + */ } override fun onAnswer() { super.onAnswer() + // startCall() + Timber.i("onShowIncomingCallUi") + } + + override fun onStateChanged(state: Int) { + super.onStateChanged(state) + Timber.i("onStateChanged${stateToString(state)}") + } + + override fun onReject() { + super.onReject() + Timber.i("onReject") + close() + } + + override fun onDisconnect() { + onDisconnect() + Timber.i("onDisconnect") + close() + } + + private fun close() { + setDisconnected(DisconnectCause(DisconnectCause.CANCELED)) + destroy() + } + + private fun startCall() { + peerConnectionManager.createPeerConnectionFactory() + peerConnectionManager.listener = this + + val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) + val frontCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isFrontFacing(it) } + ?: cameraIterator.deviceNames?.first() + ?: return + val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + + val iceServers = ArrayList().apply { + listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { + add( + PeerConnection.IceServer.builder(it) + .setUsername("xxxxx") + .setPassword("xxxxx") + .createIceServer() + ) + } + } + + peerConnectionManager.createPeerConnection(videoCapturer, iceServers) + peerConnectionManager.startCall() + } + + override fun addLocalIceCandidate(candidates: IceCandidate) { + } + + override fun addRemoteVideoTrack(videoTrack: VideoTrack) { + } + + override fun addLocalVideoTrack(videoTrack: VideoTrack) { + } + + override fun removeRemoteVideoStream(mediaStream: MediaStream) { + } + + override fun sendOffer(sessionDescription: SessionDescription) { + callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index f1d50fd7b3..51a4341336 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -82,7 +82,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // private var peerConnectionFactory: PeerConnectionFactory? = null - //private var peerConnection: PeerConnection? = null + // private var peerConnection: PeerConnection? = null // private var remoteVideoTrack: VideoTrack? = null @@ -152,12 +152,11 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis fullscreenRenderer.init(rootEglBase!!.eglBaseContext, null) fullscreenRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) - - pipRenderer.setZOrderMediaOverlay(true); - pipRenderer.setEnableHardwareScaler(true /* enabled */); - fullscreenRenderer.setEnableHardwareScaler(true /* enabled */); + pipRenderer.setZOrderMediaOverlay(true) + pipRenderer.setEnableHardwareScaler(true /* enabled */) + fullscreenRenderer.setEnableHardwareScaler(true /* enabled */) // Start with local feed in fullscreen and swap it to the pip when the call is connected. - //setSwappedFeeds(true /* isSwappedFeeds */); + // setSwappedFeeds(true /* isSwappedFeeds */); if (isFirstCreation()) { peerConnectionManager.createPeerConnectionFactory() @@ -374,9 +373,9 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // if (requestCode != CAPTURE_PERMISSION_REQUEST_CODE) { // super.onActivityResult(requestCode, resultCode, data) // } -//// mediaProjectionPermissionResultCode = resultCode; -//// mediaProjectionPermissionResultData = data; -//// startCall(); +// // mediaProjectionPermissionResultCode = resultCode; +// // mediaProjectionPermissionResultData = data; +// // startCall(); // } companion object { @@ -426,7 +425,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } override fun removeRemoteVideoStream(mediaStream: MediaStream) { - } override fun onDisconnect() { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 95a3677f61..c8b8a3316c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -70,7 +70,6 @@ class VectorCallViewModel @AssistedInject constructor( } override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { - } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt index 633f2482cf..fed93c9faa 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt @@ -16,12 +16,20 @@ package im.vector.riotx.features.call +import android.content.ComponentName +import android.content.Intent +import android.content.ServiceConnection +import android.net.Uri import android.os.Build +import android.os.IBinder import android.telecom.Connection import android.telecom.ConnectionRequest import android.telecom.ConnectionService import android.telecom.PhoneAccountHandle +import android.telecom.StatusHints +import android.telecom.TelecomManager import androidx.annotation.RequiresApi +import im.vector.riotx.core.services.CallService /** * No active calls in other apps @@ -49,4 +57,35 @@ import androidx.annotation.RequiresApi val roomId = request.extras.getString("MX_CALL_ROOM_ID") ?: return null return CallConnection(applicationContext, roomId, callId) } + + override fun onCreateIncomingConnection(connectionManagerPhoneAccount: PhoneAccountHandle?, request: ConnectionRequest?): Connection { + val roomId = request?.extras?.getString("MX_CALL_ROOM_ID") ?: return super.onCreateIncomingConnection(connectionManagerPhoneAccount, request) + val callId = request.extras.getString("MX_CALL_CALL_ID") ?: return super.onCreateIncomingConnection(connectionManagerPhoneAccount, request) + + val connection = CallConnection(applicationContext, roomId, callId) + connection.connectionCapabilities = Connection.CAPABILITY_MUTE + connection.audioModeIsVoip = true + connection.setAddress(Uri.fromParts("tel", "+905000000000", null), TelecomManager.PRESENTATION_ALLOWED) + connection.setCallerDisplayName("RiotX Caller", TelecomManager.PRESENTATION_ALLOWED) + connection.statusHints = StatusHints("Testing Hint...", null, null) + + bindService(Intent(applicationContext, CallService::class.java), CallServiceConnection(connection), 0) + connection.setInitializing() + return CallConnection(applicationContext, roomId, callId) + } + + inner class CallServiceConnection(private val callConnection: CallConnection) : ServiceConnection { + override fun onServiceConnected(name: ComponentName?, binder: IBinder?) { + val callSrvBinder = binder as CallService.CallServiceBinder + callSrvBinder.getCallService().addConnection(callConnection) + unbindService(this) + } + + override fun onServiceDisconnected(name: ComponentName?) { + } + } + + companion object { + const val TAG = "TComService" + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index e58404d8fb..b9685e188a 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -24,6 +24,7 @@ import android.os.Bundle import android.telecom.PhoneAccount import android.telecom.PhoneAccountHandle import android.telecom.TelecomManager +import android.telecom.VideoProfile import androidx.core.content.ContextCompat import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils @@ -80,8 +81,11 @@ class WebRtcPeerConnectionManager @Inject constructor( val componentName = ComponentName(BuildConfig.APPLICATION_ID, VectorConnectionService::class.java.name) val appName = context.getString(R.string.app_name) phoneAccountHandle = PhoneAccountHandle(componentName, appName) - val phoneAccount = PhoneAccount.Builder(phoneAccountHandle, appName) + val phoneAccount = PhoneAccount.Builder(phoneAccountHandle, BuildConfig.APPLICATION_ID) .setIcon(Icon.createWithResource(context, R.drawable.riotx_logo)) + .setCapabilities(PhoneAccount.CAPABILITY_SELF_MANAGED) + .setCapabilities(PhoneAccount.CAPABILITY_VIDEO_CALLING) + .setCapabilities(PhoneAccount.CAPABILITY_CALL_SUBJECT) .build() ContextCompat.getSystemService(context, TelecomManager::class.java) ?.registerPhoneAccount(phoneAccount) @@ -96,7 +100,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // Executor thread is started once and is used for all // peer connection API calls to ensure new peer connection factory is // created on the same thread as previously destroyed factory. - private val executor = Executors.newSingleThreadExecutor(); + private val executor = Executors.newSingleThreadExecutor() private val rootEglBase by lazy { EglUtils.rootEglBase } @@ -139,7 +143,6 @@ class WebRtcPeerConnectionManager @Inject constructor( true) val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) - Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") peerConnectionFactory = PeerConnectionFactory.builder() .setOptions(options) @@ -152,7 +155,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun createPeerConnection(videoCapturer: VideoCapturer, iceServers: List) { executor.execute { - Timber.v("## VOIP PeerConnectionFactory.createPeerConnection ${peerConnectionFactory}...") + Timber.v("## VOIP PeerConnectionFactory.createPeerConnection $peerConnectionFactory...") // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) @@ -165,8 +168,8 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP Local video track created") listener?.addLocalVideoTrack(it) // localSurfaceRenderer?.get()?.let { surface -> -//// it.addSink(surface) -//// } +// // it.addSink(surface) +// // } } // create a local audio track @@ -282,7 +285,7 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnection?.setLocalDescription(object : SdpObserverAdapter() { override fun onSetSuccess() { listener?.sendOffer(sessionDescription) - //callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + // callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) } }, sessionDescription) } @@ -361,10 +364,12 @@ class WebRtcPeerConnectionManager @Inject constructor( Bundle().apply { putString("MX_CALL_ROOM_ID", signalingRoomId) putString("MX_CALL_CALL_ID", callInviteContent.callId) + putParcelable(TelecomManager.EXTRA_PHONE_ACCOUNT_HANDLE, phoneAccountHandle) + putInt(TelecomManager.EXTRA_INCOMING_VIDEO_STATE, VideoProfile.STATE_BIDIRECTIONAL) + putInt(TelecomManager.EXTRA_START_CALL_WITH_VIDEO_STATE, VideoProfile.STATE_BIDIRECTIONAL) } ) } - } } } @@ -376,5 +381,3 @@ class WebRtcPeerConnectionManager @Inject constructor( close() } } - - diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index c25bd6c8da..49a469847b 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -66,7 +66,6 @@ import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.resources.StringProvider import im.vector.riotx.core.resources.UserPreferencesProvider import im.vector.riotx.core.utils.subscribeLogError -import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.command.CommandParser import im.vector.riotx.features.command.ParsedCommand import im.vector.riotx.features.crypto.verification.SupportedVerificationMethodsProvider From 4169f580b8cfa51a7dc3d5418686af6bf25d3836 Mon Sep 17 00:00:00 2001 From: onurays Date: Wed, 20 May 2020 11:08:53 +0300 Subject: [PATCH 04/83] Create foreground call service. --- .../src/main/res/values/strings.xml | 3 + vector/src/main/AndroidManifest.xml | 17 ++- .../vector/riotx/core/services/CallService.kt | 2 +- .../call/WebRtcPeerConnectionManager.kt | 9 +- .../call/service/CallHeadsUpActionReceiver.kt | 43 +++++++ .../call/service/CallHeadsUpService.kt | 118 ++++++++++++++++++ .../call/service/CallHeadsUpServiceArgs.kt | 27 ++++ .../call/{ => telecom}/CallConnection.kt | 5 +- .../{ => telecom}/VectorConnectionService.kt | 2 +- .../home/room/detail/RoomDetailFragment.kt | 6 +- .../main/res/drawable/ic_call_incoming.xml | 5 + 11 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt rename vector/src/main/java/im/vector/riotx/features/call/{ => telecom}/CallConnection.kt (94%) rename vector/src/main/java/im/vector/riotx/features/call/{ => telecom}/VectorConnectionService.kt (99%) create mode 100644 vector/src/main/res/drawable/ic_call_incoming.xml diff --git a/matrix-sdk-android/src/main/res/values/strings.xml b/matrix-sdk-android/src/main/res/values/strings.xml index 3cd7674253..a34c3a1f9f 100644 --- a/matrix-sdk-android/src/main/res/values/strings.xml +++ b/matrix-sdk-android/src/main/res/values/strings.xml @@ -362,4 +362,7 @@ %s is requesting to verify your key, but your client does not support in-chat key verification. You will need to use legacy key verification to verify keys. + Accept + Reject + diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 1cd522337e..76281afe0b 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -12,11 +12,13 @@ - - + + + + @@ -195,15 +197,24 @@ android:name=".core.services.VectorSyncService" android:exported="false" /> - + + + + = Build.VERSION_CODES.M) { ContextCompat.getSystemService(context, TelecomManager::class.java)?.let { telecomManager -> phoneAccountHandle?.let { phoneAccountHandle -> @@ -372,6 +376,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } } } + */ } override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt new file mode 100644 index 0000000000..99f163d42d --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call.service + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import timber.log.Timber + +class CallHeadsUpActionReceiver : BroadcastReceiver() { + + override fun onReceive(context: Context, intent: Intent?) { + when (intent?.getIntExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, 0)) { + CallHeadsUpService.CALL_ACTION_ANSWER -> onCallAnswerClicked() + CallHeadsUpService.CALL_ACTION_REJECT -> onCallRejectClicked() + } + + context.sendBroadcast(Intent(Intent.ACTION_CLOSE_SYSTEM_DIALOGS)) + context.stopService(Intent(context, CallHeadsUpService::class.java)) + } + + private fun onCallRejectClicked() { + Timber.d("onCallRejectClicked") + } + + private fun onCallAnswerClicked() { + Timber.d("onCallAnswerClicked") + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt new file mode 100644 index 0000000000..b8fa16d6ad --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call.service + +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Context +import android.content.Intent +import android.media.AudioAttributes +import android.media.AudioManager +import android.net.Uri +import android.os.Binder +import android.os.Build +import android.os.IBinder +import androidx.core.app.NotificationCompat +import im.vector.riotx.R + +class CallHeadsUpService : Service() { + + private val CHANNEL_ID = "CallChannel" + private val CHANNEL_NAME = "Call Channel" + private val CHANNEL_DESCRIPTION = "Call Notifications" + + private val binder: IBinder = CallHeadsUpServiceBinder() + + override fun onBind(intent: Intent): IBinder? { + return binder + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + //val callHeadsUpServiceArgs: CallHeadsUpServiceArgs? = intent?.extras?.getParcelable(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS) + + val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { + putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) + } + val rejectCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { + putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_REJECT) + } + + val answerCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) + val rejectCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_REJECT, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) + + createNotificationChannel() + + val notification = NotificationCompat + .Builder(applicationContext, CHANNEL_ID) + .setContentTitle("Title") + .setContentText("Content") + .setSmallIcon(R.drawable.ic_call_incoming) + .setPriority(NotificationCompat.PRIORITY_HIGH) + .setCategory(NotificationCompat.CATEGORY_CALL) + .addAction(R.drawable.ic_call_incoming, getString(R.string.call_notification_answer), answerCallPendingIntent) + .addAction(R.drawable.ic_call_incoming, getString(R.string.call_notification_reject), rejectCallPendingIntent) + .setAutoCancel(true) + .setSound(Uri.parse("android.resource://" + applicationContext.packageName + "/ring.ogg")) + .setFullScreenIntent(answerCallPendingIntent, true) + .build() + + startForeground(NOTIFICATION_ID, notification) + + return START_STICKY + } + + private fun createNotificationChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + + val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH).apply { + description = CHANNEL_DESCRIPTION + setSound( + Uri.parse("android.resource://" + applicationContext.packageName + "/ring.ogg"), + AudioAttributes + .Builder() + .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) + .setLegacyStreamType(AudioManager.STREAM_RING) + .build() + ) + } + applicationContext.getSystemService(NotificationManager::class.java)?.createNotificationChannel(channel) + } + + inner class CallHeadsUpServiceBinder : Binder() { + + fun getService() = this@CallHeadsUpService + } + + companion object { + private const val EXTRA_CALL_HEADS_UP_SERVICE_PARAMS = "EXTRA_CALL_PARAMS" + + const val EXTRA_CALL_ACTION_KEY = "EXTRA_CALL_ACTION_KEY" + const val CALL_ACTION_ANSWER = 100 + const val CALL_ACTION_REJECT = 101 + + private const val NOTIFICATION_ID = 999 + + fun newInstance(context: Context, callerDisplayName: String, isIncomingCall: Boolean, isVideoCall: Boolean): Intent { + val args = CallHeadsUpServiceArgs(callerDisplayName, isIncomingCall, isVideoCall) + return Intent(context, CallHeadsUpService::class.java).apply { + putExtra(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS, args) + } + } + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt new file mode 100644 index 0000000000..9769724944 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call.service + +import android.os.Parcelable +import kotlinx.android.parcel.Parcelize + +@Parcelize +data class CallHeadsUpServiceArgs( + val callerDisplayName: String, + val isIncomingCall: Boolean, + val isVideoCall: Boolean +) : Parcelable diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt similarity index 94% rename from vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt rename to vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt index 9523b4ebdd..27bdf24570 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt @@ -14,13 +14,16 @@ * limitations under the License. */ -package im.vector.riotx.features.call +package im.vector.riotx.features.call.telecom import android.content.Context import android.os.Build import android.telecom.Connection import android.telecom.DisconnectCause import androidx.annotation.RequiresApi +import im.vector.riotx.features.call.VectorCallViewActions +import im.vector.riotx.features.call.VectorCallViewModel +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import org.webrtc.Camera1Enumerator import org.webrtc.Camera2Enumerator import org.webrtc.IceCandidate diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt b/vector/src/main/java/im/vector/riotx/features/call/telecom/VectorConnectionService.kt similarity index 99% rename from vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt rename to vector/src/main/java/im/vector/riotx/features/call/telecom/VectorConnectionService.kt index fed93c9faa..8185c9fc49 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/telecom/VectorConnectionService.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package im.vector.riotx.features.call +package im.vector.riotx.features.call.telecom import android.content.ComponentName import android.content.Intent diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index 2b743ed940..efa045ddd2 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -127,7 +127,7 @@ import im.vector.riotx.features.attachments.ContactAttachment import im.vector.riotx.features.attachments.preview.AttachmentsPreviewActivity import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs import im.vector.riotx.features.attachments.toGroupedContentAttachmentData -import im.vector.riotx.features.call.VectorCallActivity +import im.vector.riotx.features.call.service.CallHeadsUpService import im.vector.riotx.features.command.Command import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity import im.vector.riotx.features.crypto.util.toImageRes @@ -485,9 +485,13 @@ class RoomDetailFragment @Inject constructor( return true } if (item.itemId == R.id.voip_call) { + /* VectorCallActivity.newIntent(requireContext(), roomDetailArgs.roomId).let { startActivity(it) } + */ + val callHeadsUpServiceIntent = Intent(requireContext(), CallHeadsUpService::class.java) + ContextCompat.startForegroundService(requireContext(), callHeadsUpServiceIntent) return true } return super.onOptionsItemSelected(item) diff --git a/vector/src/main/res/drawable/ic_call_incoming.xml b/vector/src/main/res/drawable/ic_call_incoming.xml new file mode 100644 index 0000000000..0f4e2d1d89 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_incoming.xml @@ -0,0 +1,5 @@ + + + From bda1633979a5cca988cb71059931d04b090ef6fd Mon Sep 17 00:00:00 2001 From: onurays Date: Mon, 25 May 2020 11:23:45 +0300 Subject: [PATCH 05/83] New material resources added. --- .../riotx/features/call/VectorCallActivity.kt | 10 +-- .../src/main/res/drawable/bg_call_actions.xml | 12 ++++ .../{ic_call_incoming.xml => ic_call.xml} | 0 vector/src/main/res/drawable/ic_call_end.xml | 10 +++ .../drawable/ic_call_flip_camera_active.xml | 11 +++ .../drawable/ic_call_flip_camera_default.xml | 7 ++ .../main/res/drawable/ic_call_mute_active.xml | 11 +++ .../res/drawable/ic_call_mute_default.xml | 7 ++ .../res/drawable/ic_call_speaker_active.xml | 11 +++ .../res/drawable/ic_call_speaker_default.xml | 7 ++ .../drawable/ic_call_videocam_off_active.xml | 11 +++ .../drawable/ic_call_videocam_off_default.xml | 7 ++ vector/src/main/res/drawable/ic_videocam.xml | 5 ++ vector/src/main/res/layout/activity_call.xml | 67 ++++++++++++++++++- 14 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 vector/src/main/res/drawable/bg_call_actions.xml rename vector/src/main/res/drawable/{ic_call_incoming.xml => ic_call.xml} (100%) create mode 100644 vector/src/main/res/drawable/ic_call_end.xml create mode 100644 vector/src/main/res/drawable/ic_call_flip_camera_active.xml create mode 100644 vector/src/main/res/drawable/ic_call_flip_camera_default.xml create mode 100644 vector/src/main/res/drawable/ic_call_mute_active.xml create mode 100644 vector/src/main/res/drawable/ic_call_mute_default.xml create mode 100644 vector/src/main/res/drawable/ic_call_speaker_active.xml create mode 100644 vector/src/main/res/drawable/ic_call_speaker_default.xml create mode 100644 vector/src/main/res/drawable/ic_call_videocam_off_active.xml create mode 100644 vector/src/main/res/drawable/ic_call_videocam_off_default.xml create mode 100644 vector/src/main/res/drawable/ic_videocam.xml diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 51a4341336..fef5aa0647 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -53,8 +53,10 @@ import javax.inject.Inject @Parcelize data class CallArgs( -// val callId: String? = null, - val roomId: String + val roomId: String, + val participantUserId: String, + val isIncomingCall: Boolean, + val isVideoCall: Boolean ) : Parcelable class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Listener { @@ -400,9 +402,9 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) // } - fun newIntent(context: Context, signalingRoomId: String): Intent { + fun newIntent(context: Context, roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean): Intent { return Intent(context, VectorCallActivity::class.java).apply { - putExtra(MvRx.KEY_ARG, CallArgs(roomId = signalingRoomId)) + putExtra(MvRx.KEY_ARG, CallArgs(roomId, participantUserId, isIncomingCall, isVideoCall)) } } } diff --git a/vector/src/main/res/drawable/bg_call_actions.xml b/vector/src/main/res/drawable/bg_call_actions.xml new file mode 100644 index 0000000000..f074beb8f9 --- /dev/null +++ b/vector/src/main/res/drawable/bg_call_actions.xml @@ -0,0 +1,12 @@ + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/drawable/ic_call_incoming.xml b/vector/src/main/res/drawable/ic_call.xml similarity index 100% rename from vector/src/main/res/drawable/ic_call_incoming.xml rename to vector/src/main/res/drawable/ic_call.xml diff --git a/vector/src/main/res/drawable/ic_call_end.xml b/vector/src/main/res/drawable/ic_call_end.xml new file mode 100644 index 0000000000..2879c2433e --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_end.xml @@ -0,0 +1,10 @@ + + + + diff --git a/vector/src/main/res/drawable/ic_call_flip_camera_active.xml b/vector/src/main/res/drawable/ic_call_flip_camera_active.xml new file mode 100644 index 0000000000..25590cc753 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_flip_camera_active.xml @@ -0,0 +1,11 @@ + + + + diff --git a/vector/src/main/res/drawable/ic_call_flip_camera_default.xml b/vector/src/main/res/drawable/ic_call_flip_camera_default.xml new file mode 100644 index 0000000000..75ad0133f8 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_flip_camera_default.xml @@ -0,0 +1,7 @@ + + + diff --git a/vector/src/main/res/drawable/ic_call_mute_active.xml b/vector/src/main/res/drawable/ic_call_mute_active.xml new file mode 100644 index 0000000000..757f9cfa17 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_mute_active.xml @@ -0,0 +1,11 @@ + + + + diff --git a/vector/src/main/res/drawable/ic_call_mute_default.xml b/vector/src/main/res/drawable/ic_call_mute_default.xml new file mode 100644 index 0000000000..37a0c83fec --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_mute_default.xml @@ -0,0 +1,7 @@ + + + diff --git a/vector/src/main/res/drawable/ic_call_speaker_active.xml b/vector/src/main/res/drawable/ic_call_speaker_active.xml new file mode 100644 index 0000000000..97035b1915 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_speaker_active.xml @@ -0,0 +1,11 @@ + + + + diff --git a/vector/src/main/res/drawable/ic_call_speaker_default.xml b/vector/src/main/res/drawable/ic_call_speaker_default.xml new file mode 100644 index 0000000000..2fc06a5795 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_speaker_default.xml @@ -0,0 +1,7 @@ + + + diff --git a/vector/src/main/res/drawable/ic_call_videocam_off_active.xml b/vector/src/main/res/drawable/ic_call_videocam_off_active.xml new file mode 100644 index 0000000000..106317ed56 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_videocam_off_active.xml @@ -0,0 +1,11 @@ + + + + diff --git a/vector/src/main/res/drawable/ic_call_videocam_off_default.xml b/vector/src/main/res/drawable/ic_call_videocam_off_default.xml new file mode 100644 index 0000000000..0b3d9baf04 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_videocam_off_default.xml @@ -0,0 +1,7 @@ + + + diff --git a/vector/src/main/res/drawable/ic_videocam.xml b/vector/src/main/res/drawable/ic_videocam.xml new file mode 100644 index 0000000000..b7a50f9a57 --- /dev/null +++ b/vector/src/main/res/drawable/ic_videocam.xml @@ -0,0 +1,5 @@ + + + diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index f1a2a3075f..3b158d8828 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -1,10 +1,11 @@ - + + + + + + + + + + + + + + - \ No newline at end of file + \ No newline at end of file From fb6bcc8470caaa30ae0ab2cd0cc48a7d81099ecd Mon Sep 17 00:00:00 2001 From: onurays Date: Mon, 25 May 2020 11:26:04 +0300 Subject: [PATCH 06/83] Foreground call service and action receiver implemented. --- .../call/service/CallHeadsUpActionReceiver.kt | 10 ++- .../call/service/CallHeadsUpService.kt | 81 ++++++++++++------- 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt index 99f163d42d..5f07866043 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt @@ -19,6 +19,7 @@ package im.vector.riotx.features.call.service import android.content.BroadcastReceiver import android.content.Context import android.content.Intent +import im.vector.riotx.features.settings.VectorLocale.context import timber.log.Timber class CallHeadsUpActionReceiver : BroadcastReceiver() { @@ -28,16 +29,19 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { CallHeadsUpService.CALL_ACTION_ANSWER -> onCallAnswerClicked() CallHeadsUpService.CALL_ACTION_REJECT -> onCallRejectClicked() } - - context.sendBroadcast(Intent(Intent.ACTION_CLOSE_SYSTEM_DIALOGS)) - context.stopService(Intent(context, CallHeadsUpService::class.java)) } private fun onCallRejectClicked() { Timber.d("onCallRejectClicked") + stopService() } private fun onCallAnswerClicked() { Timber.d("onCallAnswerClicked") } + + private fun stopService() { + context.sendBroadcast(Intent(Intent.ACTION_CLOSE_SYSTEM_DIALOGS)) + context.stopService(Intent(context, CallHeadsUpService::class.java)) + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt index b8fa16d6ad..653ac90f0d 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt @@ -16,20 +16,22 @@ package im.vector.riotx.features.call.service +import android.app.Notification import android.app.NotificationChannel import android.app.NotificationManager import android.app.PendingIntent import android.app.Service +import android.content.ContentResolver.SCHEME_ANDROID_RESOURCE import android.content.Context import android.content.Intent import android.media.AudioAttributes -import android.media.AudioManager import android.net.Uri import android.os.Binder import android.os.Build import android.os.IBinder import androidx.core.app.NotificationCompat import im.vector.riotx.R +import im.vector.riotx.features.call.VectorCallActivity class CallHeadsUpService : Service() { @@ -44,37 +46,57 @@ class CallHeadsUpService : Service() { } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { - //val callHeadsUpServiceArgs: CallHeadsUpServiceArgs? = intent?.extras?.getParcelable(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS) + val callHeadsUpServiceArgs: CallHeadsUpServiceArgs? = intent?.extras?.getParcelable(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS) + createNotificationChannel() + + val title = callHeadsUpServiceArgs?.participantUserId ?: "" + val description = when { + callHeadsUpServiceArgs?.isIncomingCall == false -> getString(R.string.call_ring) + callHeadsUpServiceArgs?.isVideoCall == true -> getString(R.string.incoming_video_call) + else -> getString(R.string.incoming_voice_call) + } + + val actions = if (callHeadsUpServiceArgs?.isIncomingCall == true) createAnswerAndRejectActions() else emptyList() + + createNotification(title, description, actions).also { + startForeground(NOTIFICATION_ID, it) + } + + return START_STICKY + } + + private fun createNotification(title: String, content: String, actions: List): Notification { + return NotificationCompat + .Builder(applicationContext, CHANNEL_ID) + .setContentTitle(title) + .setContentText(content) + .setSmallIcon(R.drawable.ic_call) + .setPriority(NotificationCompat.PRIORITY_MAX) + .setCategory(NotificationCompat.CATEGORY_CALL) + .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) + .setDefaults(NotificationCompat.DEFAULT_SOUND or NotificationCompat.DEFAULT_VIBRATE) + .setSound(Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg")) + .setVibrate(longArrayOf(1000, 1000)) + .setFullScreenIntent(PendingIntent.getActivity(applicationContext, 0, Intent(applicationContext, VectorCallActivity::class.java), 0), true) + .setOngoing(true) + .apply { actions.forEach { addAction(it) } } + .build() + } + + private fun createAnswerAndRejectActions(): List { val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) } val rejectCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_REJECT) } - val answerCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) val rejectCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_REJECT, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) - - createNotificationChannel() - - val notification = NotificationCompat - .Builder(applicationContext, CHANNEL_ID) - .setContentTitle("Title") - .setContentText("Content") - .setSmallIcon(R.drawable.ic_call_incoming) - .setPriority(NotificationCompat.PRIORITY_HIGH) - .setCategory(NotificationCompat.CATEGORY_CALL) - .addAction(R.drawable.ic_call_incoming, getString(R.string.call_notification_answer), answerCallPendingIntent) - .addAction(R.drawable.ic_call_incoming, getString(R.string.call_notification_reject), rejectCallPendingIntent) - .setAutoCancel(true) - .setSound(Uri.parse("android.resource://" + applicationContext.packageName + "/ring.ogg")) - .setFullScreenIntent(answerCallPendingIntent, true) - .build() - - startForeground(NOTIFICATION_ID, notification) - - return START_STICKY + return listOf( + NotificationCompat.Action(R.drawable.ic_call, getString(R.string.call_notification_answer), answerCallPendingIntent), + NotificationCompat.Action(R.drawable.vector_notification_reject_invitation, getString(R.string.call_notification_reject), rejectCallPendingIntent) + ) } private fun createNotificationChannel() { @@ -83,13 +105,16 @@ class CallHeadsUpService : Service() { val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH).apply { description = CHANNEL_DESCRIPTION setSound( - Uri.parse("android.resource://" + applicationContext.packageName + "/ring.ogg"), + Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg"), AudioAttributes .Builder() .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) - .setLegacyStreamType(AudioManager.STREAM_RING) + .setUsage(AudioAttributes.USAGE_NOTIFICATION_RINGTONE) .build() - ) + ) + lockscreenVisibility = NotificationCompat.VISIBILITY_PUBLIC + enableVibration(true) + enableLights(true) } applicationContext.getSystemService(NotificationManager::class.java)?.createNotificationChannel(channel) } @@ -108,8 +133,8 @@ class CallHeadsUpService : Service() { private const val NOTIFICATION_ID = 999 - fun newInstance(context: Context, callerDisplayName: String, isIncomingCall: Boolean, isVideoCall: Boolean): Intent { - val args = CallHeadsUpServiceArgs(callerDisplayName, isIncomingCall, isVideoCall) + fun newInstance(context: Context, roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean): Intent { + val args = CallHeadsUpServiceArgs(roomId, participantUserId, isIncomingCall, isVideoCall) return Intent(context, CallHeadsUpService::class.java).apply { putExtra(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS, args) } From 5d476e72590392bb52c8f0edbb8851b754dbbcaf Mon Sep 17 00:00:00 2001 From: onurays Date: Mon, 25 May 2020 11:27:46 +0300 Subject: [PATCH 07/83] Show the foreground service for incoming and outgoing calls. --- .../android/api/session/call/CallsListener.kt | 2 +- .../session/call/DefaultCallService.kt | 8 +- .../riotx/features/debug/DebugMenuActivity.kt | 4 +- vector/src/main/AndroidManifest.xml | 3 +- .../features/call/VectorCallViewModel.kt | 2 +- .../call/WebRtcPeerConnectionManager.kt | 80 +++++++------------ .../call/service/CallHeadsUpServiceArgs.kt | 3 +- .../home/room/detail/RoomDetailFragment.kt | 15 ++-- .../home/room/detail/RoomDetailViewModel.kt | 4 +- vector/src/main/res/menu/menu_timeline.xml | 7 ++ vector/src/main/res/values/colors_riotx.xml | 9 +++ 11 files changed, 69 insertions(+), 68 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index ff8ddb8de9..1fc4170f5c 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -41,7 +41,7 @@ interface CallsListener { // */ // fun onCallHangUp(peerSignalingClient: PeerSignalingClient) - fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) + fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index 0617a657fc..fba50a0c37 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -144,7 +144,7 @@ internal class DefaultCallService @Inject constructor( } EventType.CALL_INVITE -> { event.getClearContent().toModel()?.let { - onCallInvite(event.roomId ?: "", it) + onCallInvite(event.roomId ?: "", event.senderId ?: "", it) } } EventType.CALL_HANGUP -> { @@ -171,10 +171,12 @@ internal class DefaultCallService @Inject constructor( } } - private fun onCallInvite(roomId: String, answer: CallInviteContent) { + private fun onCallInvite(roomId: String, userId: String, answer: CallInviteContent) { + if (userId == this.userId) return + callListeners.forEach { tryThis { - it.onCallInviteReceived(roomId, answer) + it.onCallInviteReceived(roomId, userId, answer) } } } diff --git a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt index 8b4a777454..2a8bc22e2e 100644 --- a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt +++ b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt @@ -35,7 +35,6 @@ import im.vector.riotx.core.utils.PERMISSION_REQUEST_CODE_LAUNCH_CAMERA import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions import im.vector.riotx.core.utils.toast -import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.debug.sas.DebugSasEmojiActivity import im.vector.riotx.features.qrcode.QrCodeScannerActivity import kotlinx.android.synthetic.debug.activity_debug_menu.* @@ -185,7 +184,8 @@ class DebugMenuActivity : VectorBaseActivity() { fun scanQRCode() { if (checkPermissions(PERMISSIONS_FOR_TAKING_PHOTO, this, PERMISSION_REQUEST_CODE_LAUNCH_CAMERA)) { // doScanQRCode() - startActivity(VectorCallActivity.newIntent(this, "!cyIJhOLwWgmmqreHLD:matrix.org")) + // TODO. Find a better way? + //startActivity(VectorCallActivity.newIntent(this, "!cyIJhOLwWgmmqreHLD:matrix.org")) } } diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 76281afe0b..abb6c929cc 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -18,7 +18,8 @@ intent action. --> - + + diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index c8b8a3316c..4613c6200b 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -69,7 +69,7 @@ class VectorCallViewModel @AssistedInject constructor( } } - override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { + override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) { } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 57081afbe1..3ff40afdbb 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -19,21 +19,16 @@ package im.vector.riotx.features.call import android.content.ComponentName import android.content.Context import android.content.Intent -import android.graphics.drawable.Icon -import android.os.Build -import android.telecom.PhoneAccount -import android.telecom.PhoneAccountHandle -import android.telecom.TelecomManager +import android.content.ServiceConnection +import android.os.IBinder import androidx.core.content.ContextCompat import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent -import im.vector.riotx.BuildConfig -import im.vector.riotx.R +import im.vector.riotx.core.di.ActiveSessionHolder import im.vector.riotx.features.call.service.CallHeadsUpService -import im.vector.riotx.features.call.telecom.VectorConnectionService import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.DefaultVideoDecoderFactory @@ -62,8 +57,9 @@ import javax.inject.Singleton */ @Singleton class WebRtcPeerConnectionManager @Inject constructor( - private val context: Context -) : CallsListener { + private val context: Context, + private val sessionHolder: ActiveSessionHolder + ) : CallsListener { interface Listener { fun addLocalIceCandidate(candidates: IceCandidate) @@ -74,27 +70,8 @@ class WebRtcPeerConnectionManager @Inject constructor( fun sendOffer(sessionDescription: SessionDescription) } - var phoneAccountHandle: PhoneAccountHandle? = null var localMediaStream: MediaStream? = null - init { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - val componentName = ComponentName(BuildConfig.APPLICATION_ID, VectorConnectionService::class.java.name) - val appName = context.getString(R.string.app_name) - phoneAccountHandle = PhoneAccountHandle(componentName, appName) - val phoneAccount = PhoneAccount.Builder(phoneAccountHandle, BuildConfig.APPLICATION_ID) - .setIcon(Icon.createWithResource(context, R.drawable.riotx_logo)) - .setCapabilities(PhoneAccount.CAPABILITY_SELF_MANAGED) - .setCapabilities(PhoneAccount.CAPABILITY_VIDEO_CALLING) - .setCapabilities(PhoneAccount.CAPABILITY_CALL_SUBJECT) - .build() - ContextCompat.getSystemService(context, TelecomManager::class.java) - ?.registerPhoneAccount(phoneAccount) - } else { - // ignore? - } - } - var listener: Listener? = null // *Comments copied from webrtc demo app* @@ -121,6 +98,17 @@ class WebRtcPeerConnectionManager @Inject constructor( var localSurfaceRenderer: WeakReference? = null var remoteSurfaceRenderer: WeakReference? = null + var callHeadsUpService: CallHeadsUpService? = null + + private val serviceConnection = object : ServiceConnection { + override fun onServiceDisconnected(name: ComponentName?) { + } + + override fun onServiceConnected(name: ComponentName?, service: IBinder?) { + callHeadsUpService = (service as? CallHeadsUpService.CallHeadsUpServiceBinder)?.getService() + } + } + fun createPeerConnectionFactory() { executor.execute { if (peerConnectionFactory == null) { @@ -331,6 +319,7 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnectionFactory?.stopAecDump() peerConnectionFactory = null } + context.stopService(Intent(context, CallHeadsUpService::class.java)) } companion object { @@ -356,27 +345,20 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { - val callHeadsUpServiceIntent = Intent(context, CallHeadsUpService::class.java) + fun startOutgoingCall(context: Context, signalingRoomId: String, participantUserId: String, isVideoCall: Boolean) { + startHeadsUpService(signalingRoomId, sessionHolder.getActiveSession().myUserId, false, isVideoCall) + context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, isVideoCall)) + } + + override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) { + startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo()) + } + + private fun startHeadsUpService(roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean) { + val callHeadsUpServiceIntent = CallHeadsUpService.newInstance(context, roomId, participantUserId, isIncomingCall, isVideoCall) ContextCompat.startForegroundService(context, callHeadsUpServiceIntent) - /* - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - ContextCompat.getSystemService(context, TelecomManager::class.java)?.let { telecomManager -> - phoneAccountHandle?.let { phoneAccountHandle -> - telecomManager.addNewIncomingCall( - phoneAccountHandle, - Bundle().apply { - putString("MX_CALL_ROOM_ID", signalingRoomId) - putString("MX_CALL_CALL_ID", callInviteContent.callId) - putParcelable(TelecomManager.EXTRA_PHONE_ACCOUNT_HANDLE, phoneAccountHandle) - putInt(TelecomManager.EXTRA_INCOMING_VIDEO_STATE, VideoProfile.STATE_BIDIRECTIONAL) - putInt(TelecomManager.EXTRA_START_CALL_WITH_VIDEO_STATE, VideoProfile.STATE_BIDIRECTIONAL) - } - ) - } - } - } - */ + + context.bindService(Intent(context, CallHeadsUpService::class.java), serviceConnection, 0) } override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt index 9769724944..381975a2ee 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt @@ -21,7 +21,8 @@ import kotlinx.android.parcel.Parcelize @Parcelize data class CallHeadsUpServiceArgs( - val callerDisplayName: String, + val roomId: String, + val participantUserId: String, val isIncomingCall: Boolean, val isVideoCall: Boolean ) : Parcelable diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index efa045ddd2..b31ab35085 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -127,7 +127,7 @@ import im.vector.riotx.features.attachments.ContactAttachment import im.vector.riotx.features.attachments.preview.AttachmentsPreviewActivity import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs import im.vector.riotx.features.attachments.toGroupedContentAttachmentData -import im.vector.riotx.features.call.service.CallHeadsUpService +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.command.Command import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity import im.vector.riotx.features.crypto.util.toImageRes @@ -197,7 +197,8 @@ class RoomDetailFragment @Inject constructor( val roomDetailViewModelFactory: RoomDetailViewModel.Factory, private val eventHtmlRenderer: EventHtmlRenderer, private val vectorPreferences: VectorPreferences, - private val colorProvider: ColorProvider) : + private val colorProvider: ColorProvider, + private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager) : VectorBaseFragment(), TimelineEventController.Callback, VectorInviteView.Callback, @@ -484,14 +485,10 @@ class RoomDetailFragment @Inject constructor( roomDetailViewModel.handle(RoomDetailAction.ResendAll) return true } - if (item.itemId == R.id.voip_call) { - /* - VectorCallActivity.newIntent(requireContext(), roomDetailArgs.roomId).let { - startActivity(it) + if (item.itemId == R.id.voice_call || item.itemId == R.id.video_call) { + roomDetailViewModel.getOtherUserIds()?.firstOrNull()?.let { + webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) } - */ - val callHeadsUpServiceIntent = Intent(requireContext(), CallHeadsUpService::class.java) - ContextCompat.startForegroundService(requireContext(), callHeadsUpServiceIntent) return true } return super.onOptionsItemSelected(item) diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index 49a469847b..7e33d69345 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -213,6 +213,8 @@ class RoomDetailViewModel @AssistedInject constructor( } } + fun getOtherUserIds() = room.roomSummary()?.otherMemberIds + override fun handle(action: RoomDetailAction) { when (action) { is RoomDetailAction.UserIsTyping -> handleUserIsTyping(action) @@ -366,7 +368,7 @@ class RoomDetailViewModel @AssistedInject constructor( } fun isMenuItemVisible(@IdRes itemId: Int) = when (itemId) { - R.id.clear_message_queue -> + R.id.clear_message_queue -> /* For now always disable on production, worker cancellation is not working properly */ timeline.pendingEventCount() > 0 && vectorPreferences.developerMode() R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index 8e271a0285..ce20c59290 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -2,6 +2,13 @@ + #FFF8E3 #22262E + + #000000 + #000000 + #000000 + + + @android:color/transparent + @android:color/transparent + @android:color/transparent \ No newline at end of file From 743ace7e606a3556d60c2ecfb5e9b821d7d48917 Mon Sep 17 00:00:00 2001 From: onurays Date: Mon, 25 May 2020 23:28:30 +0300 Subject: [PATCH 08/83] Move voip responsibilities from views to WebRtcPeerConnectionManager. --- .../android/api/session/call/CallService.kt | 5 + .../session/call/DefaultCallService.kt | 10 ++ .../riotx/features/call/VectorCallActivity.kt | 44 +++-- .../features/call/VectorCallViewModel.kt | 33 +--- .../call/WebRtcPeerConnectionManager.kt | 150 +++++++++++------- .../features/call/telecom/CallConnection.kt | 12 +- 6 files changed, 139 insertions(+), 115 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt index bb5a81907e..554c00ab05 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -46,6 +46,11 @@ interface CallService { */ fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) + /** + * Send a hangup event + */ + fun sendHangup(callId: String, roomId: String) + fun addCallListener(listener: CallsListener) fun removeCallListener(listener: CallsListener) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index fba50a0c37..abc17cf89c 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -127,6 +127,16 @@ internal class DefaultCallService @Inject constructor( override fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) { } + override fun sendHangup(callId: String, roomId: String) { + val eventContent = CallHangupContent( + callId = callId, + version = 0 + ) + createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) + } + } + override fun addCallListener(listener: CallsListener) { if (!callListeners.contains(listener)) callListeners.add(listener) } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index fef5aa0647..3f6b38957b 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -38,17 +38,14 @@ import im.vector.riotx.core.utils.checkPermissions import io.reactivex.android.schedulers.AndroidSchedulers import io.reactivex.subjects.PublishSubject import kotlinx.android.parcel.Parcelize -import org.webrtc.Camera1Enumerator -import org.webrtc.Camera2Enumerator +import kotlinx.android.synthetic.main.activity_call.* import org.webrtc.EglBase import org.webrtc.IceCandidate import org.webrtc.MediaStream -import org.webrtc.PeerConnection import org.webrtc.RendererCommon import org.webrtc.SessionDescription import org.webrtc.SurfaceViewRenderer import org.webrtc.VideoTrack -import java.util.concurrent.TimeUnit import javax.inject.Inject @Parcelize @@ -69,6 +66,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } private val callViewModel: VectorCallViewModel by viewModel() + private lateinit var callArgs: CallArgs @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager @@ -114,10 +112,19 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + + if (intent.hasExtra(MvRx.KEY_ARG)) { + callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! + } else { + finish() + } + rootEglBase = EglUtils.rootEglBase ?: return Unit.also { finish() } + iv_end_call.setOnClickListener { callViewModel.handle(VectorCallViewActions.EndCall) } + callViewModel.viewEvents .observe() .observeOn(AndroidSchedulers.mainThread()) @@ -161,8 +168,9 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // setSwappedFeeds(true /* isSwappedFeeds */); if (isFirstCreation()) { - peerConnectionManager.createPeerConnectionFactory() + //peerConnectionManager.createPeerConnectionFactory() + /* val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) val frontCamera = cameraIterator.deviceNames ?.firstOrNull { cameraIterator.isFrontFacing(it) } @@ -170,19 +178,12 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis ?: return true val videoCapturer = cameraIterator.createCapturer(frontCamera, null) - val iceServers = ArrayList().apply { - listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { - add( - PeerConnection.IceServer.builder(it) - .setUsername("xxxxx") - .setPassword("xxxxx") - .createIceServer() - ) - } - } + peerConnectionManager.createPeerConnection(videoCapturer, iceServers) - peerConnectionManager.startCall() + */ + + //peerConnectionManager.startCall() } // PeerConnectionFactory.initialize(PeerConnectionFactory // .InitializationOptions.builder(applicationContext) @@ -322,15 +323,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // Timber.v("## VOIP onCreateFailure $p0") // } // }, constraints) - iceCandidateSource - .buffer(400, TimeUnit.MILLISECONDS) - .subscribe { - // omit empty :/ - if (it.isNotEmpty()) { - callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) - } - } - .disposeOnDestroy() peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) return false @@ -433,6 +425,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } override fun sendOffer(sessionDescription: SessionDescription) { - callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 4613c6200b..d6270ad936 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -16,7 +16,6 @@ package im.vector.riotx.features.call -import androidx.lifecycle.viewModelScope import com.airbnb.mvrx.MvRxState import com.airbnb.mvrx.MvRxViewModelFactory import com.airbnb.mvrx.ViewModelContext @@ -27,16 +26,10 @@ import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent -import im.vector.matrix.android.internal.util.awaitCallback import im.vector.riotx.core.extensions.exhaustive import im.vector.riotx.core.platform.VectorViewEvents import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.platform.VectorViewModelAction -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import org.webrtc.IceCandidate -import org.webrtc.SessionDescription -import java.util.UUID data class VectorCallViewState( val callId: String? = null, @@ -45,8 +38,7 @@ data class VectorCallViewState( sealed class VectorCallViewActions : VectorViewModelAction { - data class SendOffer(val sdp: SessionDescription) : VectorCallViewActions() - data class AddLocalIceCandidate(val iceCandidates: List) : VectorCallViewActions() + object EndCall : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { @@ -57,7 +49,8 @@ sealed class VectorCallViewEvents : VectorViewEvents { class VectorCallViewModel @AssistedInject constructor( @Assisted initialState: VectorCallViewState, - val session: Session + val session: Session, + val webRtcPeerConnectionManager: WebRtcPeerConnectionManager ) : VectorViewModel(initialState) { private val callServiceListener: CallsListener = object : CallsListener { @@ -90,25 +83,9 @@ class VectorCallViewModel @AssistedInject constructor( super.onCleared() } - override fun handle(action: VectorCallViewActions) = withState { state -> + override fun handle(action: VectorCallViewActions) = withState { when (action) { - is VectorCallViewActions.SendOffer -> { - viewModelScope.launch(Dispatchers.IO) { - awaitCallback { - val callId = state.callId ?: UUID.randomUUID().toString().also { - setState { - copy(callId = it) - } - } - session.callService().sendOfferSdp(callId, state.roomId, action.sdp, it) - } - } - } - is VectorCallViewActions.AddLocalIceCandidate -> { - viewModelScope.launch { - session.callService().sendLocalIceCandidates(state.callId ?: "", state.roomId, action.iceCandidates) - } - } + VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() }.exhaustive } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 3ff40afdbb..b466ea9f11 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -22,6 +22,7 @@ import android.content.Intent import android.content.ServiceConnection import android.os.IBinder import androidx.core.content.ContextCompat +import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent @@ -29,6 +30,8 @@ import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.riotx.core.di.ActiveSessionHolder import im.vector.riotx.features.call.service.CallHeadsUpService +import io.reactivex.disposables.Disposable +import io.reactivex.subjects.PublishSubject import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.DefaultVideoDecoderFactory @@ -47,7 +50,9 @@ import org.webrtc.VideoSource import org.webrtc.VideoTrack import timber.log.Timber import java.lang.ref.WeakReference +import java.util.UUID import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import javax.inject.Inject import javax.inject.Singleton @@ -59,7 +64,7 @@ import javax.inject.Singleton class WebRtcPeerConnectionManager @Inject constructor( private val context: Context, private val sessionHolder: ActiveSessionHolder - ) : CallsListener { +) : CallsListener { interface Listener { fun addLocalIceCandidate(candidates: IceCandidate) @@ -98,8 +103,15 @@ class WebRtcPeerConnectionManager @Inject constructor( var localSurfaceRenderer: WeakReference? = null var remoteSurfaceRenderer: WeakReference? = null + private val iceCandidateSource: PublishSubject = PublishSubject.create() + private var iceCandidateDisposable: Disposable? = null + var callHeadsUpService: CallHeadsUpService? = null + private var callId: String? = null + private var signalingRoomId: String? = null + private var participantUserId: String? = null + private val serviceConnection = object : ServiceConnection { override fun onServiceDisconnected(name: ComponentName?) { } @@ -109,7 +121,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun createPeerConnectionFactory() { + private fun createPeerConnectionFactory() { executor.execute { if (peerConnectionFactory == null) { Timber.v("## VOIP createPeerConnectionFactory") @@ -142,7 +154,56 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun createPeerConnection(videoCapturer: VideoCapturer, iceServers: List) { + private fun createPeerConnection() { + val iceServers = ArrayList().apply { + listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { + add( + PeerConnection.IceServer.builder(it) + .setUsername("xxxxx") + .setPassword("xxxxx") + .createIceServer() + ) + } + } + Timber.v("## VOIP creating peer connection... ") + peerConnection = peerConnectionFactory?.createPeerConnection( + iceServers, + object : PeerConnectionObserverAdapter() { + override fun onIceCandidate(p0: IceCandidate?) { + Timber.v("## VOIP onIceCandidate local $p0") + p0?.let { iceCandidateSource.onNext(it) } + } + + override fun onAddStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onAddStream remote $mediaStream") + mediaStream?.videoTracks?.firstOrNull()?.let { + listener?.addRemoteVideoTrack(it) + remoteVideoTrack = it + } + } + + override fun onRemoveStream(mediaStream: MediaStream?) { + mediaStream?.let { + listener?.removeRemoteVideoStream(it) + } + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + remoteVideoTrack = null + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { + listener?.onDisconnect() + } + } + } + ) + } + + // TODO REMOVE THIS FUNCTION + private fun createPeerConnection(videoCapturer: VideoCapturer) { executor.execute { Timber.v("## VOIP PeerConnectionFactory.createPeerConnection $peerConnectionFactory...") // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object @@ -183,55 +244,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // } // .disposeOnDestroy() - Timber.v("## VOIP creating peer connection... ") - peerConnection = peerConnectionFactory?.createPeerConnection( - iceServers, - object : PeerConnectionObserverAdapter() { - override fun onIceCandidate(p0: IceCandidate?) { - Timber.v("## VOIP onIceCandidate local $p0") - p0?.let { - // iceCandidateSource.onNext(it) - listener?.addLocalIceCandidate(it) - } - } - override fun onAddStream(mediaStream: MediaStream?) { - Timber.v("## VOIP onAddStream remote $mediaStream") - mediaStream?.videoTracks?.firstOrNull()?.let { - listener?.addRemoteVideoTrack(it) - remoteVideoTrack = it -// remoteSurfaceRenderer?.get()?.let { surface -> -// it.setEnabled(true) -// it.addSink(surface) -// } - } -// runOnUiThread { -// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> -// remoteVideoTrack = videoTrack -// remoteVideoTrack?.setEnabled(true) -// remoteVideoTrack?.addSink(fullscreenRenderer) -// } -// } - } - - override fun onRemoveStream(mediaStream: MediaStream?) { - mediaStream?.let { - listener?.removeRemoteVideoStream(it) - } - remoteSurfaceRenderer?.get()?.let { - remoteVideoTrack?.removeSink(it) - } - remoteVideoTrack = null - } - - override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { - Timber.v("## VOIP onIceConnectionChange $p0") - if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { - listener?.onDisconnect() - } - } - } - ) localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? localMediaStream?.addTrack(localVideoTrack) @@ -253,7 +266,22 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun startCall() { + private fun startCall() { + createPeerConnectionFactory() + createPeerConnection() + + iceCandidateDisposable = iceCandidateSource + .buffer(400, TimeUnit.MILLISECONDS) + .subscribe { + // omit empty :/ + if (it.isNotEmpty()) { + sessionHolder + .getActiveSession() + .callService() + .sendLocalIceCandidates(callId ?: "", signalingRoomId ?: "", it) + } + } + executor.execute { val constraints = MediaConstraints() constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) @@ -273,8 +301,8 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP onCreateSuccess $sessionDescription") peerConnection?.setLocalDescription(object : SdpObserverAdapter() { override fun onSetSuccess() { - listener?.sendOffer(sessionDescription) - // callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + callId = UUID.randomUUID().toString() + sessionHolder.getActiveSession().callService().sendOfferSdp(callId!!, signalingRoomId!!, sessionDescription, object : MatrixCallback {}) } }, sessionDescription) } @@ -319,6 +347,7 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnectionFactory?.stopAecDump() peerConnectionFactory = null } + iceCandidateDisposable?.dispose() context.stopService(Intent(context, CallHeadsUpService::class.java)) } @@ -346,12 +375,18 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun startOutgoingCall(context: Context, signalingRoomId: String, participantUserId: String, isVideoCall: Boolean) { + this.signalingRoomId = signalingRoomId + this.participantUserId = participantUserId startHeadsUpService(signalingRoomId, sessionHolder.getActiveSession().myUserId, false, isVideoCall) context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, isVideoCall)) + + startCall() } override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) { startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo()) + + startCall() } private fun startHeadsUpService(roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean) { @@ -361,6 +396,13 @@ class WebRtcPeerConnectionManager @Inject constructor( context.bindService(Intent(context, CallHeadsUpService::class.java), serviceConnection, 0) } + fun endCall() { + if (callId != null && signalingRoomId != null) { + sessionHolder.getActiveSession().callService().sendHangup(callId!!, signalingRoomId!!) + close() + } + } + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { } diff --git a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt index 27bdf24570..6f0c121b3d 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt @@ -21,14 +21,10 @@ import android.os.Build import android.telecom.Connection import android.telecom.DisconnectCause import androidx.annotation.RequiresApi -import im.vector.riotx.features.call.VectorCallViewActions import im.vector.riotx.features.call.VectorCallViewModel import im.vector.riotx.features.call.WebRtcPeerConnectionManager -import org.webrtc.Camera1Enumerator -import org.webrtc.Camera2Enumerator import org.webrtc.IceCandidate import org.webrtc.MediaStream -import org.webrtc.PeerConnection import org.webrtc.SessionDescription import org.webrtc.VideoTrack import timber.log.Timber @@ -91,7 +87,8 @@ import javax.inject.Inject } private fun startCall() { - peerConnectionManager.createPeerConnectionFactory() + /* + //peerConnectionManager.createPeerConnectionFactory() peerConnectionManager.listener = this val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) @@ -113,7 +110,8 @@ import javax.inject.Inject } peerConnectionManager.createPeerConnection(videoCapturer, iceServers) - peerConnectionManager.startCall() + //peerConnectionManager.startCall() + */ } override fun addLocalIceCandidate(candidates: IceCandidate) { @@ -129,6 +127,6 @@ import javax.inject.Inject } override fun sendOffer(sessionDescription: SessionDescription) { - callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } } From f50f81d32156d9d9386ab5c5f6342731e5261e78 Mon Sep 17 00:00:00 2001 From: onurays Date: Tue, 26 May 2020 14:30:22 +0300 Subject: [PATCH 09/83] Implement rejecting incoming call. --- .../call/WebRtcPeerConnectionManager.kt | 193 +++++++++--------- .../call/service/CallHeadsUpActionReceiver.kt | 18 +- 2 files changed, 111 insertions(+), 100 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index b466ea9f11..053ac055a3 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -122,39 +122,37 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun createPeerConnectionFactory() { - executor.execute { - if (peerConnectionFactory == null) { - Timber.v("## VOIP createPeerConnectionFactory") - val eglBaseContext = rootEglBase?.eglBaseContext ?: return@execute Unit.also { - Timber.e("## VOIP No EGL BASE") - } - - Timber.v("## VOIP PeerConnectionFactory.initialize") - PeerConnectionFactory.initialize(PeerConnectionFactory - .InitializationOptions.builder(context.applicationContext) - .createInitializationOptions() - ) - - val options = PeerConnectionFactory.Options() - val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( - eglBaseContext, - /* enableIntelVp8Encoder */ - true, - /* enableH264HighProfile */ - true) - val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) - - Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") - peerConnectionFactory = PeerConnectionFactory.builder() - .setOptions(options) - .setVideoEncoderFactory(defaultVideoEncoderFactory) - .setVideoDecoderFactory(defaultVideoDecoderFactory) - .createPeerConnectionFactory() + if (peerConnectionFactory == null) { + Timber.v("## VOIP createPeerConnectionFactory") + val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also { + Timber.e("## VOIP No EGL BASE") } + + Timber.v("## VOIP PeerConnectionFactory.initialize") + PeerConnectionFactory.initialize(PeerConnectionFactory + .InitializationOptions.builder(context.applicationContext) + .createInitializationOptions() + ) + + val options = PeerConnectionFactory.Options() + val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( + eglBaseContext, + /* enableIntelVp8Encoder */ + true, + /* enableH264HighProfile */ + true) + val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) + + Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") + peerConnectionFactory = PeerConnectionFactory.builder() + .setOptions(options) + .setVideoEncoderFactory(defaultVideoEncoderFactory) + .setVideoDecoderFactory(defaultVideoDecoderFactory) + .createPeerConnectionFactory() } } - private fun createPeerConnection() { + private fun createPeerConnection(observer: PeerConnectionObserverAdapter) { val iceServers = ArrayList().apply { listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { add( @@ -166,40 +164,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } } Timber.v("## VOIP creating peer connection... ") - peerConnection = peerConnectionFactory?.createPeerConnection( - iceServers, - object : PeerConnectionObserverAdapter() { - override fun onIceCandidate(p0: IceCandidate?) { - Timber.v("## VOIP onIceCandidate local $p0") - p0?.let { iceCandidateSource.onNext(it) } - } - - override fun onAddStream(mediaStream: MediaStream?) { - Timber.v("## VOIP onAddStream remote $mediaStream") - mediaStream?.videoTracks?.firstOrNull()?.let { - listener?.addRemoteVideoTrack(it) - remoteVideoTrack = it - } - } - - override fun onRemoveStream(mediaStream: MediaStream?) { - mediaStream?.let { - listener?.removeRemoteVideoStream(it) - } - remoteSurfaceRenderer?.get()?.let { - remoteVideoTrack?.removeSink(it) - } - remoteVideoTrack = null - } - - override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { - Timber.v("## VOIP onIceConnectionChange $p0") - if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { - listener?.onDisconnect() - } - } - } - ) + peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, observer) } // TODO REMOVE THIS FUNCTION @@ -244,8 +209,6 @@ class WebRtcPeerConnectionManager @Inject constructor( // } // .disposeOnDestroy() - - localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? localMediaStream?.addTrack(localVideoTrack) localMediaStream?.addTrack(audioTrack) @@ -260,6 +223,8 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun answerReceived(callId: String, answerSdp: SessionDescription) { + this.callId = callId + executor.execute { Timber.v("## answerReceived $callId") peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, answerSdp) @@ -268,50 +233,84 @@ class WebRtcPeerConnectionManager @Inject constructor( private fun startCall() { createPeerConnectionFactory() - createPeerConnection() + createPeerConnection(object : PeerConnectionObserverAdapter() { + override fun onIceCandidate(p0: IceCandidate?) { + Timber.v("## VOIP onIceCandidate local $p0") + p0?.let { iceCandidateSource.onNext(it) } + } + + override fun onAddStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onAddStream remote $mediaStream") + mediaStream?.videoTracks?.firstOrNull()?.let { + listener?.addRemoteVideoTrack(it) + remoteVideoTrack = it + } + } + + override fun onRemoveStream(mediaStream: MediaStream?) { + mediaStream?.let { + listener?.removeRemoteVideoStream(it) + } + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + remoteVideoTrack = null + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { + listener?.onDisconnect() + } + } + }) iceCandidateDisposable = iceCandidateSource .buffer(400, TimeUnit.MILLISECONDS) .subscribe { // omit empty :/ if (it.isNotEmpty()) { + Timber.v("## Sending local ice candidates to callId: $callId roomId: $signalingRoomId") sessionHolder .getActiveSession() .callService() .sendLocalIceCandidates(callId ?: "", signalingRoomId ?: "", it) } } + } - executor.execute { - val constraints = MediaConstraints() - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) + private fun sendSdpOffer() { + val constraints = MediaConstraints() + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) - Timber.v("## VOIP creating offer...") - peerConnection?.createOffer(object : SdpObserver { - override fun onSetFailure(p0: String?) { - Timber.v("## VOIP onSetFailure $p0") - } + Timber.v("## VOIP creating offer...") + peerConnection?.createOffer(object : SdpObserver { + override fun onSetFailure(p0: String?) { + Timber.v("## VOIP onSetFailure $p0") + } - override fun onSetSuccess() { - Timber.v("## VOIP onSetSuccess") - } + override fun onSetSuccess() { + Timber.v("## VOIP onSetSuccess") + } - override fun onCreateSuccess(sessionDescription: SessionDescription) { - Timber.v("## VOIP onCreateSuccess $sessionDescription") - peerConnection?.setLocalDescription(object : SdpObserverAdapter() { - override fun onSetSuccess() { - callId = UUID.randomUUID().toString() - sessionHolder.getActiveSession().callService().sendOfferSdp(callId!!, signalingRoomId!!, sessionDescription, object : MatrixCallback {}) - } - }, sessionDescription) - } + override fun onCreateSuccess(sessionDescription: SessionDescription) { + Timber.v("## VOIP onCreateSuccess $sessionDescription will set local description") + peerConnection?.setLocalDescription(object : SdpObserverAdapter() { + override fun onSetSuccess() { + Timber.v("## setLocalDescription success") + callId = UUID.randomUUID().toString() + Timber.v("## sending offer to callId: $callId roomId: $signalingRoomId") + sessionHolder.getActiveSession().callService().sendOfferSdp(callId ?: "", signalingRoomId + ?: "", sessionDescription, object : MatrixCallback {}) + } + }, sessionDescription) + } - override fun onCreateFailure(p0: String?) { - Timber.v("## VOIP onCreateFailure $p0") - } - }, constraints) - } + override fun onCreateFailure(p0: String?) { + Timber.v("## VOIP onCreateFailure $p0") + } + }, constraints) } fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { @@ -339,7 +338,7 @@ class WebRtcPeerConnectionManager @Inject constructor( executor.execute { // Do not dispose peer connection (https://bugs.chromium.org/p/webrtc/issues/detail?id=7543) peerConnection?.close() - peerConnection?.removeStream(localMediaStream) + localMediaStream?.let { peerConnection?.removeStream(it) } peerConnection = null audioSource?.dispose() videoSource?.dispose() @@ -377,13 +376,19 @@ class WebRtcPeerConnectionManager @Inject constructor( fun startOutgoingCall(context: Context, signalingRoomId: String, participantUserId: String, isVideoCall: Boolean) { this.signalingRoomId = signalingRoomId this.participantUserId = participantUserId + startHeadsUpService(signalingRoomId, sessionHolder.getActiveSession().myUserId, false, isVideoCall) context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, isVideoCall)) startCall() + sendSdpOffer() } override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) { + this.callId = callInviteContent.callId + this.signalingRoomId = signalingRoomId + this.participantUserId = participantUserId + startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo()) startCall() @@ -399,8 +404,8 @@ class WebRtcPeerConnectionManager @Inject constructor( fun endCall() { if (callId != null && signalingRoomId != null) { sessionHolder.getActiveSession().callService().sendHangup(callId!!, signalingRoomId!!) - close() } + close() } override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt index 5f07866043..aee51ff422 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt @@ -19,11 +19,22 @@ package im.vector.riotx.features.call.service import android.content.BroadcastReceiver import android.content.Context import android.content.Intent +import im.vector.riotx.core.di.HasVectorInjector +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.settings.VectorLocale.context import timber.log.Timber class CallHeadsUpActionReceiver : BroadcastReceiver() { + private lateinit var peerConnectionManager: WebRtcPeerConnectionManager + + init { + val appContext = context.applicationContext + if (appContext is HasVectorInjector) { + peerConnectionManager = appContext.injector().webRtcPeerConnectionManager() + } + } + override fun onReceive(context: Context, intent: Intent?) { when (intent?.getIntExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, 0)) { CallHeadsUpService.CALL_ACTION_ANSWER -> onCallAnswerClicked() @@ -33,15 +44,10 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { private fun onCallRejectClicked() { Timber.d("onCallRejectClicked") - stopService() + peerConnectionManager.endCall() } private fun onCallAnswerClicked() { Timber.d("onCallAnswerClicked") } - - private fun stopService() { - context.sendBroadcast(Intent(Intent.ACTION_CLOSE_SYSTEM_DIALOGS)) - context.stopService(Intent(context, CallHeadsUpService::class.java)) - } } From 37c926d178793c3b5bf8fcd7595b7046c76299ca Mon Sep 17 00:00:00 2001 From: onurays Date: Tue, 26 May 2020 15:35:03 +0300 Subject: [PATCH 10/83] Attach local video renderers. --- .../riotx/features/call/VectorCallActivity.kt | 5 +- .../call/WebRtcPeerConnectionManager.kt | 47 +++++++++++++++++-- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 3f6b38957b..b485580b88 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -123,7 +123,10 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis finish() } - iv_end_call.setOnClickListener { callViewModel.handle(VectorCallViewActions.EndCall) } + iv_end_call.setOnClickListener { + callViewModel.handle(VectorCallViewActions.EndCall) + finish() + } callViewModel.viewEvents .observe() diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 053ac055a3..092b0c785f 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -23,6 +23,7 @@ import android.content.ServiceConnection import android.os.IBinder import androidx.core.content.ContextCompat import im.vector.matrix.android.api.MatrixCallback +import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent @@ -34,6 +35,8 @@ import io.reactivex.disposables.Disposable import io.reactivex.subjects.PublishSubject import org.webrtc.AudioSource import org.webrtc.AudioTrack +import org.webrtc.Camera1Enumerator +import org.webrtc.Camera2Enumerator import org.webrtc.DefaultVideoDecoderFactory import org.webrtc.DefaultVideoEncoderFactory import org.webrtc.IceCandidate @@ -111,6 +114,7 @@ class WebRtcPeerConnectionManager @Inject constructor( private var callId: String? = null private var signalingRoomId: String? = null private var participantUserId: String? = null + private var isVideoCall: Boolean? = null private val serviceConnection = object : ServiceConnection { override fun onServiceDisconnected(name: ComponentName?) { @@ -314,11 +318,43 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { + audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + + localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? + + if (isVideoCall == true) { + val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) + val frontCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isFrontFacing(it) } + ?: cameraIterator.deviceNames?.first() + + val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + + videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + Timber.v("## VOIP Local video source created") + videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) + videoCapturer.startCapture(1280, 720, 30) + localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { + Timber.v("## VOIP Local video track created") + localSurfaceRenderer?.get()?.let { surface -> + it.addSink(surface) + } + } + localMediaStream?.addTrack(localVideoTrack) + } + localVideoTrack?.addSink(localViewRenderer) remoteVideoTrack?.let { it.setEnabled(true) it.addSink(remoteViewRenderer) } + localMediaStream?.addTrack(audioTrack) + + Timber.v("## VOIP add local stream to peer connection") + peerConnection?.addStream(localMediaStream) + localSurfaceRenderer = WeakReference(localViewRenderer) remoteSurfaceRenderer = WeakReference(remoteViewRenderer) } @@ -337,12 +373,13 @@ class WebRtcPeerConnectionManager @Inject constructor( fun close() { executor.execute { // Do not dispose peer connection (https://bugs.chromium.org/p/webrtc/issues/detail?id=7543) - peerConnection?.close() + tryThis { audioSource?.dispose() } + tryThis { videoSource?.dispose() } + tryThis { videoCapturer?.stopCapture() } + tryThis { videoCapturer?.dispose() } localMediaStream?.let { peerConnection?.removeStream(it) } + peerConnection?.close() peerConnection = null - audioSource?.dispose() - videoSource?.dispose() - videoCapturer?.dispose() peerConnectionFactory?.stopAecDump() peerConnectionFactory = null } @@ -376,6 +413,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun startOutgoingCall(context: Context, signalingRoomId: String, participantUserId: String, isVideoCall: Boolean) { this.signalingRoomId = signalingRoomId this.participantUserId = participantUserId + this.isVideoCall = isVideoCall startHeadsUpService(signalingRoomId, sessionHolder.getActiveSession().myUserId, false, isVideoCall) context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, isVideoCall)) @@ -388,6 +426,7 @@ class WebRtcPeerConnectionManager @Inject constructor( this.callId = callInviteContent.callId this.signalingRoomId = signalingRoomId this.participantUserId = participantUserId + this.isVideoCall = callInviteContent.isVideo() startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo()) From 54b154f85f89e8a51ed9a401f4138d662104c681 Mon Sep 17 00:00:00 2001 From: onurays Date: Tue, 26 May 2020 17:42:44 +0300 Subject: [PATCH 11/83] Send sdp to remote party when answer is received. --- .../riotx/features/call/VectorCallActivity.kt | 23 ++++++++++-- .../features/call/VectorCallViewModel.kt | 4 +-- .../call/WebRtcPeerConnectionManager.kt | 36 +++++++++++++------ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index b485580b88..ba32959973 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -17,10 +17,13 @@ package im.vector.riotx.features.call import android.app.KeyguardManager +import android.content.ComponentName import android.content.Context import android.content.Intent +import android.content.ServiceConnection import android.os.Build import android.os.Bundle +import android.os.IBinder import android.os.Parcelable import android.view.View import android.view.Window @@ -35,6 +38,7 @@ import im.vector.riotx.core.platform.VectorBaseActivity import im.vector.riotx.core.utils.PERMISSIONS_FOR_VIDEO_IP_CALL import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions +import im.vector.riotx.features.call.service.CallHeadsUpService import io.reactivex.android.schedulers.AndroidSchedulers import io.reactivex.subjects.PublishSubject import kotlinx.android.parcel.Parcelize @@ -88,6 +92,19 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis private val iceCandidateSource: PublishSubject = PublishSubject.create() + + + var callHeadsUpService: CallHeadsUpService? = null + private val serviceConnection = object : ServiceConnection { + override fun onServiceDisconnected(name: ComponentName?) { + finish() + } + + override fun onServiceConnected(name: ComponentName?, service: IBinder?) { + callHeadsUpService = (service as? CallHeadsUpService.CallHeadsUpServiceBinder)?.getService() + } + } + override fun doBeforeSetContentView() { // Set window styles for fullscreen-window size. Needs to be done before adding content. requestWindowFeature(Window.FEATURE_NO_TITLE) @@ -113,6 +130,8 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + bindService(Intent(this, CallHeadsUpService::class.java), serviceConnection, 0) + if (intent.hasExtra(MvRx.KEY_ARG)) { callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! } else { @@ -340,8 +359,8 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis private fun handleViewEvents(event: VectorCallViewEvents?) { when (event) { is VectorCallViewEvents.CallAnswered -> { - val sdp = SessionDescription(SessionDescription.Type.ANSWER, event.content.answer.sdp) - peerConnectionManager.answerReceived("", sdp) + //val sdp = SessionDescription(SessionDescription.Type.ANSWER, event.content.answer.sdp) + //peerConnectionManager.answerReceived("", sdp) // peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) } is VectorCallViewEvents.CallHangup -> { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index d6270ad936..07836e64f2 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -103,8 +103,8 @@ class VectorCallViewModel @AssistedInject constructor( } override fun initialState(viewModelContext: ViewModelContext): VectorCallViewState? { - val args: CallArgs = viewModelContext.args() - return VectorCallViewState(roomId = args.roomId) + //val args: CallArgs = viewModelContext.args() + return VectorCallViewState() } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 092b0c785f..e50f2bcc8f 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -94,6 +94,9 @@ class WebRtcPeerConnectionManager @Inject constructor( private var peerConnection: PeerConnection? = null + private var localViewRenderer: SurfaceViewRenderer? = null + private var remoteViewRenderer: SurfaceViewRenderer? = null + private var remoteVideoTrack: VideoTrack? = null private var localVideoTrack: VideoTrack? = null @@ -185,7 +188,6 @@ class WebRtcPeerConnectionManager @Inject constructor( localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { Timber.v("## VOIP Local video track created") - listener?.addLocalVideoTrack(it) // localSurfaceRenderer?.get()?.let { surface -> // // it.addSink(surface) // // } @@ -226,15 +228,6 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun answerReceived(callId: String, answerSdp: SessionDescription) { - this.callId = callId - - executor.execute { - Timber.v("## answerReceived $callId") - peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, answerSdp) - } - } - private fun startCall() { createPeerConnectionFactory() createPeerConnection(object : PeerConnectionObserverAdapter() { @@ -246,8 +239,16 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onAddStream(mediaStream: MediaStream?) { Timber.v("## VOIP onAddStream remote $mediaStream") mediaStream?.videoTracks?.firstOrNull()?.let { - listener?.addRemoteVideoTrack(it) remoteVideoTrack = it + remoteSurfaceRenderer?.get()?.let { surface -> + it.setEnabled(true) + it.addSink(surface) + } + mediaStream.videoTracks?.firstOrNull()?.let { videoTrack -> + remoteVideoTrack = videoTrack + remoteVideoTrack?.setEnabled(true) + remoteVideoTrack?.addSink(remoteViewRenderer) + } } } @@ -318,9 +319,14 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { + this.localViewRenderer = localViewRenderer + this.remoteViewRenderer = remoteViewRenderer audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + localViewRenderer.setMirror(true) + localVideoTrack?.addSink(localViewRenderer) + localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? if (isVideoCall == true) { @@ -429,6 +435,7 @@ class WebRtcPeerConnectionManager @Inject constructor( this.isVideoCall = callInviteContent.isVideo() startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo()) + context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, callInviteContent.isVideo())) startCall() } @@ -448,6 +455,13 @@ class WebRtcPeerConnectionManager @Inject constructor( } override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + this.callId = callAnswerContent.callId + + executor.execute { + Timber.v("## answerReceived $callId") + val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) + peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) + } } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { From 3d03bf6f91dac97c1c173a3a3a2cdd05fb39e90d Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 12:03:44 +0200 Subject: [PATCH 12/83] Add Javadoc to the model --- .../room/model/call/CallAnswerContent.kt | 22 ++++++++++++++-- .../room/model/call/CallCandidatesContent.kt | 26 +++++++++++++++++-- .../room/model/call/CallHangupContent.kt | 18 ++++++++++++- .../room/model/call/CallInviteContent.kt | 21 ++++++++------- .../session/call/DefaultCallService.kt | 3 ++- 5 files changed, 74 insertions(+), 16 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt index 24eb68bd78..f9f9cf15b2 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt @@ -19,16 +19,34 @@ package im.vector.matrix.android.api.session.room.model.call import com.squareup.moshi.Json import com.squareup.moshi.JsonClass +/** + * This event is sent by the callee when they wish to answer the call. + */ @JsonClass(generateAdapter = true) data class CallAnswerContent( + /** + * Required. The ID of the call this event relates to. + */ @Json(name = "call_id") val callId: String, - @Json(name = "version") val version: Int, - @Json(name = "answer") val answer: Answer + /** + * Required. The session description object + */ + @Json(name = "answer") val answer: Answer, + /** + * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. + */ + @Json(name = "version") val version: Int ) { @JsonClass(generateAdapter = true) data class Answer( + /** + * Required. The type of session description. Must be 'answer'. + */ @Json(name = "type") val type: String, + /** + * Required. The SDP text of the session description. + */ @Json(name = "sdp") val sdp: String ) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt index 5fb4db84a3..1375776919 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt @@ -19,17 +19,39 @@ package im.vector.matrix.android.api.session.room.model.call import com.squareup.moshi.Json import com.squareup.moshi.JsonClass +/** + * This event is sent by callers after sending an invite and by the callee after answering. + * Its purpose is to give the other party additional ICE candidates to try using to communicate. + */ @JsonClass(generateAdapter = true) data class CallCandidatesContent( + /** + * Required. The ID of the call this event relates to. + */ @Json(name = "call_id") val callId: String, - @Json(name = "version") val version: Int, - @Json(name = "candidates") val candidates: List = emptyList() + /** + * Required. Array of objects describing the candidates. + */ + @Json(name = "candidates") val candidates: List = emptyList(), + /** + * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. + */ + @Json(name = "version") val version: Int ) { @JsonClass(generateAdapter = true) data class Candidate( + /** + * Required. The SDP media type this candidate is intended for. + */ @Json(name = "sdpMid") val sdpMid: String, + /** + * Required. The index of the SDP 'm' line this candidate is intended for. + */ @Json(name = "sdpMLineIndex") val sdpMLineIndex: String, + /** + * Required. The SDP 'a' line of the candidate. + */ @Json(name = "candidate") val candidate: String ) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt index eda2486aa2..c17efd2559 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt @@ -19,8 +19,24 @@ package im.vector.matrix.android.api.session.room.model.call import com.squareup.moshi.Json import com.squareup.moshi.JsonClass +/** + * Sent by either party to signal their termination of the call. This can be sent either once + * the call has been established or before to abort the call. + */ @JsonClass(generateAdapter = true) data class CallHangupContent( + /** + * Required. The ID of the call this event relates to. + */ @Json(name = "call_id") val callId: String, - @Json(name = "version") val version: Int + /** + * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + */ + @Json(name = "version") val version: Int, + /** + * Optional error reason for the hangup. This should not be provided when the user naturally ends or rejects the call. + * When there was an error in the call negotiation, this should be `ice_failed` for when ICE negotiation fails + * or `invite_timeout` for when the other party did not answer in time. One of: ["ice_failed", "invite_timeout"] + */ + @Json(name = "reason") val reason: String? ) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt index 44c3cfbf0b..cd0df70c96 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt @@ -19,37 +19,38 @@ package im.vector.matrix.android.api.session.room.model.call import com.squareup.moshi.Json import com.squareup.moshi.JsonClass +/** + * This event is sent by the caller when they wish to establish a call. + */ @JsonClass(generateAdapter = true) data class CallInviteContent( - /** - * A unique identifier for the call. + * Required. A unique identifier for the call. */ @Json(name = "call_id") val callId: String?, /** - * The session description object + * Required. The session description object */ @Json(name = "version") val version: Int?, /** - * The version of the VoIP specification this message adheres to. This specification is version 0. + * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ @Json(name = "lifetime") val lifetime: Int?, /** - * The time in milliseconds that the invite is valid for. - * Once the invite age exceeds this value, clients should discard it. - * They should also no longer show the call as awaiting an answer in the UI. + * Required. The time in milliseconds that the invite is valid for. + * Once the invite age exceeds this value, clients should discard it. + * They should also no longer show the call as awaiting an answer in the UI. */ @Json(name = "offer") val offer: Offer? ) { - @JsonClass(generateAdapter = true) data class Offer( /** - * The type of session description (offer, answer) + * Required. The type of session description. Must be 'offer'. */ @Json(name = "type") val type: String?, /** - * The SDP text of the session description. + * Required. The SDP text of the session description. */ @Json(name = "sdp") val sdp: String? ) { diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index abc17cf89c..7b0731d8a8 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -130,7 +130,8 @@ internal class DefaultCallService @Inject constructor( override fun sendHangup(callId: String, roomId: String) { val eventContent = CallHangupContent( callId = callId, - version = 0 + version = 0, + reason = null ) createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = eventContent.toContent()).let { event -> roomEventSender.sendEvent(event) From dcae051e8522d82fa09aeeaa3dcdda8f83aa1c02 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 12:19:58 +0200 Subject: [PATCH 13/83] Create enum as per the spec and use default values when applicable --- .../room/model/call/CallAnswerContent.kt | 4 +-- .../room/model/call/CallCandidatesContent.kt | 2 +- .../room/model/call/CallHangupContent.kt | 14 +++++++--- .../room/model/call/CallInviteContent.kt | 8 +++--- .../api/session/room/model/call/SdpType.kt | 27 +++++++++++++++++++ .../session/call/DefaultCallService.kt | 21 +++------------ 6 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/SdpType.kt diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt index f9f9cf15b2..7fb575d12c 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallAnswerContent.kt @@ -35,7 +35,7 @@ data class CallAnswerContent( /** * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. */ - @Json(name = "version") val version: Int + @Json(name = "version") val version: Int = 0 ) { @JsonClass(generateAdapter = true) @@ -43,7 +43,7 @@ data class CallAnswerContent( /** * Required. The type of session description. Must be 'answer'. */ - @Json(name = "type") val type: String, + @Json(name = "type") val type: SdpType = SdpType.ANSWER, /** * Required. The SDP text of the session description. */ diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt index 1375776919..6132d1a57a 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt @@ -36,7 +36,7 @@ data class CallCandidatesContent( /** * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. */ - @Json(name = "version") val version: Int + @Json(name = "version") val version: Int = 0 ) { @JsonClass(generateAdapter = true) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt index c17efd2559..1e50bc247e 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallHangupContent.kt @@ -32,11 +32,19 @@ data class CallHangupContent( /** * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ - @Json(name = "version") val version: Int, + @Json(name = "version") val version: Int = 0, /** * Optional error reason for the hangup. This should not be provided when the user naturally ends or rejects the call. * When there was an error in the call negotiation, this should be `ice_failed` for when ICE negotiation fails * or `invite_timeout` for when the other party did not answer in time. One of: ["ice_failed", "invite_timeout"] */ - @Json(name = "reason") val reason: String? -) + @Json(name = "reason") val reason: Reason? = null +) { + enum class Reason { + @Json(name = "ice_failed") + ICE_FAILED, + + @Json(name = "invite_timeout") + INVITE_TIMEOUT + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt index cd0df70c96..1fad181fab 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt @@ -31,24 +31,24 @@ data class CallInviteContent( /** * Required. The session description object */ - @Json(name = "version") val version: Int?, + @Json(name = "offer") val offer: Offer?, /** * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ - @Json(name = "lifetime") val lifetime: Int?, + @Json(name = "version") val version: Int? = 0, /** * Required. The time in milliseconds that the invite is valid for. * Once the invite age exceeds this value, clients should discard it. * They should also no longer show the call as awaiting an answer in the UI. */ - @Json(name = "offer") val offer: Offer? + @Json(name = "lifetime") val lifetime: Int? ) { @JsonClass(generateAdapter = true) data class Offer( /** * Required. The type of session description. Must be 'offer'. */ - @Json(name = "type") val type: String?, + @Json(name = "type") val type: SdpType? = SdpType.OFFER, /** * Required. The SDP text of the session description. */ diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/SdpType.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/SdpType.kt new file mode 100644 index 0000000000..17c6d9a89f --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/SdpType.kt @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.room.model.call + +import com.squareup.moshi.Json + +enum class SdpType { + @Json(name = "offer") + OFFER, + + @Json(name = "answer") + ANSWER +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index 7b0731d8a8..2f26dce2cb 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -61,12 +61,8 @@ internal class DefaultCallService @Inject constructor( override fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { val eventContent = CallInviteContent( callId = callId, - version = 0, lifetime = CALL_TIMEOUT_MS, - offer = CallInviteContent.Offer( - type = sdp.type.canonicalForm(), - sdp = sdp.description - ) + offer = CallInviteContent.Offer(sdp = sdp.description) ) createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> @@ -83,11 +79,7 @@ internal class DefaultCallService @Inject constructor( override fun sendAnswerSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { val eventContent = CallAnswerContent( callId = callId, - version = 0, - answer = CallAnswerContent.Answer( - type = sdp.type.canonicalForm(), - sdp = sdp.description - ) + answer = CallAnswerContent.Answer(sdp = sdp.description) ) createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> @@ -104,7 +96,6 @@ internal class DefaultCallService @Inject constructor( override fun sendLocalIceCandidates(callId: String, roomId: String, candidates: List) { val eventContent = CallCandidatesContent( callId = callId, - version = 0, candidates = candidates.map { CallCandidatesContent.Candidate( sdpMid = it.sdpMid, @@ -128,11 +119,7 @@ internal class DefaultCallService @Inject constructor( } override fun sendHangup(callId: String, roomId: String) { - val eventContent = CallHangupContent( - callId = callId, - version = 0, - reason = null - ) + val eventContent = CallHangupContent(callId = callId) createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = eventContent.toContent()).let { event -> roomEventSender.sendEvent(event) } @@ -146,7 +133,7 @@ internal class DefaultCallService @Inject constructor( callListeners.remove(listener) } - fun onCallEvent(event: Event) { + internal fun onCallEvent(event: Event) { when (event.getClearType()) { EventType.CALL_ANSWER -> { event.getClearContent().toModel()?.let { From 8c9ca1e0f28659cd93231563f08898864f57e3d6 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 12:39:04 +0200 Subject: [PATCH 14/83] Cleanup listener --- .../android/api/session/call/CallsListener.kt | 2 +- .../internal/session/call/DefaultCallService.kt | 17 +++++++++-------- .../riotx/features/call/VectorCallViewModel.kt | 2 +- .../call/WebRtcPeerConnectionManager.kt | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index 1fc4170f5c..21ab8dbe93 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -41,7 +41,7 @@ interface CallsListener { // */ // fun onCallHangUp(peerSignalingClient: PeerSignalingClient) - fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) + fun onCallInviteReceived(signalingRoomId: String, fromUserId: String, callInviteContent: CallInviteContent) fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index 2f26dce2cb..72fae10a9c 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -48,7 +48,7 @@ internal class DefaultCallService @Inject constructor( private val roomEventSender: RoomEventSender ) : CallService { - private val callListeners = ArrayList() + private val callListeners = mutableSetOf() override fun getTurnServer(callback: MatrixCallback) { TODO("not implemented") // To change body of created functions use File | Settings | File Templates. @@ -126,7 +126,7 @@ internal class DefaultCallService @Inject constructor( } override fun addCallListener(listener: CallsListener) { - if (!callListeners.contains(listener)) callListeners.add(listener) + callListeners.add(listener) } override fun removeCallListener(listener: CallsListener) { @@ -154,7 +154,7 @@ internal class DefaultCallService @Inject constructor( } private fun onCallHangup(hangup: CallHangupContent) { - callListeners.forEach { + callListeners.toList().forEach { tryThis { it.onCallHangupReceived(hangup) } @@ -162,19 +162,20 @@ internal class DefaultCallService @Inject constructor( } private fun onCallAnswer(answer: CallAnswerContent) { - callListeners.forEach { + callListeners.toList().forEach { tryThis { it.onCallAnswerReceived(answer) } } } - private fun onCallInvite(roomId: String, userId: String, answer: CallInviteContent) { - if (userId == this.userId) return + private fun onCallInvite(roomId: String, fromUserId: String, invite: CallInviteContent) { + // Ignore the invitation from current user + if (fromUserId == userId) return - callListeners.forEach { + callListeners.toList().forEach { tryThis { - it.onCallInviteReceived(roomId, userId, answer) + it.onCallInviteReceived(roomId, fromUserId, invite) } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 07836e64f2..7e7a9eef8e 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -62,7 +62,7 @@ class VectorCallViewModel @AssistedInject constructor( } } - override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) { + override fun onCallInviteReceived(signalingRoomId: String, fromUserId: String, callInviteContent: CallInviteContent) { } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index e50f2bcc8f..d59dd8d101 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -428,14 +428,14 @@ class WebRtcPeerConnectionManager @Inject constructor( sendSdpOffer() } - override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) { + override fun onCallInviteReceived(signalingRoomId: String, fromUserId: String, callInviteContent: CallInviteContent) { this.callId = callInviteContent.callId this.signalingRoomId = signalingRoomId - this.participantUserId = participantUserId + this.participantUserId = fromUserId this.isVideoCall = callInviteContent.isVideo() - startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo()) - context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, callInviteContent.isVideo())) + startHeadsUpService(signalingRoomId, fromUserId, true, callInviteContent.isVideo()) + context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, fromUserId, false, callInviteContent.isVideo())) startCall() } From 2581a3433e28fb5ee3b3505da4b13045b44c395f Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 14:15:35 +0200 Subject: [PATCH 15/83] Create RoomCallService --- .../android/api/session/call/CallService.kt | 2 - .../matrix/android/api/session/room/Room.kt | 2 + .../api/session/room/call/RoomCallService.kt | 27 +++++++++++++ .../session/call/DefaultCallService.kt | 4 -- .../internal/session/room/DefaultRoom.kt | 3 ++ .../internal/session/room/RoomFactory.kt | 3 ++ .../room/call/DefaultRoomCallService.kt | 40 +++++++++++++++++++ .../home/room/detail/RoomDetailViewModel.kt | 4 +- 8 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/call/RoomCallService.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt index 554c00ab05..f691a4a920 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -24,8 +24,6 @@ interface CallService { fun getTurnServer(callback: MatrixCallback) - fun isCallSupportedInRoom(roomId: String) : Boolean - /** * Send offer SDP to the other participant. */ diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/Room.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/Room.kt index 4ae61f46e1..231eaa5806 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/Room.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/Room.kt @@ -17,6 +17,7 @@ package im.vector.matrix.android.api.session.room import androidx.lifecycle.LiveData +import im.vector.matrix.android.api.session.room.call.RoomCallService import im.vector.matrix.android.api.session.room.crypto.RoomCryptoService import im.vector.matrix.android.api.session.room.members.MembershipService import im.vector.matrix.android.api.session.room.model.RoomSummary @@ -47,6 +48,7 @@ interface Room : StateService, UploadsService, ReportingService, + RoomCallService, RelationService, RoomCryptoService, RoomPushRuleService { diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/call/RoomCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/call/RoomCallService.kt new file mode 100644 index 0000000000..b1dc0899bb --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/call/RoomCallService.kt @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.room.call + +/** + * This interface defines methods to handle calls in a room. It's implemented at the room level. + */ +interface RoomCallService { + /** + * Return true if calls (audio or video) can be performed on this Room + */ + fun canStartCall(): Boolean +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index 72fae10a9c..7fd1433dd3 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -54,10 +54,6 @@ internal class DefaultCallService @Inject constructor( TODO("not implemented") // To change body of created functions use File | Settings | File Templates. } - override fun isCallSupportedInRoom(roomId: String): Boolean { - TODO("not implemented") // To change body of created functions use File | Settings | File Templates. - } - override fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { val eventContent = CallInviteContent( callId = callId, diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/DefaultRoom.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/DefaultRoom.kt index 04d495211e..0c86b9572e 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/DefaultRoom.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/DefaultRoom.kt @@ -23,6 +23,7 @@ import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.session.crypto.CryptoService import im.vector.matrix.android.api.session.events.model.EventType import im.vector.matrix.android.api.session.room.Room +import im.vector.matrix.android.api.session.room.call.RoomCallService import im.vector.matrix.android.api.session.room.members.MembershipService import im.vector.matrix.android.api.session.room.model.RoomSummary import im.vector.matrix.android.api.session.room.model.relation.RelationService @@ -58,6 +59,7 @@ internal class DefaultRoom @Inject constructor(override val roomId: String, private val stateService: StateService, private val uploadsService: UploadsService, private val reportingService: ReportingService, + private val roomCallService: RoomCallService, private val readService: ReadService, private val typingService: TypingService, private val tagsService: TagsService, @@ -74,6 +76,7 @@ internal class DefaultRoom @Inject constructor(override val roomId: String, StateService by stateService, UploadsService by uploadsService, ReportingService by reportingService, + RoomCallService by roomCallService, ReadService by readService, TypingService by typingService, TagsService by tagsService, diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomFactory.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomFactory.kt index 0560aa80c2..7cd9134011 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomFactory.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomFactory.kt @@ -21,6 +21,7 @@ import im.vector.matrix.android.api.session.crypto.CryptoService import im.vector.matrix.android.api.session.room.Room import im.vector.matrix.android.internal.database.mapper.RoomSummaryMapper import im.vector.matrix.android.internal.session.SessionScope +import im.vector.matrix.android.internal.session.room.call.DefaultRoomCallService import im.vector.matrix.android.internal.session.room.draft.DefaultDraftService import im.vector.matrix.android.internal.session.room.membership.DefaultMembershipService import im.vector.matrix.android.internal.session.room.notification.DefaultRoomPushRuleService @@ -51,6 +52,7 @@ internal class DefaultRoomFactory @Inject constructor(private val monarchy: Mona private val stateServiceFactory: DefaultStateService.Factory, private val uploadsServiceFactory: DefaultUploadsService.Factory, private val reportingServiceFactory: DefaultReportingService.Factory, + private val roomCallServiceFactory: DefaultRoomCallService.Factory, private val readServiceFactory: DefaultReadService.Factory, private val typingServiceFactory: DefaultTypingService.Factory, private val tagsServiceFactory: DefaultTagsService.Factory, @@ -72,6 +74,7 @@ internal class DefaultRoomFactory @Inject constructor(private val monarchy: Mona stateService = stateServiceFactory.create(roomId), uploadsService = uploadsServiceFactory.create(roomId), reportingService = reportingServiceFactory.create(roomId), + roomCallService = roomCallServiceFactory.create(roomId), readService = readServiceFactory.create(roomId), typingService = typingServiceFactory.create(roomId), tagsService = tagsServiceFactory.create(roomId), diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt new file mode 100644 index 0000000000..bac45ab5dc --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.room.call + +import com.squareup.inject.assisted.Assisted +import com.squareup.inject.assisted.AssistedInject +import im.vector.matrix.android.api.extensions.orFalse +import im.vector.matrix.android.api.session.room.call.RoomCallService +import im.vector.matrix.android.internal.session.room.RoomGetter + +internal class DefaultRoomCallService @AssistedInject constructor( + @Assisted private val roomId: String, + private val roomGetter: RoomGetter +) : RoomCallService { + + @AssistedInject.Factory + interface Factory { + fun create(roomId: String): RoomCallService + } + + override fun canStartCall(): Boolean { + return roomGetter.getRoom(roomId)?.roomSummary()?.let { + it.isDirect && it.joinedMembersCount == 2 + }.orFalse() + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index 7e33d69345..5ec8d207f4 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -374,8 +374,8 @@ class RoomDetailViewModel @AssistedInject constructor( R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 R.id.open_matrix_apps -> true - R.id.voip_call -> room.roomSummary()?.isDirect == true && room.roomSummary()?.joinedMembersCount == 2 - else -> false + R.id.voice_call, R.id.video_call -> room.canStartCall() + else -> false } // PRIVATE METHODS ***************************************************************************** From 24a9931abd58999fd2a6d559de708369133142af Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 14:25:00 +0200 Subject: [PATCH 16/83] Rename some API --- .../matrix/android/api/session/call/CallService.kt | 9 ++++++--- .../android/internal/session/call/DefaultCallService.kt | 6 +++--- .../riotx/features/call/WebRtcPeerConnectionManager.kt | 9 +++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt index f691a4a920..b16747048d 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -25,14 +25,17 @@ interface CallService { fun getTurnServer(callback: MatrixCallback) /** + * Start a call * Send offer SDP to the other participant. + * @param callId a callId that the caller can create, it will be used to identify the call for other methods */ - fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) + fun startCall(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) /** + * Accept an incoming call * Send answer SDP to the other participant. */ - fun sendAnswerSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) + fun pickUp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) /** * Send Ice candidate to the other participant. @@ -47,7 +50,7 @@ interface CallService { /** * Send a hangup event */ - fun sendHangup(callId: String, roomId: String) + fun hangup(callId: String, roomId: String) fun addCallListener(listener: CallsListener) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index 7fd1433dd3..55560228cb 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -54,7 +54,7 @@ internal class DefaultCallService @Inject constructor( TODO("not implemented") // To change body of created functions use File | Settings | File Templates. } - override fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { + override fun startCall(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { val eventContent = CallInviteContent( callId = callId, lifetime = CALL_TIMEOUT_MS, @@ -72,7 +72,7 @@ internal class DefaultCallService @Inject constructor( } } - override fun sendAnswerSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { + override fun pickUp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { val eventContent = CallAnswerContent( callId = callId, answer = CallAnswerContent.Answer(sdp = sdp.description) @@ -114,7 +114,7 @@ internal class DefaultCallService @Inject constructor( override fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) { } - override fun sendHangup(callId: String, roomId: String) { + override fun hangup(callId: String, roomId: String) { val eventContent = CallHangupContent(callId = callId) createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = eventContent.toContent()).let { event -> roomEventSender.sendEvent(event) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index d59dd8d101..a45dadc122 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -304,9 +304,10 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnection?.setLocalDescription(object : SdpObserverAdapter() { override fun onSetSuccess() { Timber.v("## setLocalDescription success") - callId = UUID.randomUUID().toString() - Timber.v("## sending offer to callId: $callId roomId: $signalingRoomId") - sessionHolder.getActiveSession().callService().sendOfferSdp(callId ?: "", signalingRoomId + val id = UUID.randomUUID().toString() + callId = id + Timber.v("## sending offer to callId: $id roomId: $signalingRoomId") + sessionHolder.getActiveSession().callService().startCall(id, signalingRoomId ?: "", sessionDescription, object : MatrixCallback {}) } }, sessionDescription) @@ -449,7 +450,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun endCall() { if (callId != null && signalingRoomId != null) { - sessionHolder.getActiveSession().callService().sendHangup(callId!!, signalingRoomId!!) + sessionHolder.getActiveSession().callService().hangup(callId!!, signalingRoomId!!) } close() } From 03b9904b07d3fdfdd5f3a358e92fa55ca8b9b5ce Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 15:29:43 +0200 Subject: [PATCH 17/83] Create a MxCall interface to better handle call --- .../android/api/session/call/CallService.kt | 29 +---- .../android/api/session/call/CallsListener.kt | 2 +- .../matrix/android/api/session/call/MxCall.kt | 65 ++++++++++ .../session/call/DefaultCallService.kt | 118 +++++------------- .../internal/session/call/model/MxCallImpl.kt | 108 ++++++++++++++++ .../riotx/features/call/VectorCallActivity.kt | 10 +- .../features/call/VectorCallViewModel.kt | 3 +- .../call/WebRtcPeerConnectionManager.kt | 67 +++++----- .../call/service/CallHeadsUpService.kt | 7 +- .../call/service/CallHeadsUpServiceArgs.kt | 2 +- 10 files changed, 244 insertions(+), 167 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt index b16747048d..d94e84c7b9 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -17,40 +17,15 @@ package im.vector.matrix.android.api.session.call import im.vector.matrix.android.api.MatrixCallback -import org.webrtc.IceCandidate -import org.webrtc.SessionDescription interface CallService { fun getTurnServer(callback: MatrixCallback) /** - * Start a call - * Send offer SDP to the other participant. - * @param callId a callId that the caller can create, it will be used to identify the call for other methods + * Create an outgoing call */ - fun startCall(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) - - /** - * Accept an incoming call - * Send answer SDP to the other participant. - */ - fun pickUp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) - - /** - * Send Ice candidate to the other participant. - */ - fun sendLocalIceCandidates(callId: String, roomId: String, candidates: List) - - /** - * Send removed ICE candidates to the other participant. - */ - fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) - - /** - * Send a hangup event - */ - fun hangup(callId: String, roomId: String) + fun createOutgoingCall(roomId: String, otherUserId: String, isVideoCall: Boolean): MxCall fun addCallListener(listener: CallsListener) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index 21ab8dbe93..3ab68ceb28 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -41,7 +41,7 @@ interface CallsListener { // */ // fun onCallHangUp(peerSignalingClient: PeerSignalingClient) - fun onCallInviteReceived(signalingRoomId: String, fromUserId: String, callInviteContent: CallInviteContent) + fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt new file mode 100644 index 0000000000..afb89b3bb8 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription + +interface MxCallDetail { + val isOutgoing: Boolean + val roomId: String + val otherUserId: String + val isVideoCall: Boolean +} + +/** + * Define both an incoming call and on outgoing call + */ +interface MxCall : MxCallDetail { + /** + * Pick Up the incoming call + * It has no effect on outgoing call + */ + fun accept(sdp: SessionDescription) + + /** + * Reject an incoming call + * It's an alias to hangUp + */ + fun reject() = hangUp() + + /** + * End the call + */ + fun hangUp() + + /** + * Start a call + * Send offer SDP to the other participant. + */ + fun offerSdp(sdp: SessionDescription) + + /** + * Send Ice candidate to the other participant. + */ + fun sendLocalIceCandidates(candidates: List) + + /** + * Send removed ICE candidates to the other participant. + */ + fun sendLocalIceCandidateRemovals(candidates: List) +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index 55560228cb..b0c9ccc2c4 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -20,24 +20,20 @@ import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.call.MxCall import im.vector.matrix.android.api.session.call.TurnServer -import im.vector.matrix.android.api.session.events.model.Content import im.vector.matrix.android.api.session.events.model.Event import im.vector.matrix.android.api.session.events.model.EventType -import im.vector.matrix.android.api.session.events.model.LocalEcho -import im.vector.matrix.android.api.session.events.model.UnsignedData -import im.vector.matrix.android.api.session.events.model.toContent import im.vector.matrix.android.api.session.events.model.toModel import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent -import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.matrix.android.internal.di.UserId import im.vector.matrix.android.internal.session.SessionScope +import im.vector.matrix.android.internal.session.call.model.MxCallImpl import im.vector.matrix.android.internal.session.room.send.LocalEchoEventFactory import im.vector.matrix.android.internal.session.room.send.RoomEventSender -import org.webrtc.IceCandidate -import org.webrtc.SessionDescription +import java.util.UUID import javax.inject.Inject @SessionScope @@ -54,71 +50,17 @@ internal class DefaultCallService @Inject constructor( TODO("not implemented") // To change body of created functions use File | Settings | File Templates. } - override fun startCall(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { - val eventContent = CallInviteContent( - callId = callId, - lifetime = CALL_TIMEOUT_MS, - offer = CallInviteContent.Offer(sdp = sdp.description) + override fun createOutgoingCall(roomId: String, otherUserId: String, isVideoCall: Boolean): MxCall { + return MxCallImpl( + callId = UUID.randomUUID().toString(), + isOutgoing = true, + roomId = roomId, + userId = userId, + otherUserId = otherUserId, + isVideoCall = isVideoCall, + localEchoEventFactory = localEchoEventFactory, + roomEventSender = roomEventSender ) - - createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> - roomEventSender.sendEvent(event) -// sendEventTask -// .configureWith( -// SendEventTask.Params(event = event, cryptoService = cryptoService) -// ) { -// this.callback = callback -// }.executeBy(taskExecutor) - } - } - - override fun pickUp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { - val eventContent = CallAnswerContent( - callId = callId, - answer = CallAnswerContent.Answer(sdp = sdp.description) - ) - - createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> - roomEventSender.sendEvent(event) -// sendEventTask -// .configureWith( -// SendEventTask.Params(event = event, cryptoService = cryptoService) -// ) { -// this.callback = callback -// }.executeBy(taskExecutor) - } - } - - override fun sendLocalIceCandidates(callId: String, roomId: String, candidates: List) { - val eventContent = CallCandidatesContent( - callId = callId, - candidates = candidates.map { - CallCandidatesContent.Candidate( - sdpMid = it.sdpMid, - sdpMLineIndex = it.sdpMLineIndex.toString(), - candidate = it.sdp - ) - } - ) - createEventAndLocalEcho(type = EventType.CALL_CANDIDATES, roomId = roomId, content = eventContent.toContent()).let { event -> - roomEventSender.sendEvent(event) -// sendEventTask -// .configureWith( -// SendEventTask.Params(event = event, cryptoService = cryptoService) -// ) { -// this.callback = callback -// }.executeBy(taskExecutor) - } - } - - override fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) { - } - - override fun hangup(callId: String, roomId: String) { - val eventContent = CallHangupContent(callId = callId) - createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = eventContent.toContent()).let { event -> - roomEventSender.sendEvent(event) - } } override fun addCallListener(listener: CallsListener) { @@ -137,8 +79,18 @@ internal class DefaultCallService @Inject constructor( } } EventType.CALL_INVITE -> { - event.getClearContent().toModel()?.let { - onCallInvite(event.roomId ?: "", event.senderId ?: "", it) + event.getClearContent().toModel()?.let { content -> + val incomingCall = MxCallImpl( + callId = content.callId ?: return@let, + isOutgoing = false, + roomId = event.roomId ?: return@let, + userId = userId, + otherUserId = event.senderId ?: return@let, + isVideoCall = content.isVideo(), + localEchoEventFactory = localEchoEventFactory, + roomEventSender = roomEventSender + ) + onCallInvite(incomingCall, content) } } EventType.CALL_HANGUP -> { @@ -165,31 +117,17 @@ internal class DefaultCallService @Inject constructor( } } - private fun onCallInvite(roomId: String, fromUserId: String, invite: CallInviteContent) { + private fun onCallInvite(incomingCall: MxCall, invite: CallInviteContent) { // Ignore the invitation from current user - if (fromUserId == userId) return + if (incomingCall.otherUserId == userId) return callListeners.toList().forEach { tryThis { - it.onCallInviteReceived(roomId, fromUserId, invite) + it.onCallInviteReceived(incomingCall, invite) } } } - private fun createEventAndLocalEcho(localId: String = LocalEcho.createLocalEchoId(), type: String, roomId: String, content: Content): Event { - return Event( - roomId = roomId, - originServerTs = System.currentTimeMillis(), - senderId = userId, - eventId = localId, - type = type, - content = content, - unsignedData = UnsignedData(age = null, transactionId = localId) - ).also { - localEchoEventFactory.createLocalEcho(it) - } - } - companion object { const val CALL_TIMEOUT_MS = 120_000 } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt new file mode 100644 index 0000000000..82537b2e5b --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call.model + +import im.vector.matrix.android.api.session.call.MxCall +import im.vector.matrix.android.api.session.events.model.Content +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.events.model.EventType +import im.vector.matrix.android.api.session.events.model.LocalEcho +import im.vector.matrix.android.api.session.events.model.UnsignedData +import im.vector.matrix.android.api.session.events.model.toContent +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent +import im.vector.matrix.android.api.session.room.model.call.CallHangupContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.matrix.android.internal.session.call.DefaultCallService +import im.vector.matrix.android.internal.session.room.send.LocalEchoEventFactory +import im.vector.matrix.android.internal.session.room.send.RoomEventSender +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription + +internal class MxCallImpl( + val callId: String, + override val isOutgoing: Boolean, + override val roomId: String, + private val userId: String, + override val otherUserId: String, + override val isVideoCall: Boolean, + private val localEchoEventFactory: LocalEchoEventFactory, + private val roomEventSender: RoomEventSender +) : MxCall { + + override fun offerSdp(sdp: SessionDescription) { + if (!isOutgoing) return + + CallInviteContent( + callId = callId, + lifetime = DefaultCallService.CALL_TIMEOUT_MS, + offer = CallInviteContent.Offer(sdp = sdp.description) + ) + .let { createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = it.toContent()) } + .also { roomEventSender.sendEvent(it) } + } + + override fun sendLocalIceCandidates(candidates: List) { + CallCandidatesContent( + callId = callId, + candidates = candidates.map { + CallCandidatesContent.Candidate( + sdpMid = it.sdpMid, + sdpMLineIndex = it.sdpMLineIndex.toString(), + candidate = it.sdp + ) + } + ) + .let { createEventAndLocalEcho(type = EventType.CALL_CANDIDATES, roomId = roomId, content = it.toContent()) } + .also { roomEventSender.sendEvent(it) } + } + + override fun sendLocalIceCandidateRemovals(candidates: List) { + } + + override fun hangUp() { + CallHangupContent( + callId = callId + ) + .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) } + .also { roomEventSender.sendEvent(it) } + } + + override fun accept(sdp: SessionDescription) { + if (isOutgoing) return + + CallAnswerContent( + callId = callId, + answer = CallAnswerContent.Answer(sdp = sdp.description) + ) + .let { createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = it.toContent()) } + .also { roomEventSender.sendEvent(it) } + } + + private fun createEventAndLocalEcho(localId: String = LocalEcho.createLocalEchoId(), type: String, roomId: String, content: Content): Event { + return Event( + roomId = roomId, + originServerTs = System.currentTimeMillis(), + senderId = userId, + eventId = localId, + type = type, + content = content, + unsignedData = UnsignedData(age = null, transactionId = localId) + ) + .also { localEchoEventFactory.createLocalEcho(it) } + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index ba32959973..af1bea1d6c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -32,6 +32,7 @@ import butterknife.BindView import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel import im.vector.matrix.android.api.session.call.EglUtils +import im.vector.matrix.android.api.session.call.MxCallDetail import im.vector.riotx.R import im.vector.riotx.core.di.ScreenComponent import im.vector.riotx.core.platform.VectorBaseActivity @@ -92,8 +93,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis private val iceCandidateSource: PublishSubject = PublishSubject.create() - - var callHeadsUpService: CallHeadsUpService? = null private val serviceConnection = object : ServiceConnection { override fun onServiceDisconnected(name: ComponentName?) { @@ -363,7 +362,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis //peerConnectionManager.answerReceived("", sdp) // peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) } - is VectorCallViewEvents.CallHangup -> { + is VectorCallViewEvents.CallHangup -> { finish() } } @@ -416,9 +415,9 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) // } - fun newIntent(context: Context, roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean): Intent { + fun newIntent(context: Context, mxCall: MxCallDetail): Intent { return Intent(context, VectorCallActivity::class.java).apply { - putExtra(MvRx.KEY_ARG, CallArgs(roomId, participantUserId, isIncomingCall, isVideoCall)) + putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall)) } } } @@ -447,6 +446,5 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } override fun sendOffer(sessionDescription: SessionDescription) { - } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 7e7a9eef8e..296dfe5582 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -23,6 +23,7 @@ import com.squareup.inject.assisted.Assisted import com.squareup.inject.assisted.AssistedInject import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.call.MxCall import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent @@ -62,7 +63,7 @@ class VectorCallViewModel @AssistedInject constructor( } } - override fun onCallInviteReceived(signalingRoomId: String, fromUserId: String, callInviteContent: CallInviteContent) { + override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index a45dadc122..a283414310 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -22,10 +22,11 @@ import android.content.Intent import android.content.ServiceConnection import android.os.IBinder import androidx.core.content.ContextCompat -import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils +import im.vector.matrix.android.api.session.call.MxCall +import im.vector.matrix.android.api.session.call.MxCallDetail import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent @@ -53,7 +54,6 @@ import org.webrtc.VideoSource import org.webrtc.VideoTrack import timber.log.Timber import java.lang.ref.WeakReference -import java.util.UUID import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import javax.inject.Inject @@ -114,10 +114,7 @@ class WebRtcPeerConnectionManager @Inject constructor( var callHeadsUpService: CallHeadsUpService? = null - private var callId: String? = null - private var signalingRoomId: String? = null - private var participantUserId: String? = null - private var isVideoCall: Boolean? = null + private var currentCall: MxCall? = null private val serviceConnection = object : ServiceConnection { override fun onServiceDisconnected(name: ComponentName?) { @@ -275,11 +272,8 @@ class WebRtcPeerConnectionManager @Inject constructor( .subscribe { // omit empty :/ if (it.isNotEmpty()) { - Timber.v("## Sending local ice candidates to callId: $callId roomId: $signalingRoomId") - sessionHolder - .getActiveSession() - .callService() - .sendLocalIceCandidates(callId ?: "", signalingRoomId ?: "", it) + Timber.v("## Sending local ice candidates to call") + currentCall?.sendLocalIceCandidates(it) } } } @@ -304,11 +298,8 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnection?.setLocalDescription(object : SdpObserverAdapter() { override fun onSetSuccess() { Timber.v("## setLocalDescription success") - val id = UUID.randomUUID().toString() - callId = id - Timber.v("## sending offer to callId: $id roomId: $signalingRoomId") - sessionHolder.getActiveSession().callService().startCall(id, signalingRoomId - ?: "", sessionDescription, object : MatrixCallback {}) + Timber.v("## sending offer") + currentCall?.offerSdp(sessionDescription) } }, sessionDescription) } @@ -330,7 +321,7 @@ class WebRtcPeerConnectionManager @Inject constructor( localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? - if (isVideoCall == true) { + if (currentCall?.isVideoCall == true) { val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) val frontCamera = cameraIterator.deviceNames ?.firstOrNull { cameraIterator.isFrontFacing(it) } @@ -417,55 +408,55 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun startOutgoingCall(context: Context, signalingRoomId: String, participantUserId: String, isVideoCall: Boolean) { - this.signalingRoomId = signalingRoomId - this.participantUserId = participantUserId - this.isVideoCall = isVideoCall + fun startOutgoingCall(context: Context, signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { + val createdCall = sessionHolder.getSafeActiveSession()?.callService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return + currentCall = createdCall - startHeadsUpService(signalingRoomId, sessionHolder.getActiveSession().myUserId, false, isVideoCall) - context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, isVideoCall)) + startHeadsUpService(createdCall) + context.startActivity(VectorCallActivity.newIntent(context, createdCall)) startCall() sendSdpOffer() } - override fun onCallInviteReceived(signalingRoomId: String, fromUserId: String, callInviteContent: CallInviteContent) { - this.callId = callInviteContent.callId - this.signalingRoomId = signalingRoomId - this.participantUserId = fromUserId - this.isVideoCall = callInviteContent.isVideo() + override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { + // TODO What if a call is currently active? + if (currentCall != null) { + Timber.w("TODO: Automatically reject incoming call?") + return + } - startHeadsUpService(signalingRoomId, fromUserId, true, callInviteContent.isVideo()) - context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, fromUserId, false, callInviteContent.isVideo())) + currentCall = mxCall + + startHeadsUpService(mxCall) + context.startActivity(VectorCallActivity.newIntent(context, mxCall)) startCall() } - private fun startHeadsUpService(roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean) { - val callHeadsUpServiceIntent = CallHeadsUpService.newInstance(context, roomId, participantUserId, isIncomingCall, isVideoCall) + private fun startHeadsUpService(mxCall: MxCallDetail) { + val callHeadsUpServiceIntent = CallHeadsUpService.newInstance(context, mxCall) ContextCompat.startForegroundService(context, callHeadsUpServiceIntent) context.bindService(Intent(context, CallHeadsUpService::class.java), serviceConnection, 0) } fun endCall() { - if (callId != null && signalingRoomId != null) { - sessionHolder.getActiveSession().callService().hangup(callId!!, signalingRoomId!!) - } + currentCall?.hangUp() + currentCall = null close() } override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { - this.callId = callAnswerContent.callId - executor.execute { - Timber.v("## answerReceived $callId") + Timber.v("## answerReceived") val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) } } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { + currentCall = null close() } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt index 653ac90f0d..f8ec1c6ee1 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt @@ -30,6 +30,7 @@ import android.os.Binder import android.os.Build import android.os.IBinder import androidx.core.app.NotificationCompat +import im.vector.matrix.android.api.session.call.MxCallDetail import im.vector.riotx.R import im.vector.riotx.features.call.VectorCallActivity @@ -50,7 +51,7 @@ class CallHeadsUpService : Service() { createNotificationChannel() - val title = callHeadsUpServiceArgs?.participantUserId ?: "" + val title = callHeadsUpServiceArgs?.otherUserId ?: "" val description = when { callHeadsUpServiceArgs?.isIncomingCall == false -> getString(R.string.call_ring) callHeadsUpServiceArgs?.isVideoCall == true -> getString(R.string.incoming_video_call) @@ -133,8 +134,8 @@ class CallHeadsUpService : Service() { private const val NOTIFICATION_ID = 999 - fun newInstance(context: Context, roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean): Intent { - val args = CallHeadsUpServiceArgs(roomId, participantUserId, isIncomingCall, isVideoCall) + fun newInstance(context: Context, mxCall: MxCallDetail): Intent { + val args = CallHeadsUpServiceArgs(mxCall.roomId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall) return Intent(context, CallHeadsUpService::class.java).apply { putExtra(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS, args) } diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt index 381975a2ee..1bc857e4a7 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt @@ -22,7 +22,7 @@ import kotlinx.android.parcel.Parcelize @Parcelize data class CallHeadsUpServiceArgs( val roomId: String, - val participantUserId: String, + val otherUserId: String, val isIncomingCall: Boolean, val isVideoCall: Boolean ) : Parcelable From c0988ba6d909b5718f380cf776f514942045c484 Mon Sep 17 00:00:00 2001 From: onurays Date: Fri, 29 May 2020 00:53:40 +0300 Subject: [PATCH 18/83] Merge conflicts and implement answer function. --- .../internal/session/call/model/MxCallImpl.kt | 2 +- .../riotx/features/call/VectorCallActivity.kt | 1 + .../call/WebRtcPeerConnectionManager.kt | 58 ++++++++++++++++++- .../call/service/CallHeadsUpActionReceiver.kt | 1 + 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt index 82537b2e5b..fdee9c295b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt @@ -89,7 +89,7 @@ internal class MxCallImpl( callId = callId, answer = CallAnswerContent.Answer(sdp = sdp.description) ) - .let { createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = it.toContent()) } + .let { createEventAndLocalEcho(type = EventType.CALL_ANSWER, roomId = roomId, content = it.toContent()) } .also { roomEventSender.sendEvent(it) } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index af1bea1d6c..c21251c9ed 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -417,6 +417,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis fun newIntent(context: Context, mxCall: MxCallDetail): Intent { return Intent(context, VectorCallActivity::class.java).apply { + flags = Intent.FLAG_ACTIVITY_NEW_TASK putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall)) } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index a283414310..ec5f6f2048 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -281,7 +281,7 @@ class WebRtcPeerConnectionManager @Inject constructor( private fun sendSdpOffer() { val constraints = MediaConstraints() constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) Timber.v("## VOIP creating offer...") peerConnection?.createOffer(object : SdpObserver { @@ -429,9 +429,10 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall = mxCall startHeadsUpService(mxCall) - context.startActivity(VectorCallActivity.newIntent(context, mxCall)) - startCall() + + val sdp = SessionDescription(SessionDescription.Type.OFFER, callInviteContent.offer?.sdp) + peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) } private fun startHeadsUpService(mxCall: MxCallDetail) { @@ -441,6 +442,57 @@ class WebRtcPeerConnectionManager @Inject constructor( context.bindService(Intent(context, CallHeadsUpService::class.java), serviceConnection, 0) } + fun answerCall() { + if (currentCall != null) { + val constraints = MediaConstraints().apply { + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) + } + + peerConnection?.createAnswer(object : SdpObserver { + override fun onCreateSuccess(sessionDescription: SessionDescription) { + Timber.v("## createAnswer onCreateSuccess") + + val sdp = SessionDescription(sessionDescription.type, sessionDescription.description) + + peerConnection?.setLocalDescription(object : SdpObserver { + override fun onSetSuccess() { + currentCall?.accept(sdp) + + currentCall?.let { + context.startActivity(VectorCallActivity.newIntent(context, it)) + } + } + + override fun onCreateSuccess(localSdp: SessionDescription) {} + + override fun onSetFailure(p0: String?) { + endCall() + } + + override fun onCreateFailure(p0: String?) { + endCall() + } + }, sdp) + } + + override fun onSetSuccess() { + Timber.v("## createAnswer onSetSuccess") + } + + override fun onSetFailure(error: String) { + Timber.v("answerCall.onSetFailure failed: $error") + endCall() + } + + override fun onCreateFailure(error: String) { + Timber.v("answerCall.onCreateFailure failed: $error") + endCall() + } + }, constraints) + } + } + fun endCall() { currentCall?.hangUp() currentCall = null diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt index aee51ff422..4671217312 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt @@ -49,5 +49,6 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { private fun onCallAnswerClicked() { Timber.d("onCallAnswerClicked") + peerConnectionManager.answerCall() } } From 125d61eb685bcee3c177bc8ba274296003745e59 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 23:30:31 +0200 Subject: [PATCH 19/83] Rename parameters --- .../format/DisplayableEventFormatter.kt | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt index 1d178054ac..39b5e56d99 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt @@ -29,13 +29,12 @@ import me.gujun.android.span.span import javax.inject.Inject class DisplayableEventFormatter @Inject constructor( -// private val sessionHolder: ActiveSessionHolder, private val stringProvider: StringProvider, private val colorProvider: ColorProvider, private val noticeEventFormatter: NoticeEventFormatter ) { - fun format(timelineEvent: TimelineEvent, appendAuthor: Boolean): CharSequence { + fun format(timelineEvent: TimelineEvent, prependAuthor: Boolean): CharSequence { if (timelineEvent.root.isRedacted()) { return noticeEventFormatter.formatRedactedEvent(timelineEvent.root) } @@ -55,32 +54,31 @@ class DisplayableEventFormatter @Inject constructor( timelineEvent.getLastMessageContent()?.let { messageContent -> when (messageContent.msgType) { MessageType.MSGTYPE_VERIFICATION_REQUEST -> { - return simpleFormat(senderName, stringProvider.getString(R.string.verification_request), appendAuthor) + return simpleFormat(senderName, stringProvider.getString(R.string.verification_request), prependAuthor) } MessageType.MSGTYPE_IMAGE -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_image), appendAuthor) + return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_image), prependAuthor) } MessageType.MSGTYPE_AUDIO -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_audio_file), appendAuthor) + return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_audio_file), prependAuthor) } MessageType.MSGTYPE_VIDEO -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_video), appendAuthor) + return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_video), prependAuthor) } MessageType.MSGTYPE_FILE -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_file), appendAuthor) + return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_file), prependAuthor) } MessageType.MSGTYPE_TEXT -> { - if (messageContent.isReply()) { + return if (messageContent.isReply()) { // Skip reply prefix, and show important // TODO add a reply image span ? - return simpleFormat(senderName, timelineEvent.getTextEditableContent() - ?: messageContent.body, appendAuthor) + simpleFormat(senderName, timelineEvent.getTextEditableContent() ?: messageContent.body, prependAuthor) } else { - return simpleFormat(senderName, messageContent.body, appendAuthor) + simpleFormat(senderName, messageContent.body, prependAuthor) } } else -> { - return simpleFormat(senderName, messageContent.body, appendAuthor) + return simpleFormat(senderName, messageContent.body, prependAuthor) } } } @@ -96,8 +94,8 @@ class DisplayableEventFormatter @Inject constructor( return span { } } - private fun simpleFormat(senderName: String, body: CharSequence, appendAuthor: Boolean): CharSequence { - return if (appendAuthor) { + private fun simpleFormat(senderName: String, body: CharSequence, prependAuthor: Boolean): CharSequence { + return if (prependAuthor) { span { text = senderName textColor = colorProvider.getColorFromAttribute(R.attr.riotx_text_primary) From d3f93984d45644e46003c0dc05c378332002d9c6 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 23:35:21 +0200 Subject: [PATCH 20/83] Compact coding --- .../format/DisplayableEventFormatter.kt | 44 +++++++------------ 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt index 39b5e56d99..0f628d2ce4 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt @@ -53,45 +53,35 @@ class DisplayableEventFormatter @Inject constructor( EventType.MESSAGE -> { timelineEvent.getLastMessageContent()?.let { messageContent -> when (messageContent.msgType) { - MessageType.MSGTYPE_VERIFICATION_REQUEST -> { - return simpleFormat(senderName, stringProvider.getString(R.string.verification_request), prependAuthor) - } - MessageType.MSGTYPE_IMAGE -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_image), prependAuthor) - } - MessageType.MSGTYPE_AUDIO -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_audio_file), prependAuthor) - } - MessageType.MSGTYPE_VIDEO -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_video), prependAuthor) - } - MessageType.MSGTYPE_FILE -> { - return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_file), prependAuthor) - } - MessageType.MSGTYPE_TEXT -> { - return if (messageContent.isReply()) { + MessageType.MSGTYPE_VERIFICATION_REQUEST -> + simpleFormat(senderName, stringProvider.getString(R.string.verification_request), prependAuthor) + MessageType.MSGTYPE_IMAGE -> + simpleFormat(senderName, stringProvider.getString(R.string.sent_an_image), prependAuthor) + MessageType.MSGTYPE_AUDIO -> + simpleFormat(senderName, stringProvider.getString(R.string.sent_an_audio_file), prependAuthor) + MessageType.MSGTYPE_VIDEO -> + simpleFormat(senderName, stringProvider.getString(R.string.sent_a_video), prependAuthor) + MessageType.MSGTYPE_FILE -> + simpleFormat(senderName, stringProvider.getString(R.string.sent_a_file), prependAuthor) + MessageType.MSGTYPE_TEXT -> + if (messageContent.isReply()) { // Skip reply prefix, and show important // TODO add a reply image span ? simpleFormat(senderName, timelineEvent.getTextEditableContent() ?: messageContent.body, prependAuthor) } else { simpleFormat(senderName, messageContent.body, prependAuthor) } - } - else -> { - return simpleFormat(senderName, messageContent.body, prependAuthor) - } + else -> + simpleFormat(senderName, messageContent.body, prependAuthor) } - } + } ?: span { } } - else -> { - return span { + else -> + span { text = noticeEventFormatter.format(timelineEvent) ?: "" textStyle = "italic" } - } } - - return span { } } private fun simpleFormat(senderName: String, body: CharSequence, prependAuthor: Boolean): CharSequence { From df4aab1d73b40057649f268fb643880eb430b84c Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 23:37:10 +0200 Subject: [PATCH 21/83] Use EventType.isCallEvent() --- .../internal/session/call/CallEventsObserverTask.kt | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt index 2aeabc09f4..919600f3bc 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt @@ -57,13 +57,8 @@ internal class DefaultCallEventsObserverTask @Inject constructor( Timber.w("Event with no room id ${event.eventId}") } decryptIfNeeded(event) - when (event.getClearType()) { - EventType.CALL_INVITE, - EventType.CALL_CANDIDATES, - EventType.CALL_HANGUP, - EventType.CALL_ANSWER -> { - callService.onCallEvent(event) - } + if (EventType.isCallEvent(event.getClearType())) { + callService.onCallEvent(event) } } Timber.v("$realm : $userId") From 0bb92e9e91684857b870a457873bc87dfdd133ec Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 23:48:20 +0200 Subject: [PATCH 22/83] Hide m.call.candidates in the timeline by default. And handle them correctly when all events are displayed --- matrix-sdk-android/src/main/res/values/strings.xml | 2 +- .../room/detail/timeline/action/MessageActionsViewModel.kt | 1 + .../room/detail/timeline/factory/TimelineItemFactory.kt | 7 +++++-- .../room/detail/timeline/format/NoticeEventFormatter.kt | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/matrix-sdk-android/src/main/res/values/strings.xml b/matrix-sdk-android/src/main/res/values/strings.xml index a34c3a1f9f..dc01fc0dfc 100644 --- a/matrix-sdk-android/src/main/res/values/strings.xml +++ b/matrix-sdk-android/src/main/res/values/strings.xml @@ -42,7 +42,7 @@ %s placed a video call. You placed a video call. %s placed a voice call. - You placed a voice call. + %s sent data to setup the call. %s answered the call. You answered the call. %s ended the call. diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/action/MessageActionsViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/action/MessageActionsViewModel.kt index 909169e7b0..964eb40fc7 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/action/MessageActionsViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/action/MessageActionsViewModel.kt @@ -184,6 +184,7 @@ class MessageActionsViewModel @AssistedInject constructor(@Assisted EventType.STATE_ROOM_CANONICAL_ALIAS, EventType.STATE_ROOM_HISTORY_VISIBILITY, EventType.CALL_INVITE, + EventType.CALL_CANDIDATES, EventType.CALL_HANGUP, EventType.CALL_ANSWER -> { noticeEventFormatter.format(timelineEvent) diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/factory/TimelineItemFactory.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/factory/TimelineItemFactory.kt index c81a945bc7..462caf8e97 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/factory/TimelineItemFactory.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/factory/TimelineItemFactory.kt @@ -80,12 +80,15 @@ class TimelineItemFactory @Inject constructor(private val messageItemFactory: Me EventType.KEY_VERIFICATION_START, EventType.KEY_VERIFICATION_KEY, EventType.KEY_VERIFICATION_READY, - EventType.KEY_VERIFICATION_MAC -> { + EventType.KEY_VERIFICATION_MAC, + EventType.CALL_CANDIDATES -> { // TODO These are not filtered out by timeline when encrypted // For now manually ignore if (userPreferencesProvider.shouldShowHiddenEvents()) { noticeItemFactory.create(event, highlight, callback) - } else null + } else { + null + } } EventType.KEY_VERIFICATION_CANCEL, EventType.KEY_VERIFICATION_DONE -> { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt index c1ec2d1cac..3d3581e03b 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt @@ -68,6 +68,7 @@ class NoticeEventFormatter @Inject constructor(private val sessionHolder: Active EventType.STATE_ROOM_TOMBSTONE -> formatRoomTombstoneEvent(timelineEvent.root, timelineEvent.senderInfo.disambiguatedDisplayName) EventType.STATE_ROOM_POWER_LEVELS -> formatRoomPowerLevels(timelineEvent.root, timelineEvent.senderInfo.disambiguatedDisplayName) EventType.CALL_INVITE, + EventType.CALL_CANDIDATES, EventType.CALL_HANGUP, EventType.CALL_ANSWER -> formatCallEvent(type, timelineEvent.root, timelineEvent.senderInfo.disambiguatedDisplayName) EventType.MESSAGE, @@ -237,7 +238,7 @@ class NoticeEventFormatter @Inject constructor(private val sessionHolder: Active private fun formatCallEvent(type: String, event: Event, senderName: String?): CharSequence? { return when (type) { - EventType.CALL_INVITE -> { + EventType.CALL_INVITE -> { val content = event.getClearContent().toModel() ?: return null val isVideoCall = content.offer?.sdp == CallInviteContent.Offer.SDP_VIDEO return if (isVideoCall) { From 94ea857738dedad0054b0b710dd77ebe4fde2590 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 28 May 2020 23:53:31 +0200 Subject: [PATCH 23/83] Fix icons tint, esp in dark theme --- vector/src/main/res/menu/menu_timeline.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index ce20c59290..19db06d89f 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -2,11 +2,13 @@ + @@ -20,6 +22,7 @@ android:icon="@drawable/ic_phone" android:title="@string/call" android:visible="false" + app:iconTint="@color/riotx_accent" app:showAsAction="always" tools:visible="true" /> From 928da82dde8db291eb1bed13b8cd0099144641d0 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Fri, 29 May 2020 10:54:09 +0200 Subject: [PATCH 24/83] Make menu item live --- .../android/api/session/room/model/RoomSummary.kt | 3 +++ .../session/room/call/DefaultRoomCallService.kt | 4 +--- .../home/room/detail/RoomDetailFragment.kt | 15 +++++++++++++-- .../home/room/detail/RoomDetailViewModel.kt | 7 +++++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/RoomSummary.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/RoomSummary.kt index 3c95fd47fc..efe3899ad1 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/RoomSummary.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/RoomSummary.kt @@ -62,6 +62,9 @@ data class RoomSummary constructor( val isFavorite: Boolean get() = tags.any { it.name == RoomTag.ROOM_TAG_FAVOURITE } + val canStartCall: Boolean + get() = isDirect && joinedMembersCount == 2 + companion object { const val NOT_IN_BREADCRUMBS = -1 } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt index bac45ab5dc..c23d8de37f 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/call/DefaultRoomCallService.kt @@ -33,8 +33,6 @@ internal class DefaultRoomCallService @AssistedInject constructor( } override fun canStartCall(): Boolean { - return roomGetter.getRoom(roomId)?.roomSummary()?.let { - it.isDirect && it.joinedMembersCount == 2 - }.orFalse() + return roomGetter.getRoom(roomId)?.roomSummary()?.canStartCall.orFalse() } } diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index b31ab35085..f340a48bd7 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -489,9 +489,20 @@ class RoomDetailFragment @Inject constructor( roomDetailViewModel.getOtherUserIds()?.firstOrNull()?.let { webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) } - return true + R.id.resend_all -> { + roomDetailViewModel.handle(RoomDetailAction.ResendAll) + true + } + R.id.voice_call, + R.id.video_call -> { + roomDetailViewModel.getOtherUserIds()?.firstOrNull()?.let { + // TODO CALL We should check/ask for permission here first + webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) + } + true + } + else -> super.onOptionsItemSelected(item) } - return super.onOptionsItemSelected(item) } private fun displayDisabledIntegrationDialog() { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index 5ec8d207f4..cfdb396a95 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -30,6 +30,7 @@ import com.squareup.inject.assisted.AssistedInject import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.MatrixPatterns import im.vector.matrix.android.api.NoOpMatrixCallback +import im.vector.matrix.android.api.extensions.orFalse import im.vector.matrix.android.api.query.QueryStringValue import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.events.model.EventType @@ -121,8 +122,7 @@ class RoomDetailViewModel @AssistedInject constructor( } private var timelineEvents = PublishRelay.create>() - var timeline = room.createTimeline(eventId, timelineSettings) - private set + val timeline = room.createTimeline(eventId, timelineSettings) // Slot to keep a pending action during permission request var pendingAction: RoomDetailAction? = null @@ -135,6 +135,7 @@ class RoomDetailViewModel @AssistedInject constructor( private var trackUnreadMessages = AtomicBoolean(false) private var mostRecentDisplayedEvent: TimelineEvent? = null + private var canDoCall = false @AssistedInject.Factory interface Factory { @@ -1003,6 +1004,8 @@ class RoomDetailViewModel @AssistedInject constructor( val typingRoomMembers = typingHelper.toTypingRoomMembers(async.invoke()?.typingRoomMemberIds.orEmpty(), room) + canDoCall = async.invoke()?.canStartCall.orFalse() + copy( asyncRoomSummary = async, typingRoomMembers = typingRoomMembers, From ae762aa9283e31b31ee51278d80dde42461fb60e Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Fri, 29 May 2020 11:07:03 +0200 Subject: [PATCH 25/83] Cleanup --- .../java/im/vector/matrix/android/api/session/call/VoipApi.kt | 1 + .../riotx/features/home/room/detail/RoomDetailViewModel.kt | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt index dc92af8023..4a2b9cabef 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt @@ -20,6 +20,7 @@ import im.vector.matrix.android.internal.network.NetworkConstants import retrofit2.Call import retrofit2.http.GET +// TODO Move to internal internal interface VoipApi { @GET(NetworkConstants.URI_API_PREFIX_PATH_R0 + "voip/turnServer") diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index cfdb396a95..0894fbf004 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -369,8 +369,8 @@ class RoomDetailViewModel @AssistedInject constructor( } fun isMenuItemVisible(@IdRes itemId: Int) = when (itemId) { - R.id.clear_message_queue -> - /* For now always disable on production, worker cancellation is not working properly */ + R.id.clear_message_queue -> + // For now always disable when not in developer mode, worker cancellation is not working properly timeline.pendingEventCount() > 0 && vectorPreferences.developerMode() R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 From 8f5918de4da4d97d1da1ca2e62623c64f3d58bec Mon Sep 17 00:00:00 2001 From: onurays Date: Mon, 1 Jun 2020 10:41:26 +0300 Subject: [PATCH 26/83] Cleanup unused code. --- .../riotx/features/call/VectorCallActivity.kt | 254 +----------------- .../call/WebRtcPeerConnectionManager.kt | 106 ++------ .../features/call/telecom/CallConnection.kt | 22 +- 3 files changed, 26 insertions(+), 356 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index c21251c9ed..e2a9888e82 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -31,6 +31,7 @@ import android.view.WindowManager import butterknife.BindView import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel +import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCallDetail import im.vector.riotx.R @@ -41,16 +42,11 @@ import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions import im.vector.riotx.features.call.service.CallHeadsUpService import io.reactivex.android.schedulers.AndroidSchedulers -import io.reactivex.subjects.PublishSubject import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.webrtc.EglBase -import org.webrtc.IceCandidate -import org.webrtc.MediaStream import org.webrtc.RendererCommon -import org.webrtc.SessionDescription import org.webrtc.SurfaceViewRenderer -import org.webrtc.VideoTrack import javax.inject.Inject @Parcelize @@ -61,7 +57,7 @@ data class CallArgs( val isVideoCall: Boolean ) : Parcelable -class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Listener { +class VectorCallActivity : VectorBaseActivity() { override fun getLayoutRes() = R.layout.activity_call @@ -85,14 +81,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis private var rootEglBase: EglBase? = null -// private var peerConnectionFactory: PeerConnectionFactory? = null - - // private var peerConnection: PeerConnection? = null - -// private var remoteVideoTrack: VideoTrack? = null - - private val iceCandidateSource: PublishSubject = PublishSubject.create() - var callHeadsUpService: CallHeadsUpService? = null private val serviceConnection = object : ServiceConnection { override fun onServiceDisconnected(name: ComponentName?) { @@ -129,6 +117,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + tryThis { unbindService(serviceConnection) } bindService(Intent(this, CallHeadsUpService::class.java), serviceConnection, 0) if (intent.hasExtra(MvRx.KEY_ARG)) { @@ -153,15 +142,10 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis handleViewEvents(it) } .disposeOnDestroy() -// -// if (isFirstCreation()) { -// -// } if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, CAPTURE_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { start() } - peerConnectionManager.listener = this } override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { @@ -185,165 +169,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis pipRenderer.setZOrderMediaOverlay(true) pipRenderer.setEnableHardwareScaler(true /* enabled */) fullscreenRenderer.setEnableHardwareScaler(true /* enabled */) - // Start with local feed in fullscreen and swap it to the pip when the call is connected. - // setSwappedFeeds(true /* isSwappedFeeds */); - if (isFirstCreation()) { - //peerConnectionManager.createPeerConnectionFactory() - - /* - val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) - val frontCamera = cameraIterator.deviceNames - ?.firstOrNull { cameraIterator.isFrontFacing(it) } - ?: cameraIterator.deviceNames?.first() - ?: return true - val videoCapturer = cameraIterator.createCapturer(frontCamera, null) - - - - peerConnectionManager.createPeerConnection(videoCapturer, iceServers) - */ - - //peerConnectionManager.startCall() - } -// PeerConnectionFactory.initialize(PeerConnectionFactory -// .InitializationOptions.builder(applicationContext) -// .createInitializationOptions() -// ) - -// val options = PeerConnectionFactory.Options() -// val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( -// rootEglBase!!.eglBaseContext, /* enableIntelVp8Encoder */ -// true, /* enableH264HighProfile */ -// true) -// val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(rootEglBase!!.eglBaseContext) -// -// peerConnectionFactory = PeerConnectionFactory.builder() -// .setOptions(options) -// .setVideoEncoderFactory(defaultVideoEncoderFactory) -// .setVideoDecoderFactory(defaultVideoDecoderFactory) -// .createPeerConnectionFactory() - -// val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) -// val frontCamera = cameraIterator.deviceNames -// ?.firstOrNull { cameraIterator.isFrontFacing(it) } -// ?: cameraIterator.deviceNames?.first() -// ?: return true -// val videoCapturer = cameraIterator.createCapturer(frontCamera, null) -// -// // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object -// val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) -// -// val videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) -// videoCapturer.initialize(surfaceTextureHelper, this, videoSource!!.capturerObserver) -// videoCapturer.startCapture(1280, 720, 30) -// -// -// val localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) -// -// // create a local audio track -// val audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) -// val audioTrack = peerConnectionFactory?.createAudioTrack("ARDAMSa0", audioSource) - - pipRenderer.setMirror(true) -// localVideoTrack?.addSink(pipRenderer) - - /* - { - "username": "1586847781:@valere35:matrix.org", - "password": "ZzbqbqfT9O2G3WpCpesdts2lyns=", - "ttl": 86400.0, - "uris": ["turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp"] - } - */ - -// val iceServers = ArrayList().apply { -// listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { -// add( -// PeerConnection.IceServer.builder(it) -// .setUsername("1586847781:@valere35:matrix.org") -// .setPassword("ZzbqbqfT9O2G3WpCpesdts2lyns=") -// .createIceServer() -// ) -// } -// } -// -// val iceCandidateSource: PublishSubject = PublishSubject.create() -// -// iceCandidateSource -// .buffer(400, TimeUnit.MILLISECONDS) -// .subscribe { -// // omit empty :/ -// if (it.isNotEmpty()) { -// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) -// } -// } -// .disposeOnDestroy() -// -// peerConnection = peerConnectionFactory?.createPeerConnection( -// iceServers, -// object : PeerConnectionObserverAdapter() { -// override fun onIceCandidate(p0: IceCandidate?) { -// p0?.let { -// iceCandidateSource.onNext(it) -// } -// } -// -// override fun onAddStream(mediaStream: MediaStream?) { -// runOnUiThread { -// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> -// remoteVideoTrack = videoTrack -// remoteVideoTrack?.setEnabled(true) -// remoteVideoTrack?.addSink(fullscreenRenderer) -// } -// } -// } -// -// override fun onRemoveStream(mediaStream: MediaStream?) { -// remoteVideoTrack = null -// } -// -// override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { -// if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { -// // TODO prompt something? -// finish() -// } -// } -// } -// ) -// -// val localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? -// localMediaStream?.addTrack(localVideoTrack) -// localMediaStream?.addTrack(audioTrack) -// -// val constraints = MediaConstraints() -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) -// -// peerConnection?.addStream(localMediaStream) -// -// peerConnection?.createOffer(object : SdpObserver { -// override fun onSetFailure(p0: String?) { -// Timber.v("## VOIP onSetFailure $p0") -// } -// -// override fun onSetSuccess() { -// Timber.v("## VOIP onSetSuccess") -// } -// -// override fun onCreateSuccess(sessionDescription: SessionDescription) { -// Timber.v("## VOIP onCreateSuccess $sessionDescription") -// peerConnection?.setLocalDescription(object : SdpObserverAdapter() { -// override fun onSetSuccess() { -// callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) -// } -// }, sessionDescription) -// } -// -// override fun onCreateFailure(p0: String?) { -// Timber.v("## VOIP onCreateFailure $p0") -// } -// }, constraints) peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) return false @@ -351,16 +177,13 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis override fun onDestroy() { peerConnectionManager.detachRenderers() - peerConnectionManager.listener = this + tryThis { unbindService(serviceConnection) } super.onDestroy() } private fun handleViewEvents(event: VectorCallViewEvents?) { when (event) { is VectorCallViewEvents.CallAnswered -> { - //val sdp = SessionDescription(SessionDescription.Type.ANSWER, event.content.answer.sdp) - //peerConnectionManager.answerReceived("", sdp) -// peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) } is VectorCallViewEvents.CallHangup -> { finish() @@ -368,53 +191,10 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } } -// @TargetApi(17) -// private fun getDisplayMetrics(): DisplayMetrics? { -// val displayMetrics = DisplayMetrics() -// val windowManager = application.getSystemService(Context.WINDOW_SERVICE) as WindowManager -// windowManager.defaultDisplay.getRealMetrics(displayMetrics) -// return displayMetrics -// } - -// @TargetApi(21) -// private fun startScreenCapture() { -// val mediaProjectionManager: MediaProjectionManager = application.getSystemService( -// Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager -// startActivityForResult( -// mediaProjectionManager.createScreenCaptureIntent(), CAPTURE_PERMISSION_REQUEST_CODE) -// } -// -// override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { -// if (requestCode != CAPTURE_PERMISSION_REQUEST_CODE) { -// super.onActivityResult(requestCode, resultCode, data) -// } -// // mediaProjectionPermissionResultCode = resultCode; -// // mediaProjectionPermissionResultData = data; -// // startCall(); -// } - companion object { private const val CAPTURE_PERMISSION_REQUEST_CODE = 1 -// private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { -// // add all existing audio filters to avoid having echos -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) -// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) -// } - fun newIntent(context: Context, mxCall: MxCallDetail): Intent { return Intent(context, VectorCallActivity::class.java).apply { flags = Intent.FLAG_ACTIVITY_NEW_TASK @@ -422,30 +202,4 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } } } - - override fun addLocalIceCandidate(candidates: IceCandidate) { - iceCandidateSource.onNext(candidates) - } - - override fun addRemoteVideoTrack(videoTrack: VideoTrack) { - runOnUiThread { - videoTrack.setEnabled(true) - videoTrack.addSink(fullscreenRenderer) - } - } - - override fun addLocalVideoTrack(videoTrack: VideoTrack) { - runOnUiThread { - videoTrack.addSink(pipRenderer) - } - } - - override fun removeRemoteVideoStream(mediaStream: MediaStream) { - } - - override fun onDisconnect() { - } - - override fun sendOffer(sessionDescription: SessionDescription) { - } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index ec5f6f2048..4e70aef5b5 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -69,19 +69,8 @@ class WebRtcPeerConnectionManager @Inject constructor( private val sessionHolder: ActiveSessionHolder ) : CallsListener { - interface Listener { - fun addLocalIceCandidate(candidates: IceCandidate) - fun addRemoteVideoTrack(videoTrack: VideoTrack) - fun addLocalVideoTrack(videoTrack: VideoTrack) - fun removeRemoteVideoStream(mediaStream: MediaStream) - fun onDisconnect() - fun sendOffer(sessionDescription: SessionDescription) - } - var localMediaStream: MediaStream? = null - var listener: Listener? = null - // *Comments copied from webrtc demo app* // Executor thread is started once and is used for all // peer connection API calls to ensure new peer connection factory is @@ -102,7 +91,7 @@ class WebRtcPeerConnectionManager @Inject constructor( private var videoSource: VideoSource? = null private var audioSource: AudioSource? = null - private var audioTrack: AudioTrack? = null + private var localAudioTrack: AudioTrack? = null private var videoCapturer: VideoCapturer? = null @@ -171,60 +160,6 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, observer) } - // TODO REMOVE THIS FUNCTION - private fun createPeerConnection(videoCapturer: VideoCapturer) { - executor.execute { - Timber.v("## VOIP PeerConnectionFactory.createPeerConnection $peerConnectionFactory...") - // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object - val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) - - videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) - Timber.v("## VOIP Local video source created") - videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) - videoCapturer.startCapture(1280, 720, 30) - - localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { - Timber.v("## VOIP Local video track created") -// localSurfaceRenderer?.get()?.let { surface -> -// // it.addSink(surface) -// // } - } - - // create a local audio track - Timber.v("## VOIP create local audio track") - audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) - -// pipRenderer.setMirror(true) -// localVideoTrack?.addSink(pipRenderer) -// - -// val iceCandidateSource: PublishSubject = PublishSubject.create() -// -// iceCandidateSource -// .buffer(400, TimeUnit.MILLISECONDS) -// .subscribe { -// // omit empty :/ -// if (it.isNotEmpty()) { -// listener.addLocalIceCandidate() -// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) -// } -// } -// .disposeOnDestroy() - - localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? - localMediaStream?.addTrack(localVideoTrack) - localMediaStream?.addTrack(audioTrack) - -// val constraints = MediaConstraints() -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) - - Timber.v("## VOIP add local stream to peer connection") - peerConnection?.addStream(localMediaStream) - } - } - private fun startCall() { createPeerConnectionFactory() createPeerConnection(object : PeerConnectionObserverAdapter() { @@ -244,15 +179,12 @@ class WebRtcPeerConnectionManager @Inject constructor( mediaStream.videoTracks?.firstOrNull()?.let { videoTrack -> remoteVideoTrack = videoTrack remoteVideoTrack?.setEnabled(true) - remoteVideoTrack?.addSink(remoteViewRenderer) + remoteViewRenderer?.let { remoteVideoTrack?.addSink(it) } } } } override fun onRemoveStream(mediaStream: MediaStream?) { - mediaStream?.let { - listener?.removeRemoteVideoStream(it) - } remoteSurfaceRenderer?.get()?.let { remoteVideoTrack?.removeSink(it) } @@ -262,7 +194,7 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { Timber.v("## VOIP onIceConnectionChange $p0") if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { - listener?.onDisconnect() + endCall() } } }) @@ -313,8 +245,12 @@ class WebRtcPeerConnectionManager @Inject constructor( fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { this.localViewRenderer = localViewRenderer this.remoteViewRenderer = remoteViewRenderer + this.localSurfaceRenderer = WeakReference(localViewRenderer) + this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + localAudioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + localAudioTrack?.setEnabled(true) localViewRenderer.setMirror(true) localVideoTrack?.addSink(localViewRenderer) @@ -334,27 +270,27 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP Local video source created") videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) videoCapturer.startCapture(1280, 720, 30) - localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { - Timber.v("## VOIP Local video track created") - localSurfaceRenderer?.get()?.let { surface -> - it.addSink(surface) - } + localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) + Timber.v("## VOIP Local video track created") + localSurfaceRenderer?.get()?.let { surface -> + localVideoTrack?.addSink(surface) } + localVideoTrack?.setEnabled(true) + + localVideoTrack?.addSink(localViewRenderer) localMediaStream?.addTrack(localVideoTrack) + remoteVideoTrack?.let { + it.setEnabled(true) + it.addSink(remoteViewRenderer) + } } - localVideoTrack?.addSink(localViewRenderer) - remoteVideoTrack?.let { - it.setEnabled(true) - it.addSink(remoteViewRenderer) - } - localMediaStream?.addTrack(audioTrack) + localMediaStream?.addTrack(localAudioTrack) Timber.v("## VOIP add local stream to peer connection") peerConnection?.addStream(localMediaStream) - localSurfaceRenderer = WeakReference(localViewRenderer) - remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + startCall() } fun detachRenderers() { diff --git a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt index 6f0c121b3d..6273abddfd 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt @@ -23,10 +23,6 @@ import android.telecom.DisconnectCause import androidx.annotation.RequiresApi import im.vector.riotx.features.call.VectorCallViewModel import im.vector.riotx.features.call.WebRtcPeerConnectionManager -import org.webrtc.IceCandidate -import org.webrtc.MediaStream -import org.webrtc.SessionDescription -import org.webrtc.VideoTrack import timber.log.Timber import javax.inject.Inject @@ -34,7 +30,7 @@ import javax.inject.Inject private val context: Context, private val roomId: String, val callId: String -) : Connection(), WebRtcPeerConnectionManager.Listener { +) : Connection() { @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager @Inject lateinit var callViewModel: VectorCallViewModel @@ -113,20 +109,4 @@ import javax.inject.Inject //peerConnectionManager.startCall() */ } - - override fun addLocalIceCandidate(candidates: IceCandidate) { - } - - override fun addRemoteVideoTrack(videoTrack: VideoTrack) { - } - - override fun addLocalVideoTrack(videoTrack: VideoTrack) { - } - - override fun removeRemoteVideoStream(mediaStream: MediaStream) { - } - - override fun sendOffer(sessionDescription: SessionDescription) { - - } } From 4b85e39e3ebe19ff180ae007d6e7dda31d93fdcd Mon Sep 17 00:00:00 2001 From: onurays Date: Wed, 3 Jun 2020 11:36:21 +0300 Subject: [PATCH 27/83] Implementation of turn server api. --- .../android/api/session/call/CallService.kt | 3 +- .../internal/session/SessionComponent.kt | 4 +- .../android/internal/session/SessionModule.kt | 2 - .../internal/session/call/CallModule.kt | 46 +++++++++++++++++++ .../session/call/DefaultCallService.kt | 23 ++++++++-- .../session/call/GetTurnServerTask.kt | 38 +++++++++++++++ 6 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt index d94e84c7b9..805bfd53ed 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -17,10 +17,11 @@ package im.vector.matrix.android.api.session.call import im.vector.matrix.android.api.MatrixCallback +import im.vector.matrix.android.api.util.Cancelable interface CallService { - fun getTurnServer(callback: MatrixCallback) + fun getTurnServer(callback: MatrixCallback): Cancelable /** * Create an outgoing call diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionComponent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionComponent.kt index 5b64f2a60a..b95595ed23 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionComponent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionComponent.kt @@ -30,6 +30,7 @@ import im.vector.matrix.android.internal.di.SessionAssistedInjectModule import im.vector.matrix.android.internal.network.NetworkConnectivityChecker import im.vector.matrix.android.internal.session.account.AccountModule import im.vector.matrix.android.internal.session.cache.CacheModule +import im.vector.matrix.android.internal.session.call.CallModule import im.vector.matrix.android.internal.session.content.ContentModule import im.vector.matrix.android.internal.session.content.UploadContentWorker import im.vector.matrix.android.internal.session.filter.FilterModule @@ -83,7 +84,8 @@ import im.vector.matrix.android.internal.util.MatrixCoroutineDispatchers AccountDataModule::class, ProfileModule::class, SessionAssistedInjectModule::class, - AccountModule::class + AccountModule::class, + CallModule::class ] ) @SessionScope diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt index c670609411..5331b9f471 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt @@ -275,6 +275,4 @@ internal abstract class SessionModule { @Binds abstract fun bindSharedSecretStorageService(service: DefaultSharedSecretStorageService): SharedSecretStorageService - @Binds - abstract fun bindCallService(service:DefaultCallService): CallService } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt new file mode 100644 index 0000000000..4273c4cfe9 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import dagger.Binds +import dagger.Module +import dagger.Provides +import im.vector.matrix.android.api.session.call.CallService +import im.vector.matrix.android.api.session.call.VoipApi +import im.vector.matrix.android.internal.session.SessionScope +import retrofit2.Retrofit + +@Module +internal abstract class CallModule { + + @Module + companion object { + @Provides + @JvmStatic + @SessionScope + fun providesVoipApi(retrofit: Retrofit): VoipApi { + return retrofit.create(VoipApi::class.java) + } + } + + + @Binds + abstract fun bindCallService(service:DefaultCallService): CallService + + @Binds + abstract fun bindTurnServerTask(task: DefaultGetTurnServerTask): GetTurnServerTask +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index b0c9ccc2c4..f45c06fdf4 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -28,11 +28,14 @@ import im.vector.matrix.android.api.session.events.model.toModel import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.matrix.android.api.util.Cancelable import im.vector.matrix.android.internal.di.UserId import im.vector.matrix.android.internal.session.SessionScope import im.vector.matrix.android.internal.session.call.model.MxCallImpl import im.vector.matrix.android.internal.session.room.send.LocalEchoEventFactory import im.vector.matrix.android.internal.session.room.send.RoomEventSender +import im.vector.matrix.android.internal.task.TaskExecutor +import im.vector.matrix.android.internal.task.configureWith import java.util.UUID import javax.inject.Inject @@ -41,13 +44,27 @@ internal class DefaultCallService @Inject constructor( @UserId private val userId: String, private val localEchoEventFactory: LocalEchoEventFactory, - private val roomEventSender: RoomEventSender + private val roomEventSender: RoomEventSender, + private val taskExecutor: TaskExecutor, + private val turnServerTask: GetTurnServerTask ) : CallService { private val callListeners = mutableSetOf() - override fun getTurnServer(callback: MatrixCallback) { - TODO("not implemented") // To change body of created functions use File | Settings | File Templates. + override fun getTurnServer(callback: MatrixCallback): Cancelable { + return turnServerTask + .configureWith(GetTurnServerTask.Params) { + this.callback = object : MatrixCallback { + override fun onSuccess(data: TurnServer) { + callback.onSuccess(data) + } + + override fun onFailure(failure: Throwable) { + callback.onFailure(failure) + } + } + } + .executeBy(taskExecutor) } override fun createOutgoingCall(roomId: String, otherUserId: String, isVideoCall: Boolean): MxCall { diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt new file mode 100644 index 0000000000..d9e17d90eb --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import im.vector.matrix.android.api.session.call.TurnServer +import im.vector.matrix.android.api.session.call.VoipApi +import im.vector.matrix.android.internal.network.executeRequest +import im.vector.matrix.android.internal.task.Task +import org.greenrobot.eventbus.EventBus +import javax.inject.Inject + +internal abstract class GetTurnServerTask : Task { + object Params +} + +internal class DefaultGetTurnServerTask @Inject constructor(private val voipAPI: VoipApi, + private val eventBus: EventBus) : GetTurnServerTask() { + + override suspend fun execute(params: Params): TurnServer { + return executeRequest(eventBus) { + apiCall = voipAPI.getTurnServer() + } + } +} From 79f804b2d4981fc4719dc770ccda0baecdd48660 Mon Sep 17 00:00:00 2001 From: onurays Date: Wed, 3 Jun 2020 11:36:59 +0300 Subject: [PATCH 28/83] Use single sdp and stream observer. --- .../call/WebRtcPeerConnectionManager.kt | 338 ++++++++++-------- 1 file changed, 186 insertions(+), 152 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 4e70aef5b5..d5f5f90055 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -22,11 +22,13 @@ import android.content.Intent import android.content.ServiceConnection import android.os.IBinder import androidx.core.content.ContextCompat +import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCall import im.vector.matrix.android.api.session.call.MxCallDetail +import im.vector.matrix.android.api.session.call.TurnServer import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent @@ -38,6 +40,7 @@ import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.Camera1Enumerator import org.webrtc.Camera2Enumerator +import org.webrtc.DataChannel import org.webrtc.DefaultVideoDecoderFactory import org.webrtc.DefaultVideoEncoderFactory import org.webrtc.IceCandidate @@ -45,7 +48,7 @@ import org.webrtc.MediaConstraints import org.webrtc.MediaStream import org.webrtc.PeerConnection import org.webrtc.PeerConnectionFactory -import org.webrtc.SdpObserver +import org.webrtc.RtpReceiver import org.webrtc.SessionDescription import org.webrtc.SurfaceTextureHelper import org.webrtc.SurfaceViewRenderer @@ -71,18 +74,17 @@ class WebRtcPeerConnectionManager @Inject constructor( var localMediaStream: MediaStream? = null - // *Comments copied from webrtc demo app* - // Executor thread is started once and is used for all - // peer connection API calls to ensure new peer connection factory is - // created on the same thread as previously destroyed factory. private val executor = Executors.newSingleThreadExecutor() private val rootEglBase by lazy { EglUtils.rootEglBase } private var peerConnectionFactory: PeerConnectionFactory? = null - private var peerConnection: PeerConnection? = null + private var localSdp: SessionDescription? = null + private var sdpObserver = SdpObserver() + private var streamObserver = StreamObserver() + private var localViewRenderer: SurfaceViewRenderer? = null private var remoteViewRenderer: SurfaceViewRenderer? = null @@ -115,37 +117,49 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun createPeerConnectionFactory() { - if (peerConnectionFactory == null) { - Timber.v("## VOIP createPeerConnectionFactory") - val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also { - Timber.e("## VOIP No EGL BASE") - } - - Timber.v("## VOIP PeerConnectionFactory.initialize") - PeerConnectionFactory.initialize(PeerConnectionFactory - .InitializationOptions.builder(context.applicationContext) - .createInitializationOptions() - ) - - val options = PeerConnectionFactory.Options() - val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( - eglBaseContext, - /* enableIntelVp8Encoder */ - true, - /* enableH264HighProfile */ - true) - val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) - - Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") - peerConnectionFactory = PeerConnectionFactory.builder() - .setOptions(options) - .setVideoEncoderFactory(defaultVideoEncoderFactory) - .setVideoDecoderFactory(defaultVideoDecoderFactory) - .createPeerConnectionFactory() + Timber.v("## VOIP createPeerConnectionFactory") + val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also { + Timber.e("## VOIP No EGL BASE") } + + Timber.v("## VOIP PeerConnectionFactory.initialize") + PeerConnectionFactory.initialize(PeerConnectionFactory + .InitializationOptions.builder(context.applicationContext) + .createInitializationOptions() + ) + + val options = PeerConnectionFactory.Options() + val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( + eglBaseContext, + /* enableIntelVp8Encoder */ + true, + /* enableH264HighProfile */ + true) + val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) + Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") + peerConnectionFactory = PeerConnectionFactory.builder() + .setOptions(options) + .setVideoEncoderFactory(defaultVideoEncoderFactory) + .setVideoDecoderFactory(defaultVideoDecoderFactory) + .createPeerConnectionFactory() } - private fun createPeerConnection(observer: PeerConnectionObserverAdapter) { + private fun createPeerConnection(turnServer: TurnServer?) { + val iceServers = mutableListOf().apply { + turnServer?.let { server -> + server.uris?.forEach { uri -> + add( + PeerConnection + .IceServer + .builder(uri) + .setUsername(server.username) + .setPassword(server.password) + .createIceServer() + ) + } + } + } + /* val iceServers = ArrayList().apply { listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { add( @@ -156,49 +170,12 @@ class WebRtcPeerConnectionManager @Inject constructor( ) } } + */ Timber.v("## VOIP creating peer connection... ") - peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, observer) + peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, streamObserver) } private fun startCall() { - createPeerConnectionFactory() - createPeerConnection(object : PeerConnectionObserverAdapter() { - override fun onIceCandidate(p0: IceCandidate?) { - Timber.v("## VOIP onIceCandidate local $p0") - p0?.let { iceCandidateSource.onNext(it) } - } - - override fun onAddStream(mediaStream: MediaStream?) { - Timber.v("## VOIP onAddStream remote $mediaStream") - mediaStream?.videoTracks?.firstOrNull()?.let { - remoteVideoTrack = it - remoteSurfaceRenderer?.get()?.let { surface -> - it.setEnabled(true) - it.addSink(surface) - } - mediaStream.videoTracks?.firstOrNull()?.let { videoTrack -> - remoteVideoTrack = videoTrack - remoteVideoTrack?.setEnabled(true) - remoteViewRenderer?.let { remoteVideoTrack?.addSink(it) } - } - } - } - - override fun onRemoveStream(mediaStream: MediaStream?) { - remoteSurfaceRenderer?.get()?.let { - remoteVideoTrack?.removeSink(it) - } - remoteVideoTrack = null - } - - override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { - Timber.v("## VOIP onIceConnectionChange $p0") - if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { - endCall() - } - } - }) - iceCandidateDisposable = iceCandidateSource .buffer(400, TimeUnit.MILLISECONDS) .subscribe { @@ -208,38 +185,26 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall?.sendLocalIceCandidates(it) } } + + executor.execute { + sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { + override fun onSuccess(data: TurnServer?) { + createPeerConnectionFactory() + createPeerConnection(data) + } + }) + } } private fun sendSdpOffer() { - val constraints = MediaConstraints() - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) + executor.execute { + val constraints = MediaConstraints() + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) - Timber.v("## VOIP creating offer...") - peerConnection?.createOffer(object : SdpObserver { - override fun onSetFailure(p0: String?) { - Timber.v("## VOIP onSetFailure $p0") - } - - override fun onSetSuccess() { - Timber.v("## VOIP onSetSuccess") - } - - override fun onCreateSuccess(sessionDescription: SessionDescription) { - Timber.v("## VOIP onCreateSuccess $sessionDescription will set local description") - peerConnection?.setLocalDescription(object : SdpObserverAdapter() { - override fun onSetSuccess() { - Timber.v("## setLocalDescription success") - Timber.v("## sending offer") - currentCall?.offerSdp(sessionDescription) - } - }, sessionDescription) - } - - override fun onCreateFailure(p0: String?) { - Timber.v("## VOIP onCreateFailure $p0") - } - }, constraints) + Timber.v("## VOIP creating offer...") + peerConnection?.createOffer(sdpObserver, constraints) + } } fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { @@ -289,8 +254,6 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP add local stream to peer connection") peerConnection?.addStream(localMediaStream) - - startCall() } fun detachRenderers() { @@ -367,8 +330,10 @@ class WebRtcPeerConnectionManager @Inject constructor( startHeadsUpService(mxCall) startCall() - val sdp = SessionDescription(SessionDescription.Type.OFFER, callInviteContent.offer?.sdp) - peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) + executor.execute { + val sdp = SessionDescription(SessionDescription.Type.OFFER, callInviteContent.offer?.sdp) + peerConnection?.setRemoteDescription(sdpObserver, sdp) + } } private fun startHeadsUpService(mxCall: MxCallDetail) { @@ -380,55 +345,19 @@ class WebRtcPeerConnectionManager @Inject constructor( fun answerCall() { if (currentCall != null) { - val constraints = MediaConstraints().apply { - mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) - } - - peerConnection?.createAnswer(object : SdpObserver { - override fun onCreateSuccess(sessionDescription: SessionDescription) { - Timber.v("## createAnswer onCreateSuccess") - - val sdp = SessionDescription(sessionDescription.type, sessionDescription.description) - - peerConnection?.setLocalDescription(object : SdpObserver { - override fun onSetSuccess() { - currentCall?.accept(sdp) - - currentCall?.let { - context.startActivity(VectorCallActivity.newIntent(context, it)) - } - } - - override fun onCreateSuccess(localSdp: SessionDescription) {} - - override fun onSetFailure(p0: String?) { - endCall() - } - - override fun onCreateFailure(p0: String?) { - endCall() - } - }, sdp) - } - - override fun onSetSuccess() { - Timber.v("## createAnswer onSetSuccess") - } - - override fun onSetFailure(error: String) { - Timber.v("answerCall.onSetFailure failed: $error") - endCall() - } - - override fun onCreateFailure(error: String) { - Timber.v("answerCall.onCreateFailure failed: $error") - endCall() - } - }, constraints) + executor.execute { createAnswer() } } } + private fun createAnswer() { + val constraints = MediaConstraints().apply { + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) + } + + peerConnection?.createAnswer(sdpObserver, constraints) + } + fun endCall() { currentCall?.hangUp() currentCall = null @@ -439,7 +368,7 @@ class WebRtcPeerConnectionManager @Inject constructor( executor.execute { Timber.v("## answerReceived") val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) - peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) + peerConnection?.setRemoteDescription(sdpObserver, sdp) } } @@ -447,4 +376,109 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall = null close() } + + private inner class SdpObserver : org.webrtc.SdpObserver { + + override fun onCreateSuccess(origSdp: SessionDescription) { + Timber.v("## VOIP SdpObserver onCreateSuccess") + if (localSdp != null) return + + executor.execute { + localSdp = SessionDescription(origSdp.type, origSdp.description) + peerConnection?.setLocalDescription(sdpObserver, localSdp) + } + } + + override fun onSetSuccess() { + Timber.v("## VOIP SdpObserver onSetSuccess") + executor.execute { + localSdp?.let { + if (currentCall?.isOutgoing == true) { + currentCall?.offerSdp(it) + } else { + currentCall?.accept(it) + currentCall?.let { context.startActivity(VectorCallActivity.newIntent(context, it)) } + } + } + } + } + + override fun onCreateFailure(error: String) { + Timber.v("## VOIP SdpObserver onCreateFailure: $error") + } + + override fun onSetFailure(error: String) { + Timber.v("## VOIP SdpObserver onSetFailure: $error") + } + } + + private inner class StreamObserver : PeerConnection.Observer { + override fun onIceCandidate(iceCandidate: IceCandidate) { + Timber.v("## VOIP StreamObserver onIceCandidate: $iceCandidate") + iceCandidateSource.onNext(iceCandidate) + } + + override fun onDataChannel(dc: DataChannel) { + Timber.v("## VOIP StreamObserver onDataChannel: ${dc.state()}") + } + + override fun onIceConnectionReceivingChange(receiving: Boolean) { + Timber.v("## VOIP StreamObserver onIceConnectionReceivingChange: $receiving") + } + + override fun onIceConnectionChange(newState: PeerConnection.IceConnectionState) { + when (newState) { + PeerConnection.IceConnectionState.CONNECTED -> Timber.v("## VOIP StreamObserver onIceConnectionChange.CONNECTED") + PeerConnection.IceConnectionState.DISCONNECTED -> { + Timber.v("## VOIP StreamObserver onIceConnectionChange.DISCONNECTED") + endCall() + } + PeerConnection.IceConnectionState.FAILED -> Timber.v("## VOIP StreamObserver onIceConnectionChange.FAILED") + else -> Timber.v("## VOIP StreamObserver onIceConnectionChange.$newState") + } + } + + override fun onAddStream(stream: MediaStream) { + Timber.v("## VOIP StreamObserver onAddStream: $stream") + executor.execute { + if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) return@execute + + if (stream.videoTracks.size == 1) { + remoteVideoTrack = stream.videoTracks.first() + remoteVideoTrack?.setEnabled(true) + remoteViewRenderer?.let { remoteVideoTrack?.addSink(it) } + } + } + } + + override fun onRemoveStream(stream: MediaStream) { + Timber.v("## VOIP StreamObserver onRemoveStream") + executor.execute { + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + remoteVideoTrack = null + } + } + + override fun onIceGatheringChange(newState: PeerConnection.IceGatheringState) { + Timber.v("## VOIP StreamObserver onIceGatheringChange: $newState") + } + + override fun onSignalingChange(newState: PeerConnection.SignalingState) { + Timber.v("## VOIP StreamObserver onSignalingChange: $newState") + } + + override fun onIceCandidatesRemoved(candidates: Array) { + Timber.v("## VOIP StreamObserver onIceCandidatesRemoved: ${candidates.contentToString()}") + } + + override fun onRenegotiationNeeded() { + Timber.v("## VOIP StreamObserver onRenegotiationNeeded") + } + + override fun onAddTrack(p0: RtpReceiver?, p1: Array?) { + Timber.v("## VOIP StreamObserver onAddTrack") + } + } } From 24cea5110e2d91264e870a11175f46ee7c424ff3 Mon Sep 17 00:00:00 2001 From: onurays Date: Thu, 4 Jun 2020 11:43:21 +0300 Subject: [PATCH 29/83] Show / hide call views according to call type. --- .../riotx/features/call/VectorCallActivity.kt | 23 +++++++++++++++---- vector/src/main/res/layout/activity_call.xml | 1 + 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index e2a9888e82..b4090fa948 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -28,6 +28,7 @@ import android.os.Parcelable import android.view.View import android.view.Window import android.view.WindowManager +import androidx.core.view.isVisible import butterknife.BindView import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel @@ -130,10 +131,7 @@ class VectorCallActivity : VectorBaseActivity() { finish() } - iv_end_call.setOnClickListener { - callViewModel.handle(VectorCallViewActions.EndCall) - finish() - } + configureCallViews() callViewModel.viewEvents .observe() @@ -148,6 +146,23 @@ class VectorCallActivity : VectorBaseActivity() { } } + private fun configureCallViews() { + if (callArgs.isVideoCall) { + iv_call_speaker.isVisible = false + iv_call_flip_camera.isVisible = true + iv_call_videocam_off.isVisible = true + } else { + iv_call_speaker.isVisible = true + iv_call_flip_camera.isVisible = false + iv_call_videocam_off.isVisible = false + } + + iv_end_call.setOnClickListener { + callViewModel.handle(VectorCallViewActions.EndCall) + finish() + } + } + override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { if (requestCode == CAPTURE_PERMISSION_REQUEST_CODE && allGranted(grantResults)) { start() diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index 3b158d8828..be6165815f 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -36,6 +36,7 @@ android:id="@+id/layout_call_actions" android:layout_width="match_parent" android:layout_height="80dp" + android:layout_marginBottom="48dp" app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintStart_toStartOf="parent" app:layout_constraintEnd_toEndOf="parent" From 4d288ddd552dc0f2bd3a078b80fd7feaa4f3853b Mon Sep 17 00:00:00 2001 From: onurays Date: Thu, 4 Jun 2020 12:03:35 +0300 Subject: [PATCH 30/83] Require turn server before creating PeerConnection. --- .../call/WebRtcPeerConnectionManager.kt | 138 +++++++++--------- 1 file changed, 73 insertions(+), 65 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index d5f5f90055..d35fd3eef8 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -142,12 +142,14 @@ class WebRtcPeerConnectionManager @Inject constructor( .setVideoEncoderFactory(defaultVideoEncoderFactory) .setVideoDecoderFactory(defaultVideoDecoderFactory) .createPeerConnectionFactory() + + attachViewRenderersInternal() } private fun createPeerConnection(turnServer: TurnServer?) { val iceServers = mutableListOf().apply { turnServer?.let { server -> - server.uris?.forEach { uri -> + server.uris?.forEach { uri -> add( PeerConnection .IceServer @@ -159,23 +161,11 @@ class WebRtcPeerConnectionManager @Inject constructor( } } } - /* - val iceServers = ArrayList().apply { - listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { - add( - PeerConnection.IceServer.builder(it) - .setUsername("xxxxx") - .setPassword("xxxxx") - .createIceServer() - ) - } - } - */ Timber.v("## VOIP creating peer connection... ") peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, streamObserver) } - private fun startCall() { + private fun startCall(turnServer: TurnServer?) { iceCandidateDisposable = iceCandidateSource .buffer(400, TimeUnit.MILLISECONDS) .subscribe { @@ -187,12 +177,8 @@ class WebRtcPeerConnectionManager @Inject constructor( } executor.execute { - sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { - override fun onSuccess(data: TurnServer?) { - createPeerConnectionFactory() - createPeerConnection(data) - } - }) + createPeerConnectionFactory() + createPeerConnection(turnServer) } } @@ -208,52 +194,62 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { - this.localViewRenderer = localViewRenderer - this.remoteViewRenderer = remoteViewRenderer - this.localSurfaceRenderer = WeakReference(localViewRenderer) - this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + executor.execute { + this.localViewRenderer = localViewRenderer + this.remoteViewRenderer = remoteViewRenderer + this.localSurfaceRenderer = WeakReference(localViewRenderer) + this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) - audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - localAudioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) - localAudioTrack?.setEnabled(true) - - localViewRenderer.setMirror(true) - localVideoTrack?.addSink(localViewRenderer) - - localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? - - if (currentCall?.isVideoCall == true) { - val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) - val frontCamera = cameraIterator.deviceNames - ?.firstOrNull { cameraIterator.isFrontFacing(it) } - ?: cameraIterator.deviceNames?.first() - - val videoCapturer = cameraIterator.createCapturer(frontCamera, null) - - videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) - val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) - Timber.v("## VOIP Local video source created") - videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) - videoCapturer.startCapture(1280, 720, 30) - localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) - Timber.v("## VOIP Local video track created") - localSurfaceRenderer?.get()?.let { surface -> - localVideoTrack?.addSink(surface) - } - localVideoTrack?.setEnabled(true) - - localVideoTrack?.addSink(localViewRenderer) - localMediaStream?.addTrack(localVideoTrack) - remoteVideoTrack?.let { - it.setEnabled(true) - it.addSink(remoteViewRenderer) + if (peerConnection != null) { + attachViewRenderersInternal() } } + } - localMediaStream?.addTrack(localAudioTrack) + private fun attachViewRenderersInternal() { + executor.execute { + audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + localAudioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + localAudioTrack?.setEnabled(true) - Timber.v("## VOIP add local stream to peer connection") - peerConnection?.addStream(localMediaStream) + localViewRenderer?.setMirror(true) + localVideoTrack?.addSink(localViewRenderer) + + localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? + + if (currentCall?.isVideoCall == true) { + val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) + val frontCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isFrontFacing(it) } + ?: cameraIterator.deviceNames?.first() + + val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + + videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + Timber.v("## VOIP Local video source created") + videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) + videoCapturer.startCapture(1280, 720, 30) + localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) + Timber.v("## VOIP Local video track created") + localSurfaceRenderer?.get()?.let { surface -> + localVideoTrack?.addSink(surface) + } + localVideoTrack?.setEnabled(true) + + localVideoTrack?.addSink(localViewRenderer) + localMediaStream?.addTrack(localVideoTrack) + remoteVideoTrack?.let { + it.setEnabled(true) + it.addSink(remoteViewRenderer) + } + } + + localMediaStream?.addTrack(localAudioTrack) + + Timber.v("## VOIP add local stream to peer connection") + peerConnection?.addStream(localMediaStream) + } } fun detachRenderers() { @@ -314,8 +310,12 @@ class WebRtcPeerConnectionManager @Inject constructor( startHeadsUpService(createdCall) context.startActivity(VectorCallActivity.newIntent(context, createdCall)) - startCall() - sendSdpOffer() + sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { + override fun onSuccess(data: TurnServer?) { + startCall(data) + sendSdpOffer() + } + }) } override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { @@ -328,10 +328,18 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall = mxCall startHeadsUpService(mxCall) - startCall() + sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { + override fun onSuccess(data: TurnServer?) { + startCall(data) + setInviteRemoteDescription(callInviteContent.offer?.sdp) + } + }) + } + + private fun setInviteRemoteDescription(description: String?) { executor.execute { - val sdp = SessionDescription(SessionDescription.Type.OFFER, callInviteContent.offer?.sdp) + val sdp = SessionDescription(SessionDescription.Type.OFFER, description) peerConnection?.setRemoteDescription(sdpObserver, sdp) } } From 435a6b2f1a099f2f451bfd1f0b0308a2b188abe8 Mon Sep 17 00:00:00 2001 From: onurays Date: Thu, 4 Jun 2020 13:10:06 +0300 Subject: [PATCH 31/83] Add ice candidates to peer connection. --- .../im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index d35fd3eef8..5e301aa0a5 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -172,6 +172,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // omit empty :/ if (it.isNotEmpty()) { Timber.v("## Sending local ice candidates to call") + it.forEach { peerConnection?.addIceCandidate(it) } currentCall?.sendLocalIceCandidates(it) } } From 9006acb66aee274d6e8c9de9705f4e840f66aa98 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 11 Jun 2020 11:33:02 +0200 Subject: [PATCH 32/83] =?UTF-8?q?WIP=20|=C2=A0Avoid=20re-negociation=20=20?= =?UTF-8?q?pre-agree-upon=20=20signaling/negotiation.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../matrix/android/api/extensions/Try.kt | 7 +- .../matrix/android/api/session/Session.kt | 3 +- ...CallService.kt => CallSignalingService.kt} | 4 +- .../android/api/session/call/CallState.kt | 42 ++ .../android/api/session/call/CallsListener.kt | 3 + .../matrix/android/api/session/call/MxCall.kt | 10 + .../api/session/call/PeerSignalingClient.kt | 66 --- .../room/model/call/CallCandidatesContent.kt | 2 +- .../internal/session/DefaultSession.kt | 6 +- .../android/internal/session/SessionModule.kt | 6 +- .../session/call/CallEventsObserverTask.kt | 2 +- .../internal/session/call/CallModule.kt | 4 +- ...vice.kt => DefaultCallSignalingService.kt} | 51 +- .../internal/session/call/model/MxCallImpl.kt | 50 +- .../src/main/res/values/strings.xml | 2 +- vector/src/main/AndroidManifest.xml | 7 +- .../vector/riotx/core/services/CallService.kt | 75 ++- .../riotx/features/call/CallControlsView.kt | 104 ++++ .../riotx/features/call/SdpObserverAdapter.kt | 2 +- .../riotx/features/call/VectorCallActivity.kt | 202 +++++-- .../features/call/VectorCallViewModel.kt | 114 +++- .../call/WebRtcPeerConnectionManager.kt | 559 +++++++++++------- .../call/service/CallHeadsUpActionReceiver.kt | 34 +- .../call/service/CallHeadsUpService.kt | 324 +++++----- .../call/service/CallHeadsUpServiceArgs.kt | 1 + .../features/call/telecom/TelecomUtils.kt | 29 + .../riotx/features/login/LoginViewModel.kt | 5 +- .../notifications/NotificationUtils.kt | 76 ++- vector/src/main/res/drawable/ic_call.xml | 17 +- vector/src/main/res/drawable/ic_call_end.xml | 22 +- .../main/res/drawable/ic_microphone_off.xml | 41 ++ .../main/res/drawable/ic_microphone_on.xml | 10 + .../main/res/drawable/ic_more_vertical.xml | 18 + vector/src/main/res/drawable/ic_video.xml | 22 + .../main/res/drawable/oval_destructive.xml | 11 + .../src/main/res/drawable/oval_positive.xml | 11 + vector/src/main/res/layout/activity_call.xml | 188 ++++-- .../res/layout/fragment_call_controls.xml | 207 +++++++ vector/src/main/res/menu/menu_timeline.xml | 2 +- 39 files changed, 1705 insertions(+), 634 deletions(-) rename matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/{CallService.kt => CallSignalingService.kt} (92%) create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt delete mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt rename matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/{DefaultCallService.kt => DefaultCallSignalingService.kt} (80%) create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/telecom/TelecomUtils.kt create mode 100644 vector/src/main/res/drawable/ic_microphone_off.xml create mode 100644 vector/src/main/res/drawable/ic_microphone_on.xml create mode 100644 vector/src/main/res/drawable/ic_more_vertical.xml create mode 100644 vector/src/main/res/drawable/ic_video.xml create mode 100644 vector/src/main/res/drawable/oval_destructive.xml create mode 100644 vector/src/main/res/drawable/oval_positive.xml create mode 100644 vector/src/main/res/layout/fragment_call_controls.xml diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/extensions/Try.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/extensions/Try.kt index 3afcac08c1..3d80a94156 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/extensions/Try.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/extensions/Try.kt @@ -16,10 +16,15 @@ package im.vector.matrix.android.api.extensions -inline fun tryThis(operation: () -> A): A? { +import timber.log.Timber + +inline fun tryThis(message: String? = null, operation: () -> A): A? { return try { operation() } catch (any: Throwable) { + if (message != null) { + Timber.e(any, message) + } null } } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt index ab1588f8c5..666983256e 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt @@ -24,7 +24,7 @@ import im.vector.matrix.android.api.pushrules.PushRuleService import im.vector.matrix.android.api.session.account.AccountService import im.vector.matrix.android.api.session.accountdata.AccountDataService import im.vector.matrix.android.api.session.cache.CacheService -import im.vector.matrix.android.api.session.call.CallService +import im.vector.matrix.android.api.session.call.CallSignalingService import im.vector.matrix.android.api.session.content.ContentUploadStateTracker import im.vector.matrix.android.api.session.content.ContentUrlResolver import im.vector.matrix.android.api.session.crypto.CryptoService @@ -155,6 +155,7 @@ interface Session : * Returns the identity service associated with the session */ fun identityService(): IdentityService + fun callService(): CallSignalingService /** * Returns the widget service associated with the session diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallSignalingService.kt similarity index 92% rename from matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt rename to matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallSignalingService.kt index 805bfd53ed..faf85b0ce5 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallSignalingService.kt @@ -19,7 +19,7 @@ package im.vector.matrix.android.api.session.call import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.util.Cancelable -interface CallService { +interface CallSignalingService { fun getTurnServer(callback: MatrixCallback): Cancelable @@ -31,4 +31,6 @@ interface CallService { fun addCallListener(listener: CallsListener) fun removeCallListener(listener: CallsListener) + + fun getCallWithId(callId: String) : MxCall? } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt new file mode 100644 index 0000000000..92fbf405ab --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +enum class CallState { + + /** Idle, setting up objects */ + IDLE, + + /** Dialing. Outgoing call is signaling the remote peer */ + DIALING, + + /** Answering. Incoming call is responding to remote peer */ + ANSWERING, + + /** Remote ringing. Outgoing call, ICE negotiation is complete */ + REMOTE_RINGING, + + /** Local ringing. Incoming call, ICE negotiation is complete */ + LOCAL_RINGING, + + /** Connected. Incoming/Outgoing call, the call is connected */ + CONNECTED, + + /** Terminated. Incoming/Outgoing call, the call is terminated */ + TERMINATED, + +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index 3ab68ceb28..49207f9ed0 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -17,6 +17,7 @@ package im.vector.matrix.android.api.session.call import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent @@ -43,6 +44,8 @@ interface CallsListener { fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) + fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) + fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) fun onCallHangupReceived(callHangupContent: CallHangupContent) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt index afb89b3bb8..f6b8813188 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/MxCall.kt @@ -20,6 +20,7 @@ import org.webrtc.IceCandidate import org.webrtc.SessionDescription interface MxCallDetail { + val callId: String val isOutgoing: Boolean val roomId: String val otherUserId: String @@ -30,6 +31,8 @@ interface MxCallDetail { * Define both an incoming call and on outgoing call */ interface MxCall : MxCallDetail { + + var state: CallState /** * Pick Up the incoming call * It has no effect on outgoing call @@ -62,4 +65,11 @@ interface MxCall : MxCallDetail { * Send removed ICE candidates to the other participant. */ fun sendLocalIceCandidateRemovals(candidates: List) + + fun addListener(listener: StateListener) + fun removeListener(listener: StateListener) + + interface StateListener { + fun onStateUpdate(call: MxCall) + } } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt deleted file mode 100644 index ab99f8875a..0000000000 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt +++ /dev/null @@ -1,66 +0,0 @@ -// /* -// * Copyright (c) 2020 New Vector Ltd -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// package im.vector.matrix.android.api.session.call -// -// import im.vector.matrix.android.api.MatrixCallback -// import org.webrtc.IceCandidate -// import org.webrtc.SessionDescription -// -// interface PeerSignalingClient { -// -// val callID: String -// -// fun addListener(listener: SignalingListener) -// -// /** -// * Send offer SDP to the other participant. -// */ -// fun sendOfferSdp(sdp: SessionDescription, callback: MatrixCallback) -// -// /** -// * Send answer SDP to the other participant. -// */ -// fun sendAnswerSdp(sdp: SessionDescription, callback: MatrixCallback) -// -// /** -// * Send Ice candidate to the other participant. -// */ -// fun sendLocalIceCandidates(candidates: List) -// -// /** -// * Send removed ICE candidates to the other participant. -// */ -// fun sendLocalIceCandidateRemovals(candidates: List) -// -// -// interface SignalingListener { -// /** -// * Callback fired once remote SDP is received. -// */ -// fun onRemoteDescription(sdp: SessionDescription) -// -// /** -// * Callback fired once remote Ice candidate is received. -// */ -// fun onRemoteIceCandidate(candidate: IceCandidate) -// -// /** -// * Callback fired once remote Ice candidate removals are received. -// */ -// fun onRemoteIceCandidatesRemoved(candidates: List) -// } -// } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt index 6132d1a57a..4b71320c32 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallCandidatesContent.kt @@ -48,7 +48,7 @@ data class CallCandidatesContent( /** * Required. The index of the SDP 'm' line this candidate is intended for. */ - @Json(name = "sdpMLineIndex") val sdpMLineIndex: String, + @Json(name = "sdpMLineIndex") val sdpMLineIndex: Int, /** * Required. The SDP 'a' line of the candidate. */ diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt index 8b3affd82f..3202b84173 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt @@ -28,7 +28,7 @@ import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.account.AccountService import im.vector.matrix.android.api.session.accountdata.AccountDataService import im.vector.matrix.android.api.session.cache.CacheService -import im.vector.matrix.android.api.session.call.CallService +import im.vector.matrix.android.api.session.call.CallSignalingService import im.vector.matrix.android.api.session.content.ContentUploadStateTracker import im.vector.matrix.android.api.session.content.ContentUrlResolver import im.vector.matrix.android.api.session.crypto.CryptoService @@ -112,7 +112,7 @@ internal class DefaultSession @Inject constructor( private val taskExecutor: TaskExecutor, private val widgetDependenciesHolder: WidgetDependenciesHolder, private val shieldTrustUpdater: ShieldTrustUpdater, - private val callService: Lazy) + private val callSignalingService: Lazy) : Session, RoomService by roomService.get(), RoomDirectoryService by roomDirectoryService.get(), @@ -247,7 +247,7 @@ internal class DefaultSession @Inject constructor( override fun integrationManagerService() = integrationManagerService - override fun callService(): CallService = callService.get() + override fun callService(): CallSignalingService = callSignalingService.get() override fun addListener(listener: Session.Listener) { sessionListeners.addListener(listener) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt index 5331b9f471..b756c9dbc1 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt @@ -33,7 +33,6 @@ import im.vector.matrix.android.api.crypto.MXCryptoConfig import im.vector.matrix.android.api.session.InitialSyncProgressService import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.accountdata.AccountDataService -import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.homeserver.HomeServerCapabilitiesService import im.vector.matrix.android.api.session.securestorage.SecureStorageService import im.vector.matrix.android.api.session.securestorage.SharedSecretStorageService @@ -57,9 +56,10 @@ import im.vector.matrix.android.internal.network.NetworkCallbackStrategy import im.vector.matrix.android.internal.network.NetworkConnectivityChecker import im.vector.matrix.android.internal.network.PreferredNetworkCallbackStrategy import im.vector.matrix.android.internal.network.RetrofitFactory -import im.vector.matrix.android.internal.network.interceptors.CurlLoggingInterceptor +import im.vector.matrix.android.internal.network.httpclient.addAccessTokenInterceptor +import im.vector.matrix.android.internal.network.token.AccessTokenProvider +import im.vector.matrix.android.internal.network.token.HomeserverAccessTokenProvider import im.vector.matrix.android.internal.session.call.CallEventObserver -import im.vector.matrix.android.internal.session.call.DefaultCallService import im.vector.matrix.android.internal.session.group.GroupSummaryUpdater import im.vector.matrix.android.internal.session.homeserver.DefaultHomeServerCapabilitiesService import im.vector.matrix.android.internal.session.room.EventRelationsAggregationUpdater diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt index 919600f3bc..b8879739f4 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt @@ -39,7 +39,7 @@ internal interface CallEventsObserverTask : Task() + private val activeCalls = mutableListOf() + override fun getTurnServer(callback: MatrixCallback): Cancelable { return turnServerTask .configureWith(GetTurnServerTask.Params) { @@ -77,7 +80,9 @@ internal class DefaultCallService @Inject constructor( isVideoCall = isVideoCall, localEchoEventFactory = localEchoEventFactory, roomEventSender = roomEventSender - ) + ).also { + activeCalls.add(it) + } } override fun addCallListener(listener: CallsListener) { @@ -88,14 +93,24 @@ internal class DefaultCallService @Inject constructor( callListeners.remove(listener) } + override fun getCallWithId(callId: String): MxCall? { + return activeCalls.find { it.callId == callId } + } + internal fun onCallEvent(event: Event) { + // TODO if handled by other of my sessions + // this test is too simple, should notify upstream + if (event.senderId == userId) { + //ignore local echos! + return + } when (event.getClearType()) { - EventType.CALL_ANSWER -> { + EventType.CALL_ANSWER -> { event.getClearContent().toModel()?.let { onCallAnswer(it) } } - EventType.CALL_INVITE -> { + EventType.CALL_INVITE -> { event.getClearContent().toModel()?.let { content -> val incomingCall = MxCallImpl( callId = content.callId ?: return@let, @@ -107,14 +122,24 @@ internal class DefaultCallService @Inject constructor( localEchoEventFactory = localEchoEventFactory, roomEventSender = roomEventSender ) + activeCalls.add(incomingCall) onCallInvite(incomingCall, content) } } - EventType.CALL_HANGUP -> { - event.getClearContent().toModel()?.let { - onCallHangup(it) + EventType.CALL_HANGUP -> { + event.getClearContent().toModel()?.let { content -> + onCallHangup(content) + activeCalls.removeAll { it.callId == content.callId } } } + EventType.CALL_CANDIDATES -> { + event.getClearContent().toModel()?.let { content -> + activeCalls.firstOrNull { it.callId == content.callId }?.let { + onCallIceCandidate(it, content) + } + } + } + } } @@ -145,6 +170,14 @@ internal class DefaultCallService @Inject constructor( } } + private fun onCallIceCandidate(incomingCall: MxCall, candidates: CallCandidatesContent) { + callListeners.toList().forEach { + tryThis { + it.onCallIceCandidateReceived(incomingCall, candidates) + } + } + } + companion object { const val CALL_TIMEOUT_MS = 120_000 } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt index fdee9c295b..fe22099018 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt @@ -16,6 +16,7 @@ package im.vector.matrix.android.internal.session.call.model +import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.MxCall import im.vector.matrix.android.api.session.events.model.Content import im.vector.matrix.android.api.session.events.model.Event @@ -27,14 +28,15 @@ import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent -import im.vector.matrix.android.internal.session.call.DefaultCallService +import im.vector.matrix.android.internal.session.call.DefaultCallSignalingService import im.vector.matrix.android.internal.session.room.send.LocalEchoEventFactory import im.vector.matrix.android.internal.session.room.send.RoomEventSender import org.webrtc.IceCandidate import org.webrtc.SessionDescription +import timber.log.Timber internal class MxCallImpl( - val callId: String, + override val callId: String, override val isOutgoing: Boolean, override val roomId: String, private val userId: String, @@ -44,12 +46,47 @@ internal class MxCallImpl( private val roomEventSender: RoomEventSender ) : MxCall { + override var state: CallState = CallState.IDLE + set(value) { + field = value + dispatchStateChange() + } + + private val listeners = mutableListOf() + + override fun addListener(listener: MxCall.StateListener) { + listeners.add(listener) + } + + override fun removeListener(listener: MxCall.StateListener) { + listeners.remove(listener) + } + + private fun dispatchStateChange() { + listeners.forEach { + try { + it.onStateUpdate(this) + } catch (failure: Throwable) { + Timber.d("dispatchStateChange failed for call $callId : ${failure.localizedMessage}") + } + } + } + + init { + if (isOutgoing) { + state = CallState.DIALING + } else { + state = CallState.LOCAL_RINGING + } + } + + override fun offerSdp(sdp: SessionDescription) { if (!isOutgoing) return - + state = CallState.REMOTE_RINGING CallInviteContent( callId = callId, - lifetime = DefaultCallService.CALL_TIMEOUT_MS, + lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, offer = CallInviteContent.Offer(sdp = sdp.description) ) .let { createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = it.toContent()) } @@ -62,7 +99,7 @@ internal class MxCallImpl( candidates = candidates.map { CallCandidatesContent.Candidate( sdpMid = it.sdpMid, - sdpMLineIndex = it.sdpMLineIndex.toString(), + sdpMLineIndex = it.sdpMLineIndex, candidate = it.sdp ) } @@ -80,11 +117,12 @@ internal class MxCallImpl( ) .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) } .also { roomEventSender.sendEvent(it) } + state = CallState.TERMINATED } override fun accept(sdp: SessionDescription) { if (isOutgoing) return - + state = CallState.ANSWERING CallAnswerContent( callId = callId, answer = CallAnswerContent.Answer(sdp = sdp.description) diff --git a/matrix-sdk-android/src/main/res/values/strings.xml b/matrix-sdk-android/src/main/res/values/strings.xml index dc01fc0dfc..c749441746 100644 --- a/matrix-sdk-android/src/main/res/values/strings.xml +++ b/matrix-sdk-android/src/main/res/values/strings.xml @@ -363,6 +363,6 @@ %s is requesting to verify your key, but your client does not support in-chat key verification. You will need to use legacy key verification to verify keys. Accept - Reject + Decline diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index abb6c929cc..50c9eabadb 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -19,6 +19,7 @@ + @@ -206,9 +207,9 @@ - + + + diff --git a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt index 99d4b23760..6e86f54797 100644 --- a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt @@ -15,15 +15,16 @@ * limitations under the License. */ -@file:Suppress("UNUSED_PARAMETER") package im.vector.riotx.core.services import android.content.Context import android.content.Intent import android.os.Binder +import android.text.TextUtils import androidx.core.content.ContextCompat import im.vector.riotx.core.extensions.vectorComponent +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.call.telecom.CallConnection import im.vector.riotx.features.notifications.NotificationUtils import timber.log.Timber @@ -41,6 +42,7 @@ class CallService : VectorService() { private var mCallIdInProgress: String? = null private lateinit var notificationUtils: NotificationUtils + private lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager /** * incoming (foreground notification) @@ -50,6 +52,7 @@ class CallService : VectorService() { override fun onCreate() { super.onCreate() notificationUtils = vectorComponent().notificationUtils() + webRtcPeerConnectionManager = vectorComponent().webRtcPeerConnectionManager() } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { @@ -87,46 +90,46 @@ class CallService : VectorService() { private fun displayIncomingCallNotification(intent: Intent) { Timber.v("displayIncomingCallNotification") - // TODO - /* + // the incoming call in progress is already displayed if (!TextUtils.isEmpty(mIncomingCallId)) { Timber.v("displayIncomingCallNotification : the incoming call in progress is already displayed") } else if (!TextUtils.isEmpty(mCallIdInProgress)) { Timber.v("displayIncomingCallNotification : a 'call in progress' notification is displayed") - } else if (null == CallsManager.getSharedInstance().activeCall) { + } else +// if (null == webRtcPeerConnectionManager.currentCall) + { val callId = intent.getStringExtra(EXTRA_CALL_ID) Timber.v("displayIncomingCallNotification : display the dedicated notification") - val notification = NotificationUtils.buildIncomingCallNotification( - this, + val notification = notificationUtils.buildIncomingCallNotification( intent.getBooleanExtra(EXTRA_IS_VIDEO, false), - intent.getStringExtra(EXTRA_ROOM_NAME), - intent.getStringExtra(EXTRA_MATRIX_ID), - callId) + intent.getStringExtra(EXTRA_ROOM_NAME) ?: "", + intent.getStringExtra(EXTRA_MATRIX_ID) ?: "", + callId ?: "") startForeground(NOTIFICATION_ID, notification) mIncomingCallId = callId // turn the screen on for 3 seconds - if (Matrix.getInstance(VectorApp.getInstance())!!.pushManager.isScreenTurnedOn) { - try { - val pm = getSystemService(Context.POWER_SERVICE) as PowerManager - val wl = pm.newWakeLock( - WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON or PowerManager.ACQUIRE_CAUSES_WAKEUP, - CallService::class.java.simpleName) - wl.acquire(3000) - wl.release() - } catch (re: RuntimeException) { - Timber.e(re, "displayIncomingCallNotification : failed to turn screen on ") - } - - } - } else { - Timber.i("displayIncomingCallNotification : do not display the incoming call notification because there is a pending call") - }// test if there is no active call - */ +// if (Matrix.getInstance(VectorApp.getInstance())!!.pushManager.isScreenTurnedOn) { +// try { +// val pm = getSystemService(Context.POWER_SERVICE) as PowerManager +// val wl = pm.newWakeLock( +// WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON or PowerManager.ACQUIRE_CAUSES_WAKEUP, +// CallService::class.java.simpleName) +// wl.acquire(3000) +// wl.release() +// } catch (re: RuntimeException) { +// Timber.e(re, "displayIncomingCallNotification : failed to turn screen on ") +// } +// +// } + } +// else { +// Timber.i("displayIncomingCallNotification : do not display the incoming call notification because there is a pending call") +// } } /** @@ -169,6 +172,7 @@ class CallService : VectorService() { private const val ACTION_INCOMING_CALL = "im.vector.riotx.core.services.CallService.INCOMING_CALL" private const val ACTION_PENDING_CALL = "im.vector.riotx.core.services.CallService.PENDING_CALL" private const val ACTION_NO_ACTIVE_CALL = "im.vector.riotx.core.services.CallService.NO_ACTIVE_CALL" +// private const val ACTION_ON_ACTIVE_CALL = "im.vector.riotx.core.services.CallService.ACTIVE_CALL" private const val EXTRA_IS_VIDEO = "EXTRA_IS_VIDEO" private const val EXTRA_ROOM_NAME = "EXTRA_ROOM_NAME" @@ -195,6 +199,25 @@ class CallService : VectorService() { ContextCompat.startForegroundService(context, intent) } +// fun onActiveCall(context: Context, +// isVideo: Boolean, +// roomName: String, +// roomId: String, +// matrixId: String, +// callId: String) { +// val intent = Intent(context, CallService::class.java) +// .apply { +// action = ACTION_ON_ACTIVE_CALL +// putExtra(EXTRA_IS_VIDEO, isVideo) +// putExtra(EXTRA_ROOM_NAME, roomName) +// putExtra(EXTRA_ROOM_ID, roomId) +// putExtra(EXTRA_MATRIX_ID, matrixId) +// putExtra(EXTRA_CALL_ID, callId) +// } +// +// ContextCompat.startForegroundService(context, intent) +// } + fun onPendingCall(context: Context, isVideo: Boolean, roomName: String, diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt new file mode 100644 index 0000000000..4f9cfe1d7f --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.content.Context +import android.util.AttributeSet +import android.view.ViewGroup +import android.widget.LinearLayout +import androidx.constraintlayout.widget.ConstraintLayout +import androidx.core.view.isVisible +import butterknife.BindView +import butterknife.ButterKnife +import butterknife.OnClick +import im.vector.matrix.android.api.session.call.CallState +import im.vector.riotx.R + +class CallControlsView @JvmOverloads constructor( + context: Context, attrs: AttributeSet? = null, defStyleAttr: Int = 0 +) : LinearLayout(context, attrs, defStyleAttr) { + + var interactionListener: InteractionListener? = null + + @BindView(R.id.incomingRingingControls) + lateinit var incomingRingingControls: ViewGroup +// @BindView(R.id.iv_icr_accept_call) +// lateinit var incomingRingingControlAccept: ImageView +// @BindView(R.id.iv_icr_end_call) +// lateinit var incomingRingingControlDecline: ImageView + + @BindView(R.id.connectedControls) + lateinit var connectedControls: ViewGroup + + init { + ConstraintLayout.inflate(context, R.layout.fragment_call_controls, this) + //layoutParams = FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT) + ButterKnife.bind(this) + } + + @OnClick(R.id.iv_icr_accept_call) + fun acceptIncomingCall() { + interactionListener?.didAcceptIncomingCall() + } + + @OnClick(R.id.iv_icr_end_call) + fun declineIncomingCall() { + interactionListener?.didDeclineIncomingCall() + } + + @OnClick(R.id.iv_end_call) + fun endOngoingCall() { + interactionListener?.didEndCall() + } + + @OnClick(R.id.iv_end_call) + fun hangupCall() { + } + + fun updateForState(callState: CallState?) { + when (callState) { + CallState.DIALING -> { + } + CallState.ANSWERING -> { + incomingRingingControls.isVisible = false + connectedControls.isVisible = false + } + CallState.REMOTE_RINGING -> { + } + CallState.LOCAL_RINGING -> { + incomingRingingControls.isVisible = true + connectedControls.isVisible = false + } + CallState.CONNECTED -> { + incomingRingingControls.isVisible = false + connectedControls.isVisible = true + } + CallState.TERMINATED, + CallState.IDLE, + null -> { + incomingRingingControls.isVisible = false + connectedControls.isVisible = false + } + } + } + + interface InteractionListener { + fun didAcceptIncomingCall() + fun didDeclineIncomingCall() + fun didEndCall() + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt b/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt index 0e15c97052..4c9964a4c4 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt @@ -20,7 +20,7 @@ import org.webrtc.SdpObserver import org.webrtc.SessionDescription import timber.log.Timber -abstract class SdpObserverAdapter : SdpObserver { +open class SdpObserverAdapter : SdpObserver { override fun onSetFailure(p0: String?) { Timber.e("## SdpObserver: onSetFailure $p0") } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index b4090fa948..ceffe8d15f 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -16,52 +16,58 @@ package im.vector.riotx.features.call +//import im.vector.riotx.features.call.service.CallHeadsUpService import android.app.KeyguardManager -import android.content.ComponentName import android.content.Context import android.content.Intent -import android.content.ServiceConnection import android.os.Build import android.os.Bundle -import android.os.IBinder import android.os.Parcelable import android.view.View import android.view.Window import android.view.WindowManager +import androidx.core.view.isInvisible import androidx.core.view.isVisible import butterknife.BindView import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel -import im.vector.matrix.android.api.extensions.tryThis +import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCallDetail import im.vector.riotx.R import im.vector.riotx.core.di.ScreenComponent import im.vector.riotx.core.platform.VectorBaseActivity +import im.vector.riotx.core.services.CallService +import im.vector.riotx.core.utils.PERMISSIONS_FOR_AUDIO_IP_CALL import im.vector.riotx.core.utils.PERMISSIONS_FOR_VIDEO_IP_CALL import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions -import im.vector.riotx.features.call.service.CallHeadsUpService +import im.vector.riotx.features.home.AvatarRenderer import io.reactivex.android.schedulers.AndroidSchedulers import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.webrtc.EglBase import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer +import timber.log.Timber import javax.inject.Inject @Parcelize data class CallArgs( val roomId: String, + val callId: String?, val participantUserId: String, val isIncomingCall: Boolean, - val isVideoCall: Boolean + val isVideoCall: Boolean, + val autoAccept: Boolean ) : Parcelable -class VectorCallActivity : VectorBaseActivity() { +class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionListener { override fun getLayoutRes() = R.layout.activity_call + @Inject lateinit var avatarRenderer: AvatarRenderer + override fun injectWith(injector: ScreenComponent) { super.injectWith(injector) injector.inject(this) @@ -80,18 +86,21 @@ class VectorCallActivity : VectorBaseActivity() { @BindView(R.id.fullscreen_video_view) lateinit var fullscreenRenderer: SurfaceViewRenderer + @BindView(R.id.callControls) + lateinit var callControlsView: CallControlsView + private var rootEglBase: EglBase? = null - var callHeadsUpService: CallHeadsUpService? = null - private val serviceConnection = object : ServiceConnection { - override fun onServiceDisconnected(name: ComponentName?) { - finish() - } - - override fun onServiceConnected(name: ComponentName?, service: IBinder?) { - callHeadsUpService = (service as? CallHeadsUpService.CallHeadsUpServiceBinder)?.getService() - } - } +// var callHeadsUpService: CallHeadsUpService? = null +// private val serviceConnection = object : ServiceConnection { +// override fun onServiceDisconnected(name: ComponentName?) { +// finish() +// } +// +// override fun onServiceConnected(name: ComponentName?, service: IBinder?) { +// callHeadsUpService = (service as? CallHeadsUpService.CallHeadsUpServiceBinder)?.getService() +// } +// } override fun doBeforeSetContentView() { // Set window styles for fullscreen-window size. Needs to be done before adding content. @@ -118,8 +127,11 @@ class VectorCallActivity : VectorBaseActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - tryThis { unbindService(serviceConnection) } - bindService(Intent(this, CallHeadsUpService::class.java), serviceConnection, 0) +// window.addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED); +// window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + +// tryThis { unbindService(serviceConnection) } +// bindService(Intent(this, CallHeadsUpService::class.java), serviceConnection, 0) if (intent.hasExtra(MvRx.KEY_ARG)) { callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! @@ -127,12 +139,29 @@ class VectorCallActivity : VectorBaseActivity() { finish() } + if (isFirstCreation()) { + // Reduce priority of notification as the activity is on screen + CallService.onPendingCall( + this, + callArgs.isVideoCall, + callArgs.participantUserId, + callArgs.roomId, + "", + callArgs.callId ?: "" + ) + } + rootEglBase = EglUtils.rootEglBase ?: return Unit.also { finish() } configureCallViews() + callViewModel.subscribe(this) { + renderState(it) + } + + callViewModel.viewEvents .observe() .observeOn(AndroidSchedulers.mainThread()) @@ -141,26 +170,89 @@ class VectorCallActivity : VectorBaseActivity() { } .disposeOnDestroy() - if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, CAPTURE_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { - start() + if (callArgs.isVideoCall) { + if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, CAPTURE_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { + start() + } + } else { + if (checkPermissions(PERMISSIONS_FOR_AUDIO_IP_CALL, this, CAPTURE_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_record_audio)) { + start() + } + } + } + + private fun renderState(state: VectorCallViewState) { + Timber.v("## VOIP renderState call $state") + callControlsView.updateForState(state.callState.invoke()) + when (state.callState.invoke()) { + CallState.IDLE -> { + + } + CallState.DIALING -> { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + callStatusText.setText(R.string.call_ring) + state.otherUserMatrixItem.invoke()?.let { + avatarRenderer.render(it, otherMemberAvatar) + participantNameText.text = it.getBestName() + callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) + } + } + CallState.ANSWERING -> { + callInfoGroup.isVisible = true + callStatusText.setText(R.string.call_connecting) + state.otherUserMatrixItem.invoke()?.let { + avatarRenderer.render(it, otherMemberAvatar) + } +// fullscreenRenderer.isVisible = true +// pipRenderer.isVisible = true + } + CallState.REMOTE_RINGING -> { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + callStatusText.setText( + if (state.isVideoCall) R.string.incoming_video_call else R.string.incoming_voice_call + ) + } + CallState.LOCAL_RINGING -> { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + state.otherUserMatrixItem.invoke()?.let { + avatarRenderer.render(it, otherMemberAvatar) + participantNameText.text = it.getBestName() + callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) + } + } + CallState.CONNECTED -> { + // TODO only if is video call + callVideoGroup.isVisible = true + callInfoGroup.isVisible = false + } + CallState.TERMINATED -> { + finish() + } + null -> { + + } } } private fun configureCallViews() { - if (callArgs.isVideoCall) { - iv_call_speaker.isVisible = false - iv_call_flip_camera.isVisible = true - iv_call_videocam_off.isVisible = true - } else { - iv_call_speaker.isVisible = true - iv_call_flip_camera.isVisible = false - iv_call_videocam_off.isVisible = false - } - - iv_end_call.setOnClickListener { - callViewModel.handle(VectorCallViewActions.EndCall) - finish() - } + callControlsView.interactionListener = this +// if (callArgs.isVideoCall) { +// iv_call_speaker.isVisible = false +// iv_call_flip_camera.isVisible = true +// iv_call_videocam_off.isVisible = true +// } else { +// iv_call_speaker.isVisible = true +// iv_call_flip_camera.isVisible = false +// iv_call_videocam_off.isVisible = false +// } +// +// iv_end_call.setOnClickListener { +// callViewModel.handle(VectorCallViewActions.EndCall) +// finish() +// } } override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { @@ -186,13 +278,14 @@ class VectorCallActivity : VectorBaseActivity() { fullscreenRenderer.setEnableHardwareScaler(true /* enabled */) - peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) + peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, + intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() }) return false } override fun onDestroy() { peerConnectionManager.detachRenderers() - tryThis { unbindService(serviceConnection) } +// tryThis { unbindService(serviceConnection) } super.onDestroy() } @@ -209,12 +302,45 @@ class VectorCallActivity : VectorBaseActivity() { companion object { private const val CAPTURE_PERMISSION_REQUEST_CODE = 1 + private const val EXTRA_MODE = "EXTRA_MODE" + + const val OUTGOING_CREATED = "OUTGOING_CREATED" + const val INCOMING_RINGING = "INCOMING_RINGING" + const val INCOMING_ACCEPT = "INCOMING_ACCEPT" fun newIntent(context: Context, mxCall: MxCallDetail): Intent { return Intent(context, VectorCallActivity::class.java).apply { flags = Intent.FLAG_ACTIVITY_NEW_TASK - putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall)) + putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.callId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall, false)) + putExtra(EXTRA_MODE, OUTGOING_CREATED) + } + } + + fun newIntent(context: Context, + callId: String?, + roomId: String, + otherUserId: String, + isIncomingCall: Boolean, + isVideoCall: Boolean, + accept: Boolean, + mode: String?): Intent { + return Intent(context, VectorCallActivity::class.java).apply { + flags = Intent.FLAG_ACTIVITY_NEW_TASK + putExtra(MvRx.KEY_ARG, CallArgs(roomId, callId, otherUserId, isIncomingCall, isVideoCall, accept)) + putExtra(EXTRA_MODE, mode) } } } + + override fun didAcceptIncomingCall() { + callViewModel.handle(VectorCallViewActions.AcceptCall) + } + + override fun didDeclineIncomingCall() { + callViewModel.handle(VectorCallViewActions.DeclineCall) + } + + override fun didEndCall() { + callViewModel.handle(VectorCallViewActions.EndCall) + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 296dfe5582..3d4c5b6445 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -16,17 +16,22 @@ package im.vector.riotx.features.call +import com.airbnb.mvrx.Async +import com.airbnb.mvrx.Loading import com.airbnb.mvrx.MvRxState import com.airbnb.mvrx.MvRxViewModelFactory +import com.airbnb.mvrx.Success +import com.airbnb.mvrx.Uninitialized import com.airbnb.mvrx.ViewModelContext import com.squareup.inject.assisted.Assisted import com.squareup.inject.assisted.AssistedInject import im.vector.matrix.android.api.session.Session -import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.MxCall import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent -import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.matrix.android.api.util.MatrixItem +import im.vector.matrix.android.api.util.toMatrixItem import im.vector.riotx.core.extensions.exhaustive import im.vector.riotx.core.platform.VectorViewEvents import im.vector.riotx.core.platform.VectorViewModel @@ -34,65 +39,117 @@ import im.vector.riotx.core.platform.VectorViewModelAction data class VectorCallViewState( val callId: String? = null, - val roomId: String = "" + val roomId: String = "", + val isVideoCall: Boolean, + val otherUserMatrixItem: Async = Uninitialized, + val callState: Async = Uninitialized ) : MvRxState sealed class VectorCallViewActions : VectorViewModelAction { - object EndCall : VectorCallViewActions() + object AcceptCall : VectorCallViewActions() + object DeclineCall : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() data class CallHangup(val content: CallHangupContent) : VectorCallViewEvents() + object CallAccepted : VectorCallViewEvents() } class VectorCallViewModel @AssistedInject constructor( @Assisted initialState: VectorCallViewState, + @Assisted val args: CallArgs, val session: Session, val webRtcPeerConnectionManager: WebRtcPeerConnectionManager ) : VectorViewModel(initialState) { - private val callServiceListener: CallsListener = object : CallsListener { - override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { - withState { state -> - if (callAnswerContent.callId == state.callId) { - _viewEvents.post(VectorCallViewEvents.CallAnswered(callAnswerContent)) - } - } - } + // private val callServiceListener: CallsListener = object : CallsListener { +// override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { +// withState { state -> +// if (callAnswerContent.callId == state.callId) { +// _viewEvents.post(VectorCallViewEvents.CallAnswered(callAnswerContent)) +// } +// } +// } +// +// override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { +// } +// +// override fun onCallHangupReceived(callHangupContent: CallHangupContent) { +// withState { state -> +// if (callHangupContent.callId == state.callId) { +// _viewEvents.post(VectorCallViewEvents.CallHangup(callHangupContent)) +// } +// } +// } +// } + var autoReplyIfNeeded: Boolean = false - override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { - } + var call: MxCall? = null - override fun onCallHangupReceived(callHangupContent: CallHangupContent) { - withState { state -> - if (callHangupContent.callId == state.callId) { - _viewEvents.post(VectorCallViewEvents.CallHangup(callHangupContent)) - } + private val callStateListener = object : MxCall.StateListener { + override fun onStateUpdate(call: MxCall) { + setState { + copy( + callState = Success(call.state) + ) } } } init { - session.callService().addCallListener(callServiceListener) + + autoReplyIfNeeded = args.autoAccept + + initialState.callId?.let { + session.callService().getCallWithId(it)?.let { mxCall -> + this.call = mxCall + mxCall.otherUserId + val item: MatrixItem? = session.getUser(mxCall.otherUserId)?.toMatrixItem() + + mxCall.addListener(callStateListener) + setState { + copy( + isVideoCall = mxCall.isVideoCall, + callState = Success(mxCall.state), + otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized + ) + } + } + } + + //session.callService().addCallListener(callServiceListener) } override fun onCleared() { - session.callService().removeCallListener(callServiceListener) + //session.callService().removeCallListener(callServiceListener) + this.call?.removeListener(callStateListener) super.onCleared() } override fun handle(action: VectorCallViewActions) = withState { when (action) { - VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() + VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() + VectorCallViewActions.AcceptCall -> { + setState { + copy(callState = Loading()) + } + webRtcPeerConnectionManager.acceptIncomingCall() + } + VectorCallViewActions.DeclineCall -> { + setState { + copy(callState = Loading()) + } + webRtcPeerConnectionManager.endCall() + } }.exhaustive } @AssistedInject.Factory interface Factory { - fun create(initialState: VectorCallViewState): VectorCallViewModel + fun create(initialState: VectorCallViewState, args: CallArgs): VectorCallViewModel } companion object : MvRxViewModelFactory { @@ -100,12 +157,17 @@ class VectorCallViewModel @AssistedInject constructor( @JvmStatic override fun create(viewModelContext: ViewModelContext, state: VectorCallViewState): VectorCallViewModel? { val callActivity: VectorCallActivity = viewModelContext.activity() - return callActivity.viewModelFactory.create(state) + val callArgs: CallArgs = viewModelContext.args() + return callActivity.viewModelFactory.create(state, callArgs) } override fun initialState(viewModelContext: ViewModelContext): VectorCallViewState? { - //val args: CallArgs = viewModelContext.args() - return VectorCallViewState() + val args: CallArgs = viewModelContext.args() + return VectorCallViewState( + callId = args.callId, + roomId = args.roomId, + isVideoCall = args.isVideoCall + ) } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 5e301aa0a5..1e95d7b7c5 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -16,26 +16,22 @@ package im.vector.riotx.features.call -import android.content.ComponentName import android.content.Context -import android.content.Intent -import android.content.ServiceConnection -import android.os.IBinder -import androidx.core.content.ContextCompat import im.vector.matrix.android.api.MatrixCallback -import im.vector.matrix.android.api.extensions.tryThis +import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCall -import im.vector.matrix.android.api.session.call.MxCallDetail import im.vector.matrix.android.api.session.call.TurnServer import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.riotx.core.di.ActiveSessionHolder -import im.vector.riotx.features.call.service.CallHeadsUpService +import im.vector.riotx.core.services.CallService import io.reactivex.disposables.Disposable import io.reactivex.subjects.PublishSubject +import io.reactivex.subjects.ReplaySubject import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.Camera1Enumerator @@ -72,51 +68,86 @@ class WebRtcPeerConnectionManager @Inject constructor( private val sessionHolder: ActiveSessionHolder ) : CallsListener { - var localMediaStream: MediaStream? = null + data class CallContext( + val mxCall: MxCall, + + var peerConnection: PeerConnection? = null, + + var localMediaStream: MediaStream? = null, + var remoteMediaStream: MediaStream? = null, + + var localAudioSource: AudioSource? = null, + var localAudioTrack: AudioTrack? = null, + + var localVideoSource: VideoSource? = null, + var localVideoTrack: VideoTrack? = null, + + var remoteVideoTrack: VideoTrack? = null + ) { + + var offerSdp: CallInviteContent.Offer? = null + + val iceCandidateSource: PublishSubject = PublishSubject.create() + private val iceCandidateDisposable = iceCandidateSource + .buffer(300, TimeUnit.MILLISECONDS) + .subscribe { + // omit empty :/ + if (it.isNotEmpty()) { + Timber.v("## Sending local ice candidates to call") + //it.forEach { peerConnection?.addIceCandidate(it) } + mxCall.sendLocalIceCandidates(it) + } + } + + var remoteCandidateSource: ReplaySubject? = null + var remoteIceCandidateDisposable: Disposable? = null + + fun release() { + + remoteIceCandidateDisposable?.dispose() + iceCandidateDisposable?.dispose() + + peerConnection?.close() + peerConnection?.dispose() + + localAudioSource?.dispose() + localVideoSource?.dispose() + + localAudioSource = null + localAudioTrack = null + localVideoSource = null + localVideoTrack = null + localMediaStream = null + remoteMediaStream = null + } + } + +// var localMediaStream: MediaStream? = null private val executor = Executors.newSingleThreadExecutor() private val rootEglBase by lazy { EglUtils.rootEglBase } private var peerConnectionFactory: PeerConnectionFactory? = null - private var peerConnection: PeerConnection? = null - private var localSdp: SessionDescription? = null - private var sdpObserver = SdpObserver() - private var streamObserver = StreamObserver() - - private var localViewRenderer: SurfaceViewRenderer? = null - private var remoteViewRenderer: SurfaceViewRenderer? = null - - private var remoteVideoTrack: VideoTrack? = null - private var localVideoTrack: VideoTrack? = null - - private var videoSource: VideoSource? = null - private var audioSource: AudioSource? = null - private var localAudioTrack: AudioTrack? = null +// private var localSdp: SessionDescription? = null private var videoCapturer: VideoCapturer? = null var localSurfaceRenderer: WeakReference? = null var remoteSurfaceRenderer: WeakReference? = null - private val iceCandidateSource: PublishSubject = PublishSubject.create() - private var iceCandidateDisposable: Disposable? = null + var currentCall: CallContext? = null - var callHeadsUpService: CallHeadsUpService? = null - - private var currentCall: MxCall? = null - - private val serviceConnection = object : ServiceConnection { - override fun onServiceDisconnected(name: ComponentName?) { - } - - override fun onServiceConnected(name: ComponentName?, service: IBinder?) { - callHeadsUpService = (service as? CallHeadsUpService.CallHeadsUpServiceBinder)?.getService() + init { + // TODO do this lazyly + executor.execute { + createPeerConnectionFactory() } } private fun createPeerConnectionFactory() { + if (peerConnectionFactory != null) return Timber.v("## VOIP createPeerConnectionFactory") val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also { Timber.e("## VOIP No EGL BASE") @@ -143,10 +174,10 @@ class WebRtcPeerConnectionManager @Inject constructor( .setVideoDecoderFactory(defaultVideoDecoderFactory) .createPeerConnectionFactory() - attachViewRenderersInternal() + //attachViewRenderersInternal() } - private fun createPeerConnection(turnServer: TurnServer?) { + private fun createPeerConnection(callContext: CallContext, turnServer: TurnServer?) { val iceServers = mutableListOf().apply { turnServer?.let { server -> server.uris?.forEach { uri -> @@ -161,124 +192,228 @@ class WebRtcPeerConnectionManager @Inject constructor( } } } - Timber.v("## VOIP creating peer connection... ") - peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, streamObserver) + Timber.v("## VOIP creating peer connection...with iceServers ${iceServers} ") + callContext.peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, StreamObserver(callContext)) } - private fun startCall(turnServer: TurnServer?) { - iceCandidateDisposable = iceCandidateSource - .buffer(400, TimeUnit.MILLISECONDS) - .subscribe { - // omit empty :/ - if (it.isNotEmpty()) { - Timber.v("## Sending local ice candidates to call") - it.forEach { peerConnection?.addIceCandidate(it) } - currentCall?.sendLocalIceCandidates(it) + private fun sendSdpOffer(callContext: CallContext) { +// executor.execute { + val constraints = MediaConstraints() + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) + + Timber.v("## VOIP creating offer...") + callContext.peerConnection?.createOffer(object : SdpObserverAdapter() { + override fun onCreateSuccess(p0: SessionDescription?) { + if (p0 == null) return +// localSdp = p0 + callContext.peerConnection?.setLocalDescription(object : SdpObserverAdapter() {}, p0) + // send offer to peer + currentCall?.mxCall?.offerSdp(p0) + } + }, constraints) +// } + } + + private fun getTurnServer(callback: ((TurnServer?) -> Unit)) { + sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { + override fun onSuccess(data: TurnServer?) { + callback(data) + } + + override fun onFailure(failure: Throwable) { + callback(null) + } + }) + } + + fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { + Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer") + this.localSurfaceRenderer = WeakReference(localViewRenderer) + this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + getTurnServer { turnServer -> + val call = currentCall ?: return@getTurnServer + when (mode) { + VectorCallActivity.INCOMING_ACCEPT -> { + internalAcceptIncomingCall(call, turnServer) + } + VectorCallActivity.INCOMING_RINGING -> { + // wait until accepted to create peer connection + // TODO eventually we could already display local stream in PIP? + } + VectorCallActivity.OUTGOING_CREATED -> { + executor.execute { + // 1. Create RTCPeerConnection + createPeerConnection(call, turnServer) + + // 2. Access camera (if video call) + microphone, create local stream + createLocalStream(call) + + // 3. add local stream + call.localMediaStream?.let { call.peerConnection?.addStream(it) } + attachViewRenderersInternal() + + // create an offer, set local description and send via signaling + sendSdpOffer(call) + + Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}") + call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({ + Timber.v("## VOIP adding remote ice candidate $it") + call.peerConnection?.addIceCandidate(it) + }, { + Timber.v("## VOIP failed to add remote ice candidate $it") + }) } } - - executor.execute { - createPeerConnectionFactory() - createPeerConnection(turnServer) - } - } - - private fun sendSdpOffer() { - executor.execute { - val constraints = MediaConstraints() - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) - - Timber.v("## VOIP creating offer...") - peerConnection?.createOffer(sdpObserver, constraints) - } - } - - fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { - executor.execute { - this.localViewRenderer = localViewRenderer - this.remoteViewRenderer = remoteViewRenderer - this.localSurfaceRenderer = WeakReference(localViewRenderer) - this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) - - if (peerConnection != null) { - attachViewRenderersInternal() + else -> { + // sink existing tracks (configuration change, e.g screen rotation) + attachViewRenderersInternal() + } } + + } + } + + private fun internalAcceptIncomingCall(callContext: CallContext, turnServer: TurnServer?) { + executor.execute { + // 1) create peer connection + createPeerConnection(callContext, turnServer) + + // create sdp using offer, and set remote description + // the offer has beed stored when invite was received + callContext.offerSdp?.sdp?.let { + SessionDescription(SessionDescription.Type.OFFER, it) + }?.let { + callContext.peerConnection?.setRemoteDescription(SdpObserverAdapter(), it) + } + // 2) Access camera + microphone, create local stream + createLocalStream(callContext) + + // 2) add local stream + currentCall?.localMediaStream?.let { callContext.peerConnection?.addStream(it) } + attachViewRenderersInternal() + + // create a answer, set local description and send via signaling + createAnswer() + + Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}") + callContext.remoteIceCandidateDisposable = callContext.remoteCandidateSource?.subscribe({ + Timber.v("## VOIP adding remote ice candidate $it") + callContext.peerConnection?.addIceCandidate(it) + }, { + Timber.v("## VOIP failed to add remote ice candidate $it") + }) + } + } + + private fun createLocalStream(callContext: CallContext) { + + if (callContext.localMediaStream != null) { + Timber.e("## VOIP localMediaStream already created") + return + } + if (peerConnectionFactory == null) { + Timber.e("## VOIP peerConnectionFactory is null") + return + } + val audioSource = peerConnectionFactory!!.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + val localAudioTrack = peerConnectionFactory!!.createAudioTrack(AUDIO_TRACK_ID, audioSource) + localAudioTrack?.setEnabled(true) + + callContext.localAudioSource = audioSource + callContext.localAudioTrack = localAudioTrack + + val localMediaStream = peerConnectionFactory!!.createLocalMediaStream("ARDAMS") // magic value? + + //Add audio track + localMediaStream?.addTrack(localAudioTrack) + + callContext.localMediaStream = localMediaStream + + //add video track if needed + if (callContext.mxCall.isVideoCall) { + val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) + val frontCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isFrontFacing(it) } + ?: cameraIterator.deviceNames?.first() + + val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + + val videoSource = peerConnectionFactory!!.createVideoSource(videoCapturer.isScreencast) + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + Timber.v("## VOIP Local video source created") + + videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) + // HD + videoCapturer.startCapture(1280, 720, 30) + + this.videoCapturer = videoCapturer + + val localVideoTrack = peerConnectionFactory!!.createVideoTrack("ARDAMSv0", videoSource) + Timber.v("## VOIP Local video track created") + localVideoTrack?.setEnabled(true) + + callContext.localVideoSource = videoSource + callContext.localVideoTrack = localVideoTrack + +// localViewRenderer?.let { localVideoTrack?.addSink(it) } + localMediaStream?.addTrack(localVideoTrack) + callContext.localMediaStream = localMediaStream +// remoteVideoTrack?.setEnabled(true) +// remoteVideoTrack?.let { +// it.setEnabled(true) +// it.addSink(remoteViewRenderer) +// } } } private fun attachViewRenderersInternal() { - executor.execute { - audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - localAudioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) - localAudioTrack?.setEnabled(true) - localViewRenderer?.setMirror(true) - localVideoTrack?.addSink(localViewRenderer) + // render local video in pip view + localSurfaceRenderer?.get()?.let { pipSurface -> + pipSurface.setMirror(true) + currentCall?.localVideoTrack?.addSink(pipSurface) + } - localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? - - if (currentCall?.isVideoCall == true) { - val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) - val frontCamera = cameraIterator.deviceNames - ?.firstOrNull { cameraIterator.isFrontFacing(it) } - ?: cameraIterator.deviceNames?.first() - - val videoCapturer = cameraIterator.createCapturer(frontCamera, null) - - videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) - val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) - Timber.v("## VOIP Local video source created") - videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) - videoCapturer.startCapture(1280, 720, 30) - localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) - Timber.v("## VOIP Local video track created") - localSurfaceRenderer?.get()?.let { surface -> - localVideoTrack?.addSink(surface) - } - localVideoTrack?.setEnabled(true) - - localVideoTrack?.addSink(localViewRenderer) - localMediaStream?.addTrack(localVideoTrack) - remoteVideoTrack?.let { - it.setEnabled(true) - it.addSink(remoteViewRenderer) - } + // If remote track exists, then sink it to surface + remoteSurfaceRenderer?.get()?.let { participantSurface -> + currentCall?.remoteVideoTrack?.let { + it.setEnabled(true) + it.addSink(participantSurface) } + } + } - localMediaStream?.addTrack(localAudioTrack) - - Timber.v("## VOIP add local stream to peer connection") - peerConnection?.addStream(localMediaStream) + fun acceptIncomingCall() { + Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") + if (currentCall?.mxCall?.state == CallState.LOCAL_RINGING) { + getTurnServer { turnServer -> + internalAcceptIncomingCall(currentCall!!, turnServer) + } } } fun detachRenderers() { + Timber.v("## VOIP detachRenderers") + //currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } localSurfaceRenderer?.get()?.let { - localVideoTrack?.removeSink(it) + currentCall?.localVideoTrack?.removeSink(it) } remoteSurfaceRenderer?.get()?.let { - remoteVideoTrack?.removeSink(it) + currentCall?.remoteVideoTrack?.removeSink(it) } localSurfaceRenderer = null remoteSurfaceRenderer = null } fun close() { + CallService.onNoActiveCall(context) executor.execute { - // Do not dispose peer connection (https://bugs.chromium.org/p/webrtc/issues/detail?id=7543) - tryThis { audioSource?.dispose() } - tryThis { videoSource?.dispose() } - tryThis { videoCapturer?.stopCapture() } - tryThis { videoCapturer?.dispose() } - localMediaStream?.let { peerConnection?.removeStream(it) } - peerConnection?.close() - peerConnection = null - peerConnectionFactory?.stopAecDump() - peerConnectionFactory = null + currentCall?.release() + videoCapturer?.stopCapture() + videoCapturer?.dispose() + videoCapturer = null } - iceCandidateDisposable?.dispose() - context.stopService(Intent(context, CallHeadsUpService::class.java)) } companion object { @@ -305,126 +440,134 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun startOutgoingCall(context: Context, signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { + Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall") val createdCall = sessionHolder.getSafeActiveSession()?.callService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return - currentCall = createdCall + val callContext = CallContext(createdCall) + currentCall = callContext - startHeadsUpService(createdCall) + executor.execute { + callContext.remoteCandidateSource = ReplaySubject.create() + } + + // start the activity now context.startActivity(VectorCallActivity.newIntent(context, createdCall)) + } - sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { - override fun onSuccess(data: TurnServer?) { - startCall(data) - sendSdpOffer() + override fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) { + Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId}") + if (currentCall?.mxCall?.callId != mxCall.callId) return Unit.also { + Timber.w("## VOIP ignore ice candidates from other call") + } + val callContext = currentCall ?: return + + executor.execute { + iceCandidatesContent.candidates.forEach { + Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId} sdp: ${it.candidate}") + val iceCandidate = IceCandidate(it.sdpMid, it.sdpMLineIndex, it.candidate) + callContext.remoteCandidateSource?.onNext(iceCandidate) } - }) + } } override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { + Timber.v("## VOIP onCallInviteReceived callId ${mxCall.callId}") // TODO What if a call is currently active? if (currentCall != null) { - Timber.w("TODO: Automatically reject incoming call?") + Timber.w("## VOIP TODO: Automatically reject incoming call?") + mxCall.hangUp() return } - currentCall = mxCall - - startHeadsUpService(mxCall) - - sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { - override fun onSuccess(data: TurnServer?) { - startCall(data) - setInviteRemoteDescription(callInviteContent.offer?.sdp) - } - }) - } - - private fun setInviteRemoteDescription(description: String?) { + val callContext = CallContext(mxCall) + currentCall = callContext executor.execute { - val sdp = SessionDescription(SessionDescription.Type.OFFER, description) - peerConnection?.setRemoteDescription(sdpObserver, sdp) + callContext.remoteCandidateSource = ReplaySubject.create() } - } - private fun startHeadsUpService(mxCall: MxCallDetail) { - val callHeadsUpServiceIntent = CallHeadsUpService.newInstance(context, mxCall) - ContextCompat.startForegroundService(context, callHeadsUpServiceIntent) + CallService.onIncomingCall(context, + mxCall.isVideoCall, + mxCall.otherUserId, + mxCall.roomId, + sessionHolder.getSafeActiveSession()?.myUserId ?: "", + mxCall.callId) - context.bindService(Intent(context, CallHeadsUpService::class.java), serviceConnection, 0) - } - - fun answerCall() { - if (currentCall != null) { - executor.execute { createAnswer() } - } + callContext.offerSdp = callInviteContent.offer } private fun createAnswer() { + Timber.w("## VOIP createAnswer") + val call = currentCall ?: return val constraints = MediaConstraints().apply { mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.isVideoCall == true) "true" else "false")) + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (call.mxCall.isVideoCall) "true" else "false")) } + executor.execute { + call.peerConnection?.createAnswer(object : SdpObserverAdapter() { - peerConnection?.createAnswer(sdpObserver, constraints) + override fun onCreateSuccess(p0: SessionDescription?) { + if (p0 == null) return + call.peerConnection?.setLocalDescription(object : SdpObserverAdapter() {}, p0) + // Now need to send it + call.mxCall.accept(p0) + } + }, constraints) + } } fun endCall() { - currentCall?.hangUp() + currentCall?.mxCall?.hangUp() currentCall = null close() } override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + val call = currentCall ?: return + if (call.mxCall.callId != callAnswerContent.callId) return Unit.also { + Timber.w("onCallAnswerReceived for non active call? ${callAnswerContent.callId}") + } executor.execute { - Timber.v("## answerReceived") + Timber.v("## VOIP onCallAnswerReceived ${callAnswerContent.callId}") val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) - peerConnection?.setRemoteDescription(sdpObserver, sdp) + call.peerConnection?.setRemoteDescription(object : SdpObserverAdapter() { + + }, sdp) } } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { + val call = currentCall ?: return + if (call.mxCall.callId != callHangupContent.callId) return Unit.also { + Timber.w("onCallHangupReceived for non active call? ${callHangupContent.callId}") + } + call.mxCall.state = CallState.TERMINATED currentCall = null close() } - private inner class SdpObserver : org.webrtc.SdpObserver { + private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer { - override fun onCreateSuccess(origSdp: SessionDescription) { - Timber.v("## VOIP SdpObserver onCreateSuccess") - if (localSdp != null) return + override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { + Timber.v("## VOIP StreamObserver onConnectionChange: $newState") + when (newState) { + PeerConnection.PeerConnectionState.CONNECTED -> { + callContext.mxCall.state = CallState.CONNECTED + } + PeerConnection.PeerConnectionState.FAILED -> { + endCall() + } + PeerConnection.PeerConnectionState.NEW, + PeerConnection.PeerConnectionState.CONNECTING, + PeerConnection.PeerConnectionState.DISCONNECTED, + PeerConnection.PeerConnectionState.CLOSED, + null -> { - executor.execute { - localSdp = SessionDescription(origSdp.type, origSdp.description) - peerConnection?.setLocalDescription(sdpObserver, localSdp) - } - } - - override fun onSetSuccess() { - Timber.v("## VOIP SdpObserver onSetSuccess") - executor.execute { - localSdp?.let { - if (currentCall?.isOutgoing == true) { - currentCall?.offerSdp(it) - } else { - currentCall?.accept(it) - currentCall?.let { context.startActivity(VectorCallActivity.newIntent(context, it)) } - } } } } - override fun onCreateFailure(error: String) { - Timber.v("## VOIP SdpObserver onCreateFailure: $error") - } - - override fun onSetFailure(error: String) { - Timber.v("## VOIP SdpObserver onSetFailure: $error") - } - } - - private inner class StreamObserver : PeerConnection.Observer { override fun onIceCandidate(iceCandidate: IceCandidate) { Timber.v("## VOIP StreamObserver onIceCandidate: $iceCandidate") - iceCandidateSource.onNext(iceCandidate) + callContext.iceCandidateSource.onNext(iceCandidate) } override fun onDataChannel(dc: DataChannel) { @@ -450,12 +593,16 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onAddStream(stream: MediaStream) { Timber.v("## VOIP StreamObserver onAddStream: $stream") executor.execute { + + // reportError("Weird-looking stream: " + stream); if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) return@execute if (stream.videoTracks.size == 1) { - remoteVideoTrack = stream.videoTracks.first() - remoteVideoTrack?.setEnabled(true) - remoteViewRenderer?.let { remoteVideoTrack?.addSink(it) } + val remoteVideoTrack = stream.videoTracks.first() + remoteVideoTrack.setEnabled(true) + callContext.remoteVideoTrack = remoteVideoTrack + // sink to renderer if attached + remoteSurfaceRenderer?.get().let { remoteVideoTrack.addSink(it) } } } } @@ -464,9 +611,9 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP StreamObserver onRemoveStream") executor.execute { remoteSurfaceRenderer?.get()?.let { - remoteVideoTrack?.removeSink(it) + callContext.remoteVideoTrack?.removeSink(it) } - remoteVideoTrack = null + callContext.remoteVideoTrack = null } } @@ -484,8 +631,16 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onRenegotiationNeeded() { Timber.v("## VOIP StreamObserver onRenegotiationNeeded") + // Should not do anything, for now we follow a pre-agreed-upon + // signaling/negotiation protocol. } + /** + * This happens when a new track of any kind is added to the media stream. + * This event is fired when the browser adds a track to the stream + * (such as when a RTCPeerConnection is renegotiated or a stream being captured using HTMLMediaElement.captureStream() + * gets a new set of tracks because the media element being captured loaded a new source. + */ override fun onAddTrack(p0: RtpReceiver?, p1: Array?) { Timber.v("## VOIP StreamObserver onAddTrack") } diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt index 4671217312..5b3b0073a8 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt @@ -21,34 +21,44 @@ import android.content.Context import android.content.Intent import im.vector.riotx.core.di.HasVectorInjector import im.vector.riotx.features.call.WebRtcPeerConnectionManager +import im.vector.riotx.features.notifications.NotificationUtils import im.vector.riotx.features.settings.VectorLocale.context import timber.log.Timber class CallHeadsUpActionReceiver : BroadcastReceiver() { private lateinit var peerConnectionManager: WebRtcPeerConnectionManager + private lateinit var notificationUtils: NotificationUtils init { val appContext = context.applicationContext if (appContext is HasVectorInjector) { peerConnectionManager = appContext.injector().webRtcPeerConnectionManager() + notificationUtils = appContext.injector().notificationUtils() } } override fun onReceive(context: Context, intent: Intent?) { - when (intent?.getIntExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, 0)) { - CallHeadsUpService.CALL_ACTION_ANSWER -> onCallAnswerClicked() - CallHeadsUpService.CALL_ACTION_REJECT -> onCallRejectClicked() - } +// when (intent?.getIntExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, 0)) { +// CallHeadsUpService.CALL_ACTION_ANSWER -> onCallAnswerClicked(context) +// CallHeadsUpService.CALL_ACTION_REJECT -> onCallRejectClicked() +// } + + // Not sure why this should be needed +// val it = Intent(Intent.ACTION_CLOSE_SYSTEM_DIALOGS) +// context.sendBroadcast(it) + + // Close the notification after the click action is performed. +// context.stopService(Intent(context, CallHeadsUpService::class.java)) } - private fun onCallRejectClicked() { - Timber.d("onCallRejectClicked") - peerConnectionManager.endCall() - } +// private fun onCallRejectClicked() { +// Timber.d("onCallRejectClicked") +// peerConnectionManager.endCall() +// } - private fun onCallAnswerClicked() { - Timber.d("onCallAnswerClicked") - peerConnectionManager.answerCall() - } +// private fun onCallAnswerClicked(context: Context) { +// Timber.d("onCallAnswerClicked") +// peerConnectionManager.answerCall(context) +// } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt index f8ec1c6ee1..181d360d14 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt @@ -1,144 +1,180 @@ -/* - * Copyright (c) 2020 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package im.vector.riotx.features.call.service - -import android.app.Notification -import android.app.NotificationChannel -import android.app.NotificationManager -import android.app.PendingIntent -import android.app.Service -import android.content.ContentResolver.SCHEME_ANDROID_RESOURCE -import android.content.Context -import android.content.Intent -import android.media.AudioAttributes -import android.net.Uri -import android.os.Binder -import android.os.Build -import android.os.IBinder -import androidx.core.app.NotificationCompat -import im.vector.matrix.android.api.session.call.MxCallDetail -import im.vector.riotx.R -import im.vector.riotx.features.call.VectorCallActivity - -class CallHeadsUpService : Service() { - - private val CHANNEL_ID = "CallChannel" - private val CHANNEL_NAME = "Call Channel" - private val CHANNEL_DESCRIPTION = "Call Notifications" - - private val binder: IBinder = CallHeadsUpServiceBinder() - - override fun onBind(intent: Intent): IBinder? { - return binder - } - - override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { - val callHeadsUpServiceArgs: CallHeadsUpServiceArgs? = intent?.extras?.getParcelable(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS) - - createNotificationChannel() - - val title = callHeadsUpServiceArgs?.otherUserId ?: "" - val description = when { - callHeadsUpServiceArgs?.isIncomingCall == false -> getString(R.string.call_ring) - callHeadsUpServiceArgs?.isVideoCall == true -> getString(R.string.incoming_video_call) - else -> getString(R.string.incoming_voice_call) - } - - val actions = if (callHeadsUpServiceArgs?.isIncomingCall == true) createAnswerAndRejectActions() else emptyList() - - createNotification(title, description, actions).also { - startForeground(NOTIFICATION_ID, it) - } - - return START_STICKY - } - - private fun createNotification(title: String, content: String, actions: List): Notification { - return NotificationCompat - .Builder(applicationContext, CHANNEL_ID) - .setContentTitle(title) - .setContentText(content) - .setSmallIcon(R.drawable.ic_call) - .setPriority(NotificationCompat.PRIORITY_MAX) - .setCategory(NotificationCompat.CATEGORY_CALL) - .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) - .setDefaults(NotificationCompat.DEFAULT_SOUND or NotificationCompat.DEFAULT_VIBRATE) - .setSound(Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg")) - .setVibrate(longArrayOf(1000, 1000)) - .setFullScreenIntent(PendingIntent.getActivity(applicationContext, 0, Intent(applicationContext, VectorCallActivity::class.java), 0), true) - .setOngoing(true) - .apply { actions.forEach { addAction(it) } } - .build() - } - - private fun createAnswerAndRejectActions(): List { - val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { - putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) - } - val rejectCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { - putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_REJECT) - } - val answerCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) - val rejectCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_REJECT, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) - return listOf( - NotificationCompat.Action(R.drawable.ic_call, getString(R.string.call_notification_answer), answerCallPendingIntent), - NotificationCompat.Action(R.drawable.vector_notification_reject_invitation, getString(R.string.call_notification_reject), rejectCallPendingIntent) - ) - } - - private fun createNotificationChannel() { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return - - val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH).apply { - description = CHANNEL_DESCRIPTION - setSound( - Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg"), - AudioAttributes - .Builder() - .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) - .setUsage(AudioAttributes.USAGE_NOTIFICATION_RINGTONE) - .build() - ) - lockscreenVisibility = NotificationCompat.VISIBILITY_PUBLIC - enableVibration(true) - enableLights(true) - } - applicationContext.getSystemService(NotificationManager::class.java)?.createNotificationChannel(channel) - } - - inner class CallHeadsUpServiceBinder : Binder() { - - fun getService() = this@CallHeadsUpService - } - - companion object { - private const val EXTRA_CALL_HEADS_UP_SERVICE_PARAMS = "EXTRA_CALL_PARAMS" - - const val EXTRA_CALL_ACTION_KEY = "EXTRA_CALL_ACTION_KEY" - const val CALL_ACTION_ANSWER = 100 - const val CALL_ACTION_REJECT = 101 - - private const val NOTIFICATION_ID = 999 - - fun newInstance(context: Context, mxCall: MxCallDetail): Intent { - val args = CallHeadsUpServiceArgs(mxCall.roomId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall) - return Intent(context, CallHeadsUpService::class.java).apply { - putExtra(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS, args) - } - } - } -} +///* +// * Copyright (c) 2020 New Vector Ltd +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package im.vector.riotx.features.call.service +// +//import android.app.Notification +//import android.app.NotificationChannel +//import android.app.NotificationManager +//import android.app.PendingIntent +//import android.app.Service +//import android.content.ContentResolver.SCHEME_ANDROID_RESOURCE +//import android.content.Context +//import android.content.Intent +//import android.media.AudioAttributes +//import android.net.Uri +//import android.os.Binder +//import android.os.Build +//import android.os.IBinder +//import androidx.core.app.NotificationCompat +//import androidx.core.content.ContextCompat +//import androidx.core.graphics.drawable.IconCompat +//import im.vector.matrix.android.api.session.call.MxCallDetail +//import im.vector.riotx.R +//import im.vector.riotx.core.extensions.vectorComponent +//import im.vector.riotx.features.call.VectorCallActivity +//import im.vector.riotx.features.notifications.NotificationUtils +//import im.vector.riotx.features.themes.ThemeUtils +// +//class CallHeadsUpService : Service() { +//// +//// private val CHANNEL_ID = "CallChannel" +//// private val CHANNEL_NAME = "Call Channel" +//// private val CHANNEL_DESCRIPTION = "Call Notifications" +// +// lateinit var notificationUtils: NotificationUtils +// private val binder: IBinder = CallHeadsUpServiceBinder() +// +// override fun onBind(intent: Intent): IBinder? { +// return binder +// } +// +// override fun onCreate() { +// super.onCreate() +// notificationUtils = vectorComponent().notificationUtils() +// } +// override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { +// val callHeadsUpServiceArgs: CallHeadsUpServiceArgs? = intent?.extras?.getParcelable(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS) +// +//// createNotificationChannel() +// +//// val title = callHeadsUpServiceArgs?.otherUserId ?: "" +//// val description = when { +//// callHeadsUpServiceArgs?.isIncomingCall == false -> getString(R.string.call_ring) +//// callHeadsUpServiceArgs?.isVideoCall == true -> getString(R.string.incoming_video_call) +//// else -> getString(R.string.incoming_voice_call) +//// } +// +// // val actions = if (callHeadsUpServiceArgs?.isIncomingCall == true) createAnswerAndRejectActions() else emptyList() +// +// notificationUtils.buildIncomingCallNotification( +// callHeadsUpServiceArgs?.isVideoCall ?: false, +// callHeadsUpServiceArgs?.otherUserId ?: "", +// callHeadsUpServiceArgs?.roomId ?: "", +// callHeadsUpServiceArgs?.callId ?: "" +// ).let { +// startForeground(NOTIFICATION_ID, it) +// } +//// createNotification(title, description, actions).also { +//// startForeground(NOTIFICATION_ID, it) +//// } +// +// return START_STICKY +// } +// +//// private fun createNotification(title: String, content: String, actions: List): Notification { +//// val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { +//// putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) +//// }.let { +//// PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, it, PendingIntent.FLAG_UPDATE_CURRENT) +//// } +//// return NotificationCompat +//// .Builder(applicationContext, CHANNEL_ID) +//// .setContentTitle(title) +//// .setContentText(content) +//// .setSmallIcon(R.drawable.ic_call) +//// .setPriority(NotificationCompat.PRIORITY_MAX) +//// .setWhen(0) +//// .setCategory(NotificationCompat.CATEGORY_CALL) +//// .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) +//// .setDefaults(NotificationCompat.DEFAULT_SOUND or NotificationCompat.DEFAULT_VIBRATE) +//// .setSound(Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg")) +//// .setVibrate(longArrayOf(1000, 1000)) +//// .setFullScreenIntent(answerCallActionReceiver, true) +//// .setOngoing(true) +//// //.setStyle(NotificationCompat.BigTextStyle()) +//// .setAutoCancel(true) +//// .apply { actions.forEach { addAction(it) } } +//// .build() +//// } +// +//// private fun createAnswerAndRejectActions(): List { +//// val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { +//// putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) +//// } +//// val rejectCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { +//// putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_REJECT) +//// } +//// val answerCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) +//// val rejectCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_REJECT, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) +//// +//// return listOf( +//// NotificationCompat.Action( +//// R.drawable.ic_call, +//// //IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), +//// getString(R.string.call_notification_answer), +//// answerCallPendingIntent +//// ), +//// NotificationCompat.Action( +//// IconCompat.createWithResource(applicationContext, R.drawable.ic_call_end).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_notice)), +//// getString(R.string.call_notification_reject), +//// rejectCallPendingIntent) +//// ) +//// } +// +//// private fun createNotificationChannel() { +//// if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return +//// +//// val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH).apply { +//// description = CHANNEL_DESCRIPTION +//// setSound( +//// Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg"), +//// AudioAttributes +//// .Builder() +//// .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) +//// .setUsage(AudioAttributes.USAGE_NOTIFICATION_RINGTONE) +//// .build() +//// ) +//// lockscreenVisibility = NotificationCompat.VISIBILITY_PUBLIC +//// enableVibration(true) +//// enableLights(true) +//// } +//// applicationContext.getSystemService(NotificationManager::class.java)?.createNotificationChannel(channel) +//// } +// +// inner class CallHeadsUpServiceBinder : Binder() { +// +// fun getService() = this@CallHeadsUpService +// } +// +// +// companion object { +// private const val EXTRA_CALL_HEADS_UP_SERVICE_PARAMS = "EXTRA_CALL_PARAMS" +// +// const val EXTRA_CALL_ACTION_KEY = "EXTRA_CALL_ACTION_KEY" +//// const val CALL_ACTION_ANSWER = 100 +// const val CALL_ACTION_REJECT = 101 +// +// private const val NOTIFICATION_ID = 999 +// +// fun newInstance(context: Context, mxCall: MxCallDetail): Intent { +// val args = CallHeadsUpServiceArgs(mxCall.callId, mxCall.roomId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall) +// return Intent(context, CallHeadsUpService::class.java).apply { +// putExtra(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS, args) +// } +// } +// } +//} diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt index 1bc857e4a7..e0b774b3da 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt @@ -21,6 +21,7 @@ import kotlinx.android.parcel.Parcelize @Parcelize data class CallHeadsUpServiceArgs( + val callId: String, val roomId: String, val otherUserId: String, val isIncomingCall: Boolean, diff --git a/vector/src/main/java/im/vector/riotx/features/call/telecom/TelecomUtils.kt b/vector/src/main/java/im/vector/riotx/features/call/telecom/TelecomUtils.kt new file mode 100644 index 0000000000..61bb8980bf --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/telecom/TelecomUtils.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call.telecom + +import android.content.Context +import android.telephony.TelephonyManager + +object TelecomUtils { + + fun isLineBusy(context: Context): Boolean { + val telephonyManager = context.getSystemService(Context.TELEPHONY_SERVICE) as? TelephonyManager + ?: return false + return telephonyManager.callState != TelephonyManager.CALL_STATE_IDLE + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt b/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt index f15cb801ed..fae9a3c607 100644 --- a/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt @@ -48,6 +48,7 @@ import im.vector.riotx.core.extensions.configureAndStart import im.vector.riotx.core.extensions.exhaustive import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.resources.StringProvider +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.notifications.PushRuleTriggerListener import im.vector.riotx.features.session.SessionListener import im.vector.riotx.features.signout.soft.SoftLogoutActivity @@ -66,7 +67,8 @@ class LoginViewModel @AssistedInject constructor( private val homeServerConnectionConfigFactory: HomeServerConnectionConfigFactory, private val sessionListener: SessionListener, private val reAuthHelper: ReAuthHelper, - private val stringProvider: StringProvider) + private val stringProvider: StringProvider, + private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager) : VectorViewModel(initialState) { @AssistedInject.Factory @@ -613,6 +615,7 @@ class LoginViewModel @AssistedInject constructor( private fun onSessionCreated(session: Session) { activeSessionHolder.setActiveSession(session) session.configureAndStart(applicationContext, pushRuleTriggerListener, sessionListener) + session.callService().addCallListener(webRtcPeerConnectionManager) setState { copy( asyncLoginAction = Success(Unit) diff --git a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt index 178235ab5f..458925178e 100755 --- a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt +++ b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt @@ -35,11 +35,14 @@ import androidx.core.app.NotificationManagerCompat import androidx.core.app.RemoteInput import androidx.core.app.TaskStackBuilder import androidx.core.content.ContextCompat +import androidx.core.graphics.drawable.IconCompat import androidx.fragment.app.Fragment import im.vector.riotx.BuildConfig import im.vector.riotx.R import im.vector.riotx.core.resources.StringProvider import im.vector.riotx.core.utils.startNotificationChannelSettingsIntent +import im.vector.riotx.features.call.VectorCallActivity +import im.vector.riotx.features.call.service.CallHeadsUpActionReceiver import im.vector.riotx.features.home.HomeActivity import im.vector.riotx.features.home.room.detail.RoomDetailActivity import im.vector.riotx.features.home.room.detail.RoomDetailArgs @@ -263,13 +266,13 @@ class NotificationUtils @Inject constructor(private val context: Context, */ @SuppressLint("NewApi") fun buildIncomingCallNotification(isVideo: Boolean, - roomName: String, - matrixId: String, + otherUserId: String, + roomId: String, callId: String): Notification { val accentColor = ContextCompat.getColor(context, R.color.notification_accent_color) val builder = NotificationCompat.Builder(context, CALL_NOTIFICATION_CHANNEL_ID) - .setContentTitle(ensureTitleNotEmpty(roomName)) + .setContentTitle(ensureTitleNotEmpty(otherUserId)) .apply { if (isVideo) { setContentText(stringProvider.getString(R.string.incoming_video_call)) @@ -282,26 +285,61 @@ class NotificationUtils @Inject constructor(private val context: Context, .setLights(accentColor, 500, 500) // Compat: Display the incoming call notification on the lock screen - builder.priority = NotificationCompat.PRIORITY_MAX + builder.priority = NotificationCompat.PRIORITY_HIGH // clear the activity stack to home activity - val intent = Intent(context, HomeActivity::class.java) - .setFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_NEW_TASK) + // val intent = Intent(context, HomeActivity::class.java) + // .setFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_NEW_TASK) // TODO .putExtra(VectorHomeActivity.EXTRA_CALL_SESSION_ID, matrixId) // TODO .putExtra(VectorHomeActivity.EXTRA_CALL_ID, callId) // Recreate the back stack - val stackBuilder = TaskStackBuilder.create(context) - .addParentStack(HomeActivity::class.java) - .addNextIntent(intent) +// val stackBuilder = TaskStackBuilder.create(context) +// .addParentStack(HomeActivity::class.java) +// .addNextIntent(intent) // android 4.3 issue // use a generator for the private requestCode. // When using 0, the intent is not created/launched when the user taps on the notification. // - val pendingIntent = stackBuilder.getPendingIntent(Random.nextInt(1000), PendingIntent.FLAG_UPDATE_CURRENT) + val requestId = Random.nextInt(1000) +// val pendingIntent = stackBuilder.getPendingIntent(requestId, PendingIntent.FLAG_UPDATE_CURRENT) + val contentPendingIntent = TaskStackBuilder.create(context) + .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) + .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, otherUserId, true, isVideo, false, VectorCallActivity.INCOMING_RINGING)) + .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) - builder.setContentIntent(pendingIntent) + val answerCallPendingIntent = TaskStackBuilder.create(context) + .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) + .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, otherUserId, true, isVideo, true, VectorCallActivity.INCOMING_ACCEPT)) + .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) + +// val answerCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { +// putExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, CallHeadsUpService.CALL_ACTION_ANSWER) +// } + val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { + // putExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, CallHeadsUpService.CALL_ACTION_REJECT) + } + //val answerCallPendingIntent = PendingIntent.getBroadcast(context, requestId, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) + val rejectCallPendingIntent = PendingIntent.getBroadcast(context, requestId + 1, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) + + builder.addAction( + NotificationCompat.Action( + R.drawable.ic_call, + //IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), + context.getString(R.string.call_notification_answer), + answerCallPendingIntent + ) + ) + + builder.addAction( + NotificationCompat.Action( + IconCompat.createWithResource(context, R.drawable.ic_call_end).setTint(ContextCompat.getColor(context, R.color.riotx_notice)), + context.getString(R.string.call_notification_reject), + rejectCallPendingIntent) + ) + + builder.setContentIntent(contentPendingIntent) return builder.build() } @@ -334,10 +372,18 @@ class NotificationUtils @Inject constructor(private val context: Context, .setSmallIcon(R.drawable.incoming_call_notification_transparent) .setCategory(NotificationCompat.CATEGORY_CALL) - // Display the pending call notification on the lock screen - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) { - builder.priority = NotificationCompat.PRIORITY_MAX - } + builder.priority = NotificationCompat.PRIORITY_DEFAULT + + val contentPendingIntent = TaskStackBuilder.create(context) + .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) + //TODO other userId + .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, "otherUserId", true, isVideo, false, null)) + .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) + + // android 4.3 issue + // use a generator for the private requestCode. + // When using 0, the intent is not created/launched when the user taps on the notification. + builder.setContentIntent(contentPendingIntent) /* TODO // Build the pending intent for when the notification is clicked diff --git a/vector/src/main/res/drawable/ic_call.xml b/vector/src/main/res/drawable/ic_call.xml index 0f4e2d1d89..430c438577 100644 --- a/vector/src/main/res/drawable/ic_call.xml +++ b/vector/src/main/res/drawable/ic_call.xml @@ -1,5 +1,14 @@ - - + + diff --git a/vector/src/main/res/drawable/ic_call_end.xml b/vector/src/main/res/drawable/ic_call_end.xml index 2879c2433e..07f7e01351 100644 --- a/vector/src/main/res/drawable/ic_call_end.xml +++ b/vector/src/main/res/drawable/ic_call_end.xml @@ -1,10 +1,14 @@ - - - + + diff --git a/vector/src/main/res/drawable/ic_microphone_off.xml b/vector/src/main/res/drawable/ic_microphone_off.xml new file mode 100644 index 0000000000..92d5044902 --- /dev/null +++ b/vector/src/main/res/drawable/ic_microphone_off.xml @@ -0,0 +1,41 @@ + + + + + + + diff --git a/vector/src/main/res/drawable/ic_microphone_on.xml b/vector/src/main/res/drawable/ic_microphone_on.xml new file mode 100644 index 0000000000..aaa9987860 --- /dev/null +++ b/vector/src/main/res/drawable/ic_microphone_on.xml @@ -0,0 +1,10 @@ + + + diff --git a/vector/src/main/res/drawable/ic_more_vertical.xml b/vector/src/main/res/drawable/ic_more_vertical.xml new file mode 100644 index 0000000000..9289a8cacb --- /dev/null +++ b/vector/src/main/res/drawable/ic_more_vertical.xml @@ -0,0 +1,18 @@ + + + + + diff --git a/vector/src/main/res/drawable/ic_video.xml b/vector/src/main/res/drawable/ic_video.xml new file mode 100644 index 0000000000..f9c57db65e --- /dev/null +++ b/vector/src/main/res/drawable/ic_video.xml @@ -0,0 +1,22 @@ + + + + diff --git a/vector/src/main/res/drawable/oval_destructive.xml b/vector/src/main/res/drawable/oval_destructive.xml new file mode 100644 index 0000000000..045a50456d --- /dev/null +++ b/vector/src/main/res/drawable/oval_destructive.xml @@ -0,0 +1,11 @@ + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/drawable/oval_positive.xml b/vector/src/main/res/drawable/oval_positive.xml new file mode 100644 index 0000000000..d2e17d746b --- /dev/null +++ b/vector/src/main/res/drawable/oval_positive.xml @@ -0,0 +1,11 @@ + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index be6165815f..8e7d2ab0a7 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -1,11 +1,11 @@ - - + android:layout_marginTop="16dp" + android:layout_marginEnd="16dp" + app:layout_constraintEnd_toEndOf="parent" + app:layout_constraintTop_toTopOf="parent" /> + + + + + app:layout_constraintStart_toStartOf="parent" + app:layout_constraintTop_toTopOf="parent" + app:layout_constraintVertical_bias="0.3" + tools:src="@tools:sample/avatars" /> - + android:layout_height="wrap_content" + android:layout_marginTop="8dp" + android:layout_marginBottom="8dp" + android:gravity="center" + android:textColor="?riotx_text_secondary" + android:textSize="12sp" + app:layout_constraintTop_toBottomOf="@id/otherMemberAvatar" + tools:text="Connecting..." /> - + - + - - + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + android:layout_height="match_parent"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index 19db06d89f..dc55a1a523 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -5,7 +5,7 @@ Date: Thu, 11 Jun 2020 15:12:46 +0200 Subject: [PATCH 33/83] revert test code --- .../java/im/vector/riotx/features/debug/DebugMenuActivity.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt index 2a8bc22e2e..a197a6f93e 100644 --- a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt +++ b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt @@ -183,9 +183,7 @@ class DebugMenuActivity : VectorBaseActivity() { @OnClick(R.id.debug_scan_qr_code) fun scanQRCode() { if (checkPermissions(PERMISSIONS_FOR_TAKING_PHOTO, this, PERMISSION_REQUEST_CODE_LAUNCH_CAMERA)) { - // doScanQRCode() - // TODO. Find a better way? - //startActivity(VectorCallActivity.newIntent(this, "!cyIJhOLwWgmmqreHLD:matrix.org")) + doScanQRCode() } } From cb964c6dcdd596c1c91e852cded3f7f4a5839331 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 11 Jun 2020 15:14:27 +0200 Subject: [PATCH 34/83] dead code --- .../riotx/features/call/CallFragment.java | 133 ------------------ 1 file changed, 133 deletions(-) delete mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallFragment.java diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java b/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java deleted file mode 100644 index 750c7b6416..0000000000 --- a/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java +++ /dev/null @@ -1,133 +0,0 @@ -///* -// * Copyright (c) 2020 New Vector Ltd -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package im.vector.riotx.features.call; -// -//import android.os.Bundle; -//import android.view.LayoutInflater; -//import android.view.View; -//import android.view.ViewGroup; -//import android.widget.ImageButton; -//import android.widget.SeekBar; -//import android.widget.TextView; -//import org.webrtc.RendererCommon.ScalingType; -// -//import androidx.fragment.app.Fragment; -// -//import im.vector.riotx.R; -// -///** -// * Fragment for call control. -// */ -//public class CallFragment extends Fragment { -// private TextView contactView; -// private ImageButton cameraSwitchButton; -// private ImageButton videoScalingButton; -// private ImageButton toggleMuteButton; -// private TextView captureFormatText; -// private SeekBar captureFormatSlider; -// private OnCallEvents callEvents; -// private ScalingType scalingType; -// private boolean videoCallEnabled = true; -// /** -// * Call control interface for container activity. -// */ -// public interface OnCallEvents { -// void onCallHangUp(); -// void onCameraSwitch(); -// void onVideoScalingSwitch(ScalingType scalingType); -// void onCaptureFormatChange(int width, int height, int framerate); -// boolean onToggleMic(); -// } -// @Override -// public View onCreateView( -// LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { -// View controlView = inflater.inflate(R.layout.fragment_call, container, false); -// // Create UI controls. -// contactView = controlView.findViewById(R.id.contact_name_call); -// ImageButton disconnectButton = controlView.findViewById(R.id.button_call_disconnect); -// cameraSwitchButton = controlView.findViewById(R.id.button_call_switch_camera); -// videoScalingButton = controlView.findViewById(R.id.button_call_scaling_mode); -// toggleMuteButton = controlView.findViewById(R.id.button_call_toggle_mic); -// captureFormatText = controlView.findViewById(R.id.capture_format_text_call); -// captureFormatSlider = controlView.findViewById(R.id.capture_format_slider_call); -// // Add buttons click events. -// disconnectButton.setOnClickListener(new View.OnClickListener() { -// @Override -// public void onClick(View view) { -// callEvents.onCallHangUp(); -// } -// }); -// cameraSwitchButton.setOnClickListener(new View.OnClickListener() { -// @Override -// public void onClick(View view) { -// callEvents.onCameraSwitch(); -// } -// }); -// videoScalingButton.setOnClickListener(new View.OnClickListener() { -// @Override -// public void onClick(View view) { -// if (scalingType == ScalingType.SCALE_ASPECT_FILL) { -// videoScalingButton.setBackgroundResource(R.drawable.ic_action_full_screen); -// scalingType = ScalingType.SCALE_ASPECT_FIT; -// } else { -// videoScalingButton.setBackgroundResource(R.drawable.ic_action_return_from_full_screen); -// scalingType = ScalingType.SCALE_ASPECT_FILL; -// } -// callEvents.onVideoScalingSwitch(scalingType); -// } -// }); -// scalingType = ScalingType.SCALE_ASPECT_FILL; -// toggleMuteButton.setOnClickListener(new View.OnClickListener() { -// @Override -// public void onClick(View view) { -// boolean enabled = callEvents.onToggleMic(); -// toggleMuteButton.setAlpha(enabled ? 1.0f : 0.3f); -// } -// }); -// return controlView; -// } -// @Override -// public void onStart() { -// super.onStart(); -// boolean captureSliderEnabled = false; -// Bundle args = getArguments(); -// if (args != null) { -// String contactName = args.getString(CallActivity.EXTRA_ROOMID); -// contactView.setText(contactName); -// videoCallEnabled = args.getBoolean(CallActivity.EXTRA_VIDEO_CALL, true); -// captureSliderEnabled = videoCallEnabled -// && args.getBoolean(CallActivity.EXTRA_VIDEO_CAPTUREQUALITYSLIDER_ENABLED, false); -// } -// if (!videoCallEnabled) { -// cameraSwitchButton.setVisibility(View.INVISIBLE); -// } -// if (captureSliderEnabled) { -// captureFormatSlider.setOnSeekBarChangeListener( -// new CaptureQualityController(captureFormatText, callEvents)); -// } else { -// captureFormatText.setVisibility(View.GONE); -// captureFormatSlider.setVisibility(View.GONE); -// } -// } -// // TODO(sakal): Replace with onAttach(Context) once we only support API level 23+. -// @SuppressWarnings("deprecation") -// @Override -// public void onAttach(Activity activity) { -// super.onAttach(activity); -// callEvents = (OnCallEvents) activity; -// } -//} From b5cdb44642d84f93b3937bff7fb4d251cd75971b Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 11 Jun 2020 15:44:14 +0200 Subject: [PATCH 35/83] Fix rebase --- .../matrix/android/api/session/Session.kt | 5 +- .../internal/session/DefaultSession.kt | 2 +- .../src/main/res/values/strings.xml | 1 + .../java/im/vector/riotx/VectorApplication.kt | 2 +- .../features/call/VectorCallViewModel.kt | 2 +- .../call/WebRtcPeerConnectionManager.kt | 4 +- .../home/room/detail/RoomDetailFragment.kt | 14 ----- .../format/DisplayableEventFormatter.kt | 53 +++++++++++-------- .../riotx/features/login/LoginViewModel.kt | 2 +- vector/src/main/res/menu/menu_timeline.xml | 2 +- 10 files changed, 42 insertions(+), 45 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt index 666983256e..ab453e5fbe 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt @@ -155,7 +155,6 @@ interface Session : * Returns the identity service associated with the session */ fun identityService(): IdentityService - fun callService(): CallSignalingService /** * Returns the widget service associated with the session @@ -168,9 +167,9 @@ interface Session : fun integrationManagerService(): IntegrationManagerService /** - * Returns the cryptoService associated with the session + * Returns the call signaling service associated with the session */ - fun callService(): CallService + fun callSignalingService(): CallSignalingService /** * Add a listener to the session. diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt index 3202b84173..816879af45 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt @@ -247,7 +247,7 @@ internal class DefaultSession @Inject constructor( override fun integrationManagerService() = integrationManagerService - override fun callService(): CallSignalingService = callSignalingService.get() + override fun callSignalingService(): CallSignalingService = callSignalingService.get() override fun addListener(listener: Session.Listener) { sessionListeners.addListener(listener) diff --git a/matrix-sdk-android/src/main/res/values/strings.xml b/matrix-sdk-android/src/main/res/values/strings.xml index c749441746..bab4f6c622 100644 --- a/matrix-sdk-android/src/main/res/values/strings.xml +++ b/matrix-sdk-android/src/main/res/values/strings.xml @@ -41,6 +41,7 @@ You changed the room name to: %1$s %s placed a video call. You placed a video call. + You placed a voice call. %s placed a voice call. %s sent data to setup the call. %s answered the call. diff --git a/vector/src/main/java/im/vector/riotx/VectorApplication.kt b/vector/src/main/java/im/vector/riotx/VectorApplication.kt index a8e597dadb..ab723c2b3d 100644 --- a/vector/src/main/java/im/vector/riotx/VectorApplication.kt +++ b/vector/src/main/java/im/vector/riotx/VectorApplication.kt @@ -125,7 +125,7 @@ class VectorApplication : Application(), HasVectorInjector, MatrixConfiguration. val lastAuthenticatedSession = authenticationService.getLastAuthenticatedSession()!! activeSessionHolder.setActiveSession(lastAuthenticatedSession) lastAuthenticatedSession.configureAndStart(applicationContext, pushRuleTriggerListener, sessionListener) - lastAuthenticatedSession.callService().addCallListener(webRtcPeerConnectionManager) + lastAuthenticatedSession.callSignalingService().addCallListener(webRtcPeerConnectionManager) } ProcessLifecycleOwner.get().lifecycle.addObserver(object : LifecycleObserver { @OnLifecycleEvent(Lifecycle.Event.ON_RESUME) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 3d4c5b6445..59ea287d5e 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -104,7 +104,7 @@ class VectorCallViewModel @AssistedInject constructor( autoReplyIfNeeded = args.autoAccept initialState.callId?.let { - session.callService().getCallWithId(it)?.let { mxCall -> + session.callSignalingService().getCallWithId(it)?.let { mxCall -> this.call = mxCall mxCall.otherUserId val item: MatrixItem? = session.getUser(mxCall.otherUserId)?.toMatrixItem() diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 1e95d7b7c5..971c0f1297 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -216,7 +216,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun getTurnServer(callback: ((TurnServer?) -> Unit)) { - sessionHolder.getActiveSession().callService().getTurnServer(object : MatrixCallback { + sessionHolder.getActiveSession().callSignalingService().getTurnServer(object : MatrixCallback { override fun onSuccess(data: TurnServer?) { callback(data) } @@ -441,7 +441,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun startOutgoingCall(context: Context, signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall") - val createdCall = sessionHolder.getSafeActiveSession()?.callService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return + val createdCall = sessionHolder.getSafeActiveSession()?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return val callContext = CallContext(createdCall) currentCall = callContext diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index f340a48bd7..d0ec5a7894 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -479,20 +479,6 @@ class RoomDetailFragment @Inject constructor( } true } - else -> super.onOptionsItemSelected(item) - } - if (item.itemId == R.id.resend_all) { - roomDetailViewModel.handle(RoomDetailAction.ResendAll) - return true - } - if (item.itemId == R.id.voice_call || item.itemId == R.id.video_call) { - roomDetailViewModel.getOtherUserIds()?.firstOrNull()?.let { - webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) - } - R.id.resend_all -> { - roomDetailViewModel.handle(RoomDetailAction.ResendAll) - true - } R.id.voice_call, R.id.video_call -> { roomDetailViewModel.getOtherUserIds()?.firstOrNull()?.let { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt index 0f628d2ce4..c931c155b1 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/DisplayableEventFormatter.kt @@ -34,7 +34,7 @@ class DisplayableEventFormatter @Inject constructor( private val noticeEventFormatter: NoticeEventFormatter ) { - fun format(timelineEvent: TimelineEvent, prependAuthor: Boolean): CharSequence { + fun format(timelineEvent: TimelineEvent, appendAuthor: Boolean): CharSequence { if (timelineEvent.root.isRedacted()) { return noticeEventFormatter.formatRedactedEvent(timelineEvent.root) } @@ -53,39 +53,50 @@ class DisplayableEventFormatter @Inject constructor( EventType.MESSAGE -> { timelineEvent.getLastMessageContent()?.let { messageContent -> when (messageContent.msgType) { - MessageType.MSGTYPE_VERIFICATION_REQUEST -> - simpleFormat(senderName, stringProvider.getString(R.string.verification_request), prependAuthor) - MessageType.MSGTYPE_IMAGE -> - simpleFormat(senderName, stringProvider.getString(R.string.sent_an_image), prependAuthor) - MessageType.MSGTYPE_AUDIO -> - simpleFormat(senderName, stringProvider.getString(R.string.sent_an_audio_file), prependAuthor) - MessageType.MSGTYPE_VIDEO -> - simpleFormat(senderName, stringProvider.getString(R.string.sent_a_video), prependAuthor) - MessageType.MSGTYPE_FILE -> - simpleFormat(senderName, stringProvider.getString(R.string.sent_a_file), prependAuthor) - MessageType.MSGTYPE_TEXT -> + MessageType.MSGTYPE_VERIFICATION_REQUEST -> { + return simpleFormat(senderName, stringProvider.getString(R.string.verification_request), appendAuthor) + } + MessageType.MSGTYPE_IMAGE -> { + return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_image), appendAuthor) + } + MessageType.MSGTYPE_AUDIO -> { + return simpleFormat(senderName, stringProvider.getString(R.string.sent_an_audio_file), appendAuthor) + } + MessageType.MSGTYPE_VIDEO -> { + return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_video), appendAuthor) + } + MessageType.MSGTYPE_FILE -> { + return simpleFormat(senderName, stringProvider.getString(R.string.sent_a_file), appendAuthor) + } + MessageType.MSGTYPE_TEXT -> { if (messageContent.isReply()) { // Skip reply prefix, and show important // TODO add a reply image span ? - simpleFormat(senderName, timelineEvent.getTextEditableContent() ?: messageContent.body, prependAuthor) + return simpleFormat(senderName, timelineEvent.getTextEditableContent() + ?: messageContent.body, appendAuthor) } else { - simpleFormat(senderName, messageContent.body, prependAuthor) + return simpleFormat(senderName, messageContent.body, appendAuthor) } - else -> - simpleFormat(senderName, messageContent.body, prependAuthor) + } + else -> { + return simpleFormat(senderName, messageContent.body, appendAuthor) + } } - } ?: span { } + } } - else -> - span { + else -> { + return span { text = noticeEventFormatter.format(timelineEvent) ?: "" textStyle = "italic" } + } } + + return span { } } - private fun simpleFormat(senderName: String, body: CharSequence, prependAuthor: Boolean): CharSequence { - return if (prependAuthor) { + private fun simpleFormat(senderName: String, body: CharSequence, appendAuthor: Boolean): CharSequence { + return if (appendAuthor) { span { text = senderName textColor = colorProvider.getColorFromAttribute(R.attr.riotx_text_primary) diff --git a/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt b/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt index fae9a3c607..abdea9698f 100644 --- a/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/login/LoginViewModel.kt @@ -615,7 +615,7 @@ class LoginViewModel @AssistedInject constructor( private fun onSessionCreated(session: Session) { activeSessionHolder.setActiveSession(session) session.configureAndStart(applicationContext, pushRuleTriggerListener, sessionListener) - session.callService().addCallListener(webRtcPeerConnectionManager) + session.callSignalingService().addCallListener(webRtcPeerConnectionManager) setState { copy( asyncLoginAction = Success(Unit) diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index dc55a1a523..701d02c3ea 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -18,7 +18,7 @@ app:showAsAction="never" /> Date: Thu, 11 Jun 2020 16:26:26 +0200 Subject: [PATCH 36/83] klint --- .../android/api/session/call/CallState.kt | 1 - .../android/internal/session/SessionModule.kt | 1 - .../internal/session/call/CallModule.kt | 1 - .../call/DefaultCallSignalingService.kt | 3 +- .../internal/session/call/model/MxCallImpl.kt | 1 - .../vector/riotx/core/services/CallService.kt | 3 - .../riotx/features/call/CallControlsView.kt | 2 +- .../riotx/features/call/VectorCallActivity.kt | 6 +- .../features/call/VectorCallViewModel.kt | 4 +- .../call/WebRtcPeerConnectionManager.kt | 20 +- .../call/service/CallHeadsUpActionReceiver.kt | 1 - .../call/service/CallHeadsUpService.kt | 216 +++++++++--------- .../notifications/NotificationUtils.kt | 6 +- 13 files changed, 122 insertions(+), 143 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt index 92fbf405ab..713735e336 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt @@ -38,5 +38,4 @@ enum class CallState { /** Terminated. Incoming/Outgoing call, the call is terminated */ TERMINATED, - } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt index b756c9dbc1..65b79e7db7 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt @@ -274,5 +274,4 @@ internal abstract class SessionModule { @Binds abstract fun bindSharedSecretStorageService(service: DefaultSharedSecretStorageService): SharedSecretStorageService - } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt index 4387b14297..6f0add43df 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt @@ -37,7 +37,6 @@ internal abstract class CallModule { } } - @Binds abstract fun bindCallService(service:DefaultCallSignalingService): CallSignalingService diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt index bbd5554cae..fa486b7310 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt @@ -101,7 +101,7 @@ internal class DefaultCallSignalingService @Inject constructor( // TODO if handled by other of my sessions // this test is too simple, should notify upstream if (event.senderId == userId) { - //ignore local echos! + // ignore local echos! return } when (event.getClearType()) { @@ -139,7 +139,6 @@ internal class DefaultCallSignalingService @Inject constructor( } } } - } } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt index fe22099018..6c9c7d7184 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt @@ -80,7 +80,6 @@ internal class MxCallImpl( } } - override fun offerSdp(sdp: SessionDescription) { if (!isOutgoing) return state = CallState.REMOTE_RINGING diff --git a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt index 6e86f54797..8337e56403 100644 --- a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt @@ -15,7 +15,6 @@ * limitations under the License. */ - package im.vector.riotx.core.services import android.content.Context @@ -90,8 +89,6 @@ class CallService : VectorService() { private fun displayIncomingCallNotification(intent: Intent) { Timber.v("displayIncomingCallNotification") - - // the incoming call in progress is already displayed if (!TextUtils.isEmpty(mIncomingCallId)) { Timber.v("displayIncomingCallNotification : the incoming call in progress is already displayed") diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index 4f9cfe1d7f..ee7a9a4797 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -46,7 +46,7 @@ class CallControlsView @JvmOverloads constructor( init { ConstraintLayout.inflate(context, R.layout.fragment_call_controls, this) - //layoutParams = FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT) + // layoutParams = FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT) ButterKnife.bind(this) } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index ceffe8d15f..4ba9af0c0e 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -16,7 +16,7 @@ package im.vector.riotx.features.call -//import im.vector.riotx.features.call.service.CallHeadsUpService +// import im.vector.riotx.features.call.service.CallHeadsUpService import android.app.KeyguardManager import android.content.Context import android.content.Intent @@ -161,7 +161,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis renderState(it) } - callViewModel.viewEvents .observe() .observeOn(AndroidSchedulers.mainThread()) @@ -186,7 +185,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callControlsView.updateForState(state.callState.invoke()) when (state.callState.invoke()) { CallState.IDLE -> { - } CallState.DIALING -> { callVideoGroup.isInvisible = true @@ -232,7 +230,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis finish() } null -> { - } } } @@ -277,7 +274,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis pipRenderer.setEnableHardwareScaler(true /* enabled */) fullscreenRenderer.setEnableHardwareScaler(true /* enabled */) - peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() }) return false diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 59ea287d5e..77366b70d0 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -120,11 +120,11 @@ class VectorCallViewModel @AssistedInject constructor( } } - //session.callService().addCallListener(callServiceListener) + // session.callService().addCallListener(callServiceListener) } override fun onCleared() { - //session.callService().removeCallListener(callServiceListener) + // session.callService().removeCallListener(callServiceListener) this.call?.removeListener(callStateListener) super.onCleared() } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 971c0f1297..9d2b8cb2e0 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -94,7 +94,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // omit empty :/ if (it.isNotEmpty()) { Timber.v("## Sending local ice candidates to call") - //it.forEach { peerConnection?.addIceCandidate(it) } + // it.forEach { peerConnection?.addIceCandidate(it) } mxCall.sendLocalIceCandidates(it) } } @@ -103,7 +103,6 @@ class WebRtcPeerConnectionManager @Inject constructor( var remoteIceCandidateDisposable: Disposable? = null fun release() { - remoteIceCandidateDisposable?.dispose() iceCandidateDisposable?.dispose() @@ -174,7 +173,7 @@ class WebRtcPeerConnectionManager @Inject constructor( .setVideoDecoderFactory(defaultVideoDecoderFactory) .createPeerConnectionFactory() - //attachViewRenderersInternal() + // attachViewRenderersInternal() } private fun createPeerConnection(callContext: CallContext, turnServer: TurnServer?) { @@ -192,7 +191,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } } } - Timber.v("## VOIP creating peer connection...with iceServers ${iceServers} ") + Timber.v("## VOIP creating peer connection...with iceServers $iceServers ") callContext.peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, StreamObserver(callContext)) } @@ -270,7 +269,6 @@ class WebRtcPeerConnectionManager @Inject constructor( attachViewRenderersInternal() } } - } } @@ -307,7 +305,6 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun createLocalStream(callContext: CallContext) { - if (callContext.localMediaStream != null) { Timber.e("## VOIP localMediaStream already created") return @@ -325,12 +322,12 @@ class WebRtcPeerConnectionManager @Inject constructor( val localMediaStream = peerConnectionFactory!!.createLocalMediaStream("ARDAMS") // magic value? - //Add audio track + // Add audio track localMediaStream?.addTrack(localAudioTrack) callContext.localMediaStream = localMediaStream - //add video track if needed + // add video track if needed if (callContext.mxCall.isVideoCall) { val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) val frontCamera = cameraIterator.deviceNames @@ -368,7 +365,6 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun attachViewRenderersInternal() { - // render local video in pip view localSurfaceRenderer?.get()?.let { pipSurface -> pipSurface.setMirror(true) @@ -395,7 +391,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun detachRenderers() { Timber.v("## VOIP detachRenderers") - //currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } + // currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } localSurfaceRenderer?.get()?.let { currentCall?.localVideoTrack?.removeSink(it) } @@ -503,7 +499,6 @@ class WebRtcPeerConnectionManager @Inject constructor( } executor.execute { call.peerConnection?.createAnswer(object : SdpObserverAdapter() { - override fun onCreateSuccess(p0: SessionDescription?) { if (p0 == null) return call.peerConnection?.setLocalDescription(object : SdpObserverAdapter() {}, p0) @@ -529,7 +524,6 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP onCallAnswerReceived ${callAnswerContent.callId}") val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) call.peerConnection?.setRemoteDescription(object : SdpObserverAdapter() { - }, sdp) } } @@ -560,7 +554,6 @@ class WebRtcPeerConnectionManager @Inject constructor( PeerConnection.PeerConnectionState.DISCONNECTED, PeerConnection.PeerConnectionState.CLOSED, null -> { - } } } @@ -593,7 +586,6 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onAddStream(stream: MediaStream) { Timber.v("## VOIP StreamObserver onAddStream: $stream") executor.execute { - // reportError("Weird-looking stream: " + stream); if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) return@execute diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt index 5b3b0073a8..6b38a272a2 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt @@ -23,7 +23,6 @@ import im.vector.riotx.core.di.HasVectorInjector import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.notifications.NotificationUtils import im.vector.riotx.features.settings.VectorLocale.context -import timber.log.Timber class CallHeadsUpActionReceiver : BroadcastReceiver() { diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt index 181d360d14..c68b22fabb 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt @@ -1,4 +1,4 @@ -///* +// /* // * Copyright (c) 2020 New Vector Ltd // * // * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,36 +14,36 @@ // * limitations under the License. // */ // -//package im.vector.riotx.features.call.service +// package im.vector.riotx.features.call.service // -//import android.app.Notification -//import android.app.NotificationChannel -//import android.app.NotificationManager -//import android.app.PendingIntent -//import android.app.Service -//import android.content.ContentResolver.SCHEME_ANDROID_RESOURCE -//import android.content.Context -//import android.content.Intent -//import android.media.AudioAttributes -//import android.net.Uri -//import android.os.Binder -//import android.os.Build -//import android.os.IBinder -//import androidx.core.app.NotificationCompat -//import androidx.core.content.ContextCompat -//import androidx.core.graphics.drawable.IconCompat -//import im.vector.matrix.android.api.session.call.MxCallDetail -//import im.vector.riotx.R -//import im.vector.riotx.core.extensions.vectorComponent -//import im.vector.riotx.features.call.VectorCallActivity -//import im.vector.riotx.features.notifications.NotificationUtils -//import im.vector.riotx.features.themes.ThemeUtils +// import android.app.Notification +// import android.app.NotificationChannel +// import android.app.NotificationManager +// import android.app.PendingIntent +// import android.app.Service +// import android.content.ContentResolver.SCHEME_ANDROID_RESOURCE +// import android.content.Context +// import android.content.Intent +// import android.media.AudioAttributes +// import android.net.Uri +// import android.os.Binder +// import android.os.Build +// import android.os.IBinder +// import androidx.core.app.NotificationCompat +// import androidx.core.content.ContextCompat +// import androidx.core.graphics.drawable.IconCompat +// import im.vector.matrix.android.api.session.call.MxCallDetail +// import im.vector.riotx.R +// import im.vector.riotx.core.extensions.vectorComponent +// import im.vector.riotx.features.call.VectorCallActivity +// import im.vector.riotx.features.notifications.NotificationUtils +// import im.vector.riotx.features.themes.ThemeUtils // -//class CallHeadsUpService : Service() { -//// -//// private val CHANNEL_ID = "CallChannel" -//// private val CHANNEL_NAME = "Call Channel" -//// private val CHANNEL_DESCRIPTION = "Call Notifications" +// class CallHeadsUpService : Service() { +// // +// // private val CHANNEL_ID = "CallChannel" +// // private val CHANNEL_NAME = "Call Channel" +// // private val CHANNEL_DESCRIPTION = "Call Notifications" // // lateinit var notificationUtils: NotificationUtils // private val binder: IBinder = CallHeadsUpServiceBinder() @@ -59,14 +59,14 @@ // override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { // val callHeadsUpServiceArgs: CallHeadsUpServiceArgs? = intent?.extras?.getParcelable(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS) // -//// createNotificationChannel() +// // createNotificationChannel() // -//// val title = callHeadsUpServiceArgs?.otherUserId ?: "" -//// val description = when { -//// callHeadsUpServiceArgs?.isIncomingCall == false -> getString(R.string.call_ring) -//// callHeadsUpServiceArgs?.isVideoCall == true -> getString(R.string.incoming_video_call) -//// else -> getString(R.string.incoming_voice_call) -//// } +// // val title = callHeadsUpServiceArgs?.otherUserId ?: "" +// // val description = when { +// // callHeadsUpServiceArgs?.isIncomingCall == false -> getString(R.string.call_ring) +// // callHeadsUpServiceArgs?.isVideoCall == true -> getString(R.string.incoming_video_call) +// // else -> getString(R.string.incoming_voice_call) +// // } // // // val actions = if (callHeadsUpServiceArgs?.isIncomingCall == true) createAnswerAndRejectActions() else emptyList() // @@ -78,82 +78,82 @@ // ).let { // startForeground(NOTIFICATION_ID, it) // } -//// createNotification(title, description, actions).also { -//// startForeground(NOTIFICATION_ID, it) -//// } +// // createNotification(title, description, actions).also { +// // startForeground(NOTIFICATION_ID, it) +// // } // // return START_STICKY // } // -//// private fun createNotification(title: String, content: String, actions: List): Notification { -//// val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { -//// putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) -//// }.let { -//// PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, it, PendingIntent.FLAG_UPDATE_CURRENT) -//// } -//// return NotificationCompat -//// .Builder(applicationContext, CHANNEL_ID) -//// .setContentTitle(title) -//// .setContentText(content) -//// .setSmallIcon(R.drawable.ic_call) -//// .setPriority(NotificationCompat.PRIORITY_MAX) -//// .setWhen(0) -//// .setCategory(NotificationCompat.CATEGORY_CALL) -//// .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) -//// .setDefaults(NotificationCompat.DEFAULT_SOUND or NotificationCompat.DEFAULT_VIBRATE) -//// .setSound(Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg")) -//// .setVibrate(longArrayOf(1000, 1000)) -//// .setFullScreenIntent(answerCallActionReceiver, true) -//// .setOngoing(true) -//// //.setStyle(NotificationCompat.BigTextStyle()) -//// .setAutoCancel(true) -//// .apply { actions.forEach { addAction(it) } } -//// .build() -//// } +// // private fun createNotification(title: String, content: String, actions: List): Notification { +// // val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { +// // putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) +// // }.let { +// // PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, it, PendingIntent.FLAG_UPDATE_CURRENT) +// // } +// // return NotificationCompat +// // .Builder(applicationContext, CHANNEL_ID) +// // .setContentTitle(title) +// // .setContentText(content) +// // .setSmallIcon(R.drawable.ic_call) +// // .setPriority(NotificationCompat.PRIORITY_MAX) +// // .setWhen(0) +// // .setCategory(NotificationCompat.CATEGORY_CALL) +// // .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) +// // .setDefaults(NotificationCompat.DEFAULT_SOUND or NotificationCompat.DEFAULT_VIBRATE) +// // .setSound(Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg")) +// // .setVibrate(longArrayOf(1000, 1000)) +// // .setFullScreenIntent(answerCallActionReceiver, true) +// // .setOngoing(true) +// // //.setStyle(NotificationCompat.BigTextStyle()) +// // .setAutoCancel(true) +// // .apply { actions.forEach { addAction(it) } } +// // .build() +// // } // -//// private fun createAnswerAndRejectActions(): List { -//// val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { -//// putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) -//// } -//// val rejectCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { -//// putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_REJECT) -//// } -//// val answerCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) -//// val rejectCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_REJECT, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) -//// -//// return listOf( -//// NotificationCompat.Action( -//// R.drawable.ic_call, -//// //IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), -//// getString(R.string.call_notification_answer), -//// answerCallPendingIntent -//// ), -//// NotificationCompat.Action( -//// IconCompat.createWithResource(applicationContext, R.drawable.ic_call_end).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_notice)), -//// getString(R.string.call_notification_reject), -//// rejectCallPendingIntent) -//// ) -//// } +// // private fun createAnswerAndRejectActions(): List { +// // val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { +// // putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) +// // } +// // val rejectCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { +// // putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_REJECT) +// // } +// // val answerCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) +// // val rejectCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_REJECT, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) +// // +// // return listOf( +// // NotificationCompat.Action( +// // R.drawable.ic_call, +// // //IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), +// // getString(R.string.call_notification_answer), +// // answerCallPendingIntent +// // ), +// // NotificationCompat.Action( +// // IconCompat.createWithResource(applicationContext, R.drawable.ic_call_end).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_notice)), +// // getString(R.string.call_notification_reject), +// // rejectCallPendingIntent) +// // ) +// // } // -//// private fun createNotificationChannel() { -//// if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return -//// -//// val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH).apply { -//// description = CHANNEL_DESCRIPTION -//// setSound( -//// Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg"), -//// AudioAttributes -//// .Builder() -//// .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) -//// .setUsage(AudioAttributes.USAGE_NOTIFICATION_RINGTONE) -//// .build() -//// ) -//// lockscreenVisibility = NotificationCompat.VISIBILITY_PUBLIC -//// enableVibration(true) -//// enableLights(true) -//// } -//// applicationContext.getSystemService(NotificationManager::class.java)?.createNotificationChannel(channel) -//// } +// // private fun createNotificationChannel() { +// // if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return +// // +// // val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH).apply { +// // description = CHANNEL_DESCRIPTION +// // setSound( +// // Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg"), +// // AudioAttributes +// // .Builder() +// // .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) +// // .setUsage(AudioAttributes.USAGE_NOTIFICATION_RINGTONE) +// // .build() +// // ) +// // lockscreenVisibility = NotificationCompat.VISIBILITY_PUBLIC +// // enableVibration(true) +// // enableLights(true) +// // } +// // applicationContext.getSystemService(NotificationManager::class.java)?.createNotificationChannel(channel) +// // } // // inner class CallHeadsUpServiceBinder : Binder() { // @@ -165,7 +165,7 @@ // private const val EXTRA_CALL_HEADS_UP_SERVICE_PARAMS = "EXTRA_CALL_PARAMS" // // const val EXTRA_CALL_ACTION_KEY = "EXTRA_CALL_ACTION_KEY" -//// const val CALL_ACTION_ANSWER = 100 +// // const val CALL_ACTION_ANSWER = 100 // const val CALL_ACTION_REJECT = 101 // // private const val NOTIFICATION_ID = 999 @@ -177,4 +177,4 @@ // } // } // } -//} +// } diff --git a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt index 458925178e..dce8c72101 100755 --- a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt +++ b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt @@ -320,13 +320,13 @@ class NotificationUtils @Inject constructor(private val context: Context, val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { // putExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, CallHeadsUpService.CALL_ACTION_REJECT) } - //val answerCallPendingIntent = PendingIntent.getBroadcast(context, requestId, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) + // val answerCallPendingIntent = PendingIntent.getBroadcast(context, requestId, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) val rejectCallPendingIntent = PendingIntent.getBroadcast(context, requestId + 1, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) builder.addAction( NotificationCompat.Action( R.drawable.ic_call, - //IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), + // IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), context.getString(R.string.call_notification_answer), answerCallPendingIntent ) @@ -376,7 +376,7 @@ class NotificationUtils @Inject constructor(private val context: Context, val contentPendingIntent = TaskStackBuilder.create(context) .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) - //TODO other userId + // TODO other userId .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, "otherUserId", true, isVideo, false, null)) .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) From 91f28bfb8a97fb54152067bee0e42cca0bb8c4f6 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 11 Jun 2020 17:45:32 +0200 Subject: [PATCH 37/83] basic toggle mute and toggle video --- .../riotx/features/call/CallControlsView.kt | 28 +++++++++-- .../riotx/features/call/VectorCallActivity.kt | 10 +++- .../features/call/VectorCallViewModel.kt | 45 +++++++++--------- .../call/WebRtcPeerConnectionManager.kt | 46 +++++++++++-------- vector/src/main/res/drawable/ic_video_off.xml | 20 ++++++++ 5 files changed, 105 insertions(+), 44 deletions(-) create mode 100644 vector/src/main/res/drawable/ic_video_off.xml diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index ee7a9a4797..5e53398320 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -19,6 +19,7 @@ package im.vector.riotx.features.call import android.content.Context import android.util.AttributeSet import android.view.ViewGroup +import android.widget.ImageView import android.widget.LinearLayout import androidx.constraintlayout.widget.ConstraintLayout import androidx.core.view.isVisible @@ -44,6 +45,13 @@ class CallControlsView @JvmOverloads constructor( @BindView(R.id.connectedControls) lateinit var connectedControls: ViewGroup + + @BindView(R.id.iv_mute_toggle) + lateinit var muteIcon: ImageView + + @BindView(R.id.iv_video_toggle) + lateinit var videoToggleIcon: ImageView + init { ConstraintLayout.inflate(context, R.layout.fragment_call_controls, this) // layoutParams = FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT) @@ -65,11 +73,21 @@ class CallControlsView @JvmOverloads constructor( interactionListener?.didEndCall() } - @OnClick(R.id.iv_end_call) - fun hangupCall() { + @OnClick(R.id.iv_mute_toggle) + fun toggleMute() { + interactionListener?.didTapToggleMute() } - fun updateForState(callState: CallState?) { + @OnClick(R.id.iv_video_toggle) + fun toggleVideo() { + interactionListener?.didTapToggleVideo() + } + + fun updateForState(state: VectorCallViewState) { + val callState = state.callState.invoke() + muteIcon.setImageResource(if (state.isAudioMuted) R.drawable.ic_microphone_off else R.drawable.ic_microphone_on) + videoToggleIcon.setImageResource(if (state.isVideoEnabled) R.drawable.ic_video else R.drawable.ic_video_off) + when (callState) { CallState.DIALING -> { } @@ -94,11 +112,15 @@ class CallControlsView @JvmOverloads constructor( connectedControls.isVisible = false } } + + } interface InteractionListener { fun didAcceptIncomingCall() fun didDeclineIncomingCall() fun didEndCall() + fun didTapToggleMute() + fun didTapToggleVideo() } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 4ba9af0c0e..05528536f3 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -182,7 +182,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private fun renderState(state: VectorCallViewState) { Timber.v("## VOIP renderState call $state") - callControlsView.updateForState(state.callState.invoke()) + callControlsView.updateForState(state) when (state.callState.invoke()) { CallState.IDLE -> { } @@ -339,4 +339,12 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis override fun didEndCall() { callViewModel.handle(VectorCallViewActions.EndCall) } + + override fun didTapToggleMute() { + callViewModel.handle(VectorCallViewActions.ToggleMute) + } + + override fun didTapToggleVideo() { + callViewModel.handle(VectorCallViewActions.ToggleVideo) + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 77366b70d0..372f5c2a8c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -41,6 +41,8 @@ data class VectorCallViewState( val callId: String? = null, val roomId: String = "", val isVideoCall: Boolean, + val isAudioMuted: Boolean = false, + val isVideoEnabled: Boolean = true, val otherUserMatrixItem: Async = Uninitialized, val callState: Async = Uninitialized ) : MvRxState @@ -49,6 +51,8 @@ sealed class VectorCallViewActions : VectorViewModelAction { object EndCall : VectorCallViewActions() object AcceptCall : VectorCallViewActions() object DeclineCall : VectorCallViewActions() + object ToggleMute : VectorCallViewActions() + object ToggleVideo : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { @@ -65,26 +69,6 @@ class VectorCallViewModel @AssistedInject constructor( val webRtcPeerConnectionManager: WebRtcPeerConnectionManager ) : VectorViewModel(initialState) { - // private val callServiceListener: CallsListener = object : CallsListener { -// override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { -// withState { state -> -// if (callAnswerContent.callId == state.callId) { -// _viewEvents.post(VectorCallViewEvents.CallAnswered(callAnswerContent)) -// } -// } -// } -// -// override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { -// } -// -// override fun onCallHangupReceived(callHangupContent: CallHangupContent) { -// withState { state -> -// if (callHangupContent.callId == state.callId) { -// _viewEvents.post(VectorCallViewEvents.CallHangup(callHangupContent)) -// } -// } -// } -// } var autoReplyIfNeeded: Boolean = false var call: MxCall? = null @@ -120,7 +104,6 @@ class VectorCallViewModel @AssistedInject constructor( } } - // session.callService().addCallListener(callServiceListener) } override fun onCleared() { @@ -144,6 +127,26 @@ class VectorCallViewModel @AssistedInject constructor( } webRtcPeerConnectionManager.endCall() } + VectorCallViewActions.ToggleMute -> { + withState { + val muted = it.isAudioMuted + webRtcPeerConnectionManager.muteCall(!muted) + setState { + copy(isAudioMuted = !muted) + } + } + } + VectorCallViewActions.ToggleVideo -> { + withState { + if(it.isVideoCall) { + val videoEnabled = it.isVideoEnabled + webRtcPeerConnectionManager.enableVideo(!videoEnabled) + setState { + copy(isVideoEnabled = !videoEnabled) + } + } + } + } }.exhaustive } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 9d2b8cb2e0..c9a15a30a4 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -196,10 +196,10 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun sendSdpOffer(callContext: CallContext) { -// executor.execute { val constraints = MediaConstraints() - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) + // These are deprecated options +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) Timber.v("## VOIP creating offer...") callContext.peerConnection?.createOffer(object : SdpObserverAdapter() { @@ -211,7 +211,7 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall?.mxCall?.offerSdp(p0) } }, constraints) -// } + } private fun getTurnServer(callback: ((TurnServer?) -> Unit)) { @@ -355,7 +355,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // localViewRenderer?.let { localVideoTrack?.addSink(it) } localMediaStream?.addTrack(localVideoTrack) - callContext.localMediaStream = localMediaStream +// callContext.localMediaStream = localMediaStream // remoteVideoTrack?.setEnabled(true) // remoteVideoTrack?.let { // it.setEnabled(true) @@ -418,20 +418,20 @@ class WebRtcPeerConnectionManager @Inject constructor( private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { // add all existing audio filters to avoid having echos - mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) - mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) - mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) - - mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) - - mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) - mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) - - mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) - mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) - - mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) - mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) +// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) } } @@ -509,6 +509,14 @@ class WebRtcPeerConnectionManager @Inject constructor( } } + fun muteCall(muted: Boolean) { + currentCall?.localAudioTrack?.setEnabled(!muted) + } + + fun enableVideo(enabled: Boolean) { + currentCall?.localVideoTrack?.setEnabled(enabled) + } + fun endCall() { currentCall?.mxCall?.hangUp() currentCall = null diff --git a/vector/src/main/res/drawable/ic_video_off.xml b/vector/src/main/res/drawable/ic_video_off.xml new file mode 100644 index 0000000000..34abdb5b51 --- /dev/null +++ b/vector/src/main/res/drawable/ic_video_off.xml @@ -0,0 +1,20 @@ + + + + From 56ed56a986adcb0eeeb3f4b1c2750147cf5a87a9 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 12 Jun 2020 10:01:20 +0200 Subject: [PATCH 38/83] let remote view resize with aspect ratio --- vector/src/main/res/layout/activity_call.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index 8e7d2ab0a7..6448a1336d 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -11,8 +11,9 @@ + android:layout_height="wrap_content" + app:layout_constraintBottom_toBottomOf="parent" + app:layout_constraintTop_toTopOf="parent" /> Date: Fri, 12 Jun 2020 13:15:06 +0200 Subject: [PATCH 39/83] Cleaning call states --- .../android/api/session/call/CallState.kt | 11 +- .../internal/session/call/model/MxCallImpl.kt | 6 +- .../riotx/features/call/CallControlsView.kt | 43 ++++--- .../riotx/features/call/VectorCallActivity.kt | 80 ++++++------- .../features/call/VectorCallViewModel.kt | 6 +- .../call/WebRtcPeerConnectionManager.kt | 107 +++++++++++++++--- .../res/layout/fragment_call_controls.xml | 2 +- 7 files changed, 164 insertions(+), 91 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt index 713735e336..25bbe39f03 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt @@ -21,17 +21,20 @@ enum class CallState { /** Idle, setting up objects */ IDLE, + /** Dialing. Outgoing call is signaling the remote peer */ DIALING, + + /** Local ringing. Incoming call offer received */ + LOCAL_RINGING, + /** Answering. Incoming call is responding to remote peer */ ANSWERING, - /** Remote ringing. Outgoing call, ICE negotiation is complete */ - REMOTE_RINGING, + /** Connecting. Incoming/Outgoing Offer and answer are known, Currently checking and testing pairs of ice candidates */ + CONNECTING, - /** Local ringing. Incoming call, ICE negotiation is complete */ - LOCAL_RINGING, /** Connected. Incoming/Outgoing call, the call is connected */ CONNECTED, diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt index 6c9c7d7184..53c075579a 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt @@ -74,15 +74,16 @@ internal class MxCallImpl( init { if (isOutgoing) { - state = CallState.DIALING + state = CallState.IDLE } else { + // because it's created on reception of an offer state = CallState.LOCAL_RINGING } } override fun offerSdp(sdp: SessionDescription) { if (!isOutgoing) return - state = CallState.REMOTE_RINGING + state = CallState.DIALING CallInviteContent( callId = callId, lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, @@ -108,6 +109,7 @@ internal class MxCallImpl( } override fun sendLocalIceCandidateRemovals(candidates: List) { + // For now we don't support this flow } override fun hangUp() { diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index 5e53398320..4c7919acc2 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -35,17 +35,16 @@ class CallControlsView @JvmOverloads constructor( var interactionListener: InteractionListener? = null - @BindView(R.id.incomingRingingControls) - lateinit var incomingRingingControls: ViewGroup -// @BindView(R.id.iv_icr_accept_call) -// lateinit var incomingRingingControlAccept: ImageView -// @BindView(R.id.iv_icr_end_call) -// lateinit var incomingRingingControlDecline: ImageView + @BindView(R.id.ringingControls) + lateinit var ringingControls: ViewGroup + @BindView(R.id.iv_icr_accept_call) + lateinit var ringingControlAccept: ImageView + @BindView(R.id.iv_icr_end_call) + lateinit var ringingControlDecline: ImageView @BindView(R.id.connectedControls) lateinit var connectedControls: ViewGroup - @BindView(R.id.iv_mute_toggle) lateinit var muteIcon: ImageView @@ -89,31 +88,31 @@ class CallControlsView @JvmOverloads constructor( videoToggleIcon.setImageResource(if (state.isVideoEnabled) R.drawable.ic_video else R.drawable.ic_video_off) when (callState) { - CallState.DIALING -> { - } - CallState.ANSWERING -> { - incomingRingingControls.isVisible = false + CallState.IDLE, + CallState.DIALING, + CallState.CONNECTING, + CallState.ANSWERING -> { + ringingControls.isVisible = true + ringingControlAccept.isVisible = false + ringingControlDecline.isVisible = true connectedControls.isVisible = false } - CallState.REMOTE_RINGING -> { - } - CallState.LOCAL_RINGING -> { - incomingRingingControls.isVisible = true + CallState.LOCAL_RINGING -> { + ringingControls.isVisible = true + ringingControlAccept.isVisible = true + ringingControlDecline.isVisible = true connectedControls.isVisible = false } - CallState.CONNECTED -> { - incomingRingingControls.isVisible = false + CallState.CONNECTED -> { + ringingControls.isVisible = false connectedControls.isVisible = true } CallState.TERMINATED, - CallState.IDLE, - null -> { - incomingRingingControls.isVisible = false + null -> { + ringingControls.isVisible = false connectedControls.isVisible = false } } - - } interface InteractionListener { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 05528536f3..5108883fed 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -91,17 +91,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private var rootEglBase: EglBase? = null -// var callHeadsUpService: CallHeadsUpService? = null -// private val serviceConnection = object : ServiceConnection { -// override fun onServiceDisconnected(name: ComponentName?) { -// finish() -// } -// -// override fun onServiceConnected(name: ComponentName?, service: IBinder?) { -// callHeadsUpService = (service as? CallHeadsUpService.CallHeadsUpServiceBinder)?.getService() -// } -// } - override fun doBeforeSetContentView() { // Set window styles for fullscreen-window size. Needs to be done before adding content. requestWindowFeature(Window.FEATURE_NO_TITLE) @@ -130,9 +119,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis // window.addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED); // window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); -// tryThis { unbindService(serviceConnection) } -// bindService(Intent(this, CallHeadsUpService::class.java), serviceConnection, 0) - if (intent.hasExtra(MvRx.KEY_ARG)) { callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! } else { @@ -184,8 +170,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis Timber.v("## VOIP renderState call $state") callControlsView.updateForState(state) when (state.callState.invoke()) { - CallState.IDLE -> { - } + CallState.IDLE, CallState.DIALING -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true @@ -196,35 +181,40 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) } } - CallState.ANSWERING -> { - callInfoGroup.isVisible = true - callStatusText.setText(R.string.call_connecting) - state.otherUserMatrixItem.invoke()?.let { - avatarRenderer.render(it, otherMemberAvatar) - } -// fullscreenRenderer.isVisible = true -// pipRenderer.isVisible = true - } - CallState.REMOTE_RINGING -> { - callVideoGroup.isInvisible = true - callInfoGroup.isVisible = true - callStatusText.setText( - if (state.isVideoCall) R.string.incoming_video_call else R.string.incoming_voice_call - ) - } + CallState.LOCAL_RINGING -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true + callStatusText.text = null state.otherUserMatrixItem.invoke()?.let { avatarRenderer.render(it, otherMemberAvatar) participantNameText.text = it.getBestName() callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) } } + + CallState.ANSWERING -> { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + callStatusText.setText(R.string.call_connecting) + state.otherUserMatrixItem.invoke()?.let { + avatarRenderer.render(it, otherMemberAvatar) + } + } + CallState.CONNECTING -> { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + callStatusText.setText(R.string.call_connecting) + } CallState.CONNECTED -> { - // TODO only if is video call - callVideoGroup.isVisible = true - callInfoGroup.isVisible = false + if (callArgs.isVideoCall) { + callVideoGroup.isVisible = true + callInfoGroup.isVisible = false + } else { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + callStatusText.text = null + } } CallState.TERMINATED -> { finish() @@ -279,20 +269,20 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis return false } - override fun onDestroy() { + override fun onPause() { peerConnectionManager.detachRenderers() -// tryThis { unbindService(serviceConnection) } - super.onDestroy() + super.onPause() } private fun handleViewEvents(event: VectorCallViewEvents?) { - when (event) { - is VectorCallViewEvents.CallAnswered -> { - } - is VectorCallViewEvents.CallHangup -> { - finish() - } - } + Timber.v("handleViewEvents $event") +// when (event) { +// is VectorCallViewEvents.CallAnswered -> { +// } +// is VectorCallViewEvents.CallHangup -> { +// finish() +// } +// } } companion object { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 372f5c2a8c..f400b6864c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -57,9 +57,9 @@ sealed class VectorCallViewActions : VectorViewModelAction { sealed class VectorCallViewEvents : VectorViewEvents { - data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() - data class CallHangup(val content: CallHangupContent) : VectorCallViewEvents() - object CallAccepted : VectorCallViewEvents() +// data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() +// data class CallHangup(val content: CallHangupContent) : VectorCallViewEvents() +// object CallAccepted : VectorCallViewEvents() } class VectorCallViewModel @AssistedInject constructor( diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index c9a15a30a4..bb7ca8f8f4 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -211,7 +211,6 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall?.mxCall?.offerSdp(p0) } }, constraints) - } private fun getTurnServer(callback: ((TurnServer?) -> Unit)) { @@ -374,7 +373,6 @@ class WebRtcPeerConnectionManager @Inject constructor( // If remote track exists, then sink it to surface remoteSurfaceRenderer?.get()?.let { participantSurface -> currentCall?.remoteVideoTrack?.let { - it.setEnabled(true) it.addSink(participantSurface) } } @@ -551,17 +549,47 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { Timber.v("## VOIP StreamObserver onConnectionChange: $newState") when (newState) { - PeerConnection.PeerConnectionState.CONNECTED -> { + /** + * Every ICE transport used by the connection is either in use (state "connected" or "completed") + * or is closed (state "closed"); in addition, at least one transport is either "connected" or "completed" + */ + PeerConnection.PeerConnectionState.CONNECTED -> { callContext.mxCall.state = CallState.CONNECTED } - PeerConnection.PeerConnectionState.FAILED -> { + /** + * One or more of the ICE transports on the connection is in the "failed" state. + */ + PeerConnection.PeerConnectionState.FAILED -> { endCall() } + /** + * At least one of the connection's ICE transports (RTCIceTransports or RTCDtlsTransports) are in the "new" state, + * and none of them are in one of the following states: "connecting", "checking", "failed", or "disconnected", + * or all of the connection's transports are in the "closed" state. + */ PeerConnection.PeerConnectionState.NEW, - PeerConnection.PeerConnectionState.CONNECTING, - PeerConnection.PeerConnectionState.DISCONNECTED, + + /** + * One or more of the ICE transports are currently in the process of establishing a connection; + * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state + */ + PeerConnection.PeerConnectionState.CONNECTING -> { + callContext.mxCall.state = CallState.CONNECTING + } + /** + * The RTCPeerConnection is closed. + * This value was in the RTCSignalingState enum (and therefore found by reading the value of the signalingState) + * property until the May 13, 2016 draft of the specification. + */ PeerConnection.PeerConnectionState.CLOSED, - null -> { + /** + * At least one of the ICE transports for the connection is in the "disconnected" state and none of the other transports are in the state "failed", + * "connecting", or "checking". + */ + PeerConnection.PeerConnectionState.DISCONNECTED -> { + + } + null -> { } } } @@ -580,14 +608,60 @@ class WebRtcPeerConnectionManager @Inject constructor( } override fun onIceConnectionChange(newState: PeerConnection.IceConnectionState) { + Timber.v("## VOIP StreamObserver onIceConnectionChange IceConnectionState:$newState") when (newState) { - PeerConnection.IceConnectionState.CONNECTED -> Timber.v("## VOIP StreamObserver onIceConnectionChange.CONNECTED") - PeerConnection.IceConnectionState.DISCONNECTED -> { - Timber.v("## VOIP StreamObserver onIceConnectionChange.DISCONNECTED") - endCall() + + /** + * the ICE agent is gathering addresses or is waiting to be given remote candidates through + * calls to RTCPeerConnection.addIceCandidate() (or both). + */ + PeerConnection.IceConnectionState.NEW -> { + + } + /** + * The ICE agent has been given one or more remote candidates and is checking pairs of local and remote candidates + * against one another to try to find a compatible match, but has not yet found a pair which will allow + * the peer connection to be made. It's possible that gathering of candidates is also still underway. + */ + PeerConnection.IceConnectionState.CHECKING -> { + + } + + /** + * A usable pairing of local and remote candidates has been found for all components of the connection, + * and the connection has been established. + * It's possible that gathering is still underway, and it's also possible that the ICE agent is still checking + * candidates against one another looking for a better connection to use. + */ + PeerConnection.IceConnectionState.CONNECTED -> { + + } + /** + * Checks to ensure that components are still connected failed for at least one component of the RTCPeerConnection. + * This is a less stringent test than "failed" and may trigger intermittently and resolve just as spontaneously on less reliable networks, + * or during temporary disconnections. When the problem resolves, the connection may return to the "connected" state. + */ + PeerConnection.IceConnectionState.DISCONNECTED -> { + } + /** + * The ICE candidate has checked all candidates pairs against one another and has failed to find compatible matches for all components of the connection. + * It is, however, possible that the ICE agent did find compatible connections for some components. + */ + PeerConnection.IceConnectionState.FAILED -> { + callContext.mxCall.hangUp() + } + /** + * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. + */ + PeerConnection.IceConnectionState.COMPLETED -> { + + } + /** + * The ICE agent for this RTCPeerConnection has shut down and is no longer handling requests. + */ + PeerConnection.IceConnectionState.CLOSED -> { + } - PeerConnection.IceConnectionState.FAILED -> Timber.v("## VOIP StreamObserver onIceConnectionChange.FAILED") - else -> Timber.v("## VOIP StreamObserver onIceConnectionChange.$newState") } } @@ -595,7 +669,12 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP StreamObserver onAddStream: $stream") executor.execute { // reportError("Weird-looking stream: " + stream); - if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) return@execute + if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) { + Timber.e("## VOIP StreamObserver weird looking stream: $stream") + //TODO maybe do something more?? + callContext.mxCall.hangUp() + return@execute + } if (stream.videoTracks.size == 1) { val remoteVideoTrack = stream.videoTracks.first() diff --git a/vector/src/main/res/layout/fragment_call_controls.xml b/vector/src/main/res/layout/fragment_call_controls.xml index 75b119f55c..a1c8c313a7 100644 --- a/vector/src/main/res/layout/fragment_call_controls.xml +++ b/vector/src/main/res/layout/fragment_call_controls.xml @@ -8,7 +8,7 @@ Date: Fri, 12 Jun 2020 17:38:17 +0200 Subject: [PATCH 40/83] Basic return to call Ux in Room detail --- .../vector/riotx/core/di/ViewModelModule.kt | 6 +++ .../riotx/core/ui/views/ActiveCallView.kt | 46 ++++++++++++++++ .../riotx/features/call/CallControlsView.kt | 10 ++++ .../call/SharedActiveCallViewModel.kt | 51 ++++++++++++++++++ .../riotx/features/call/VectorCallActivity.kt | 45 +++++++++------- .../call/WebRtcPeerConnectionManager.kt | 28 ++++++++-- .../home/room/detail/RoomDetailFragment.kt | 54 ++++++++++++++++++- .../res/layout/fragment_call_controls.xml | 14 ++--- .../main/res/layout/fragment_room_detail.xml | 8 ++- .../main/res/layout/view_active_call_view.xml | 43 +++++++++++++++ vector/src/main/res/values/strings.xml | 2 + 11 files changed, 274 insertions(+), 33 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallView.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt create mode 100644 vector/src/main/res/layout/view_active_call_view.xml diff --git a/vector/src/main/java/im/vector/riotx/core/di/ViewModelModule.kt b/vector/src/main/java/im/vector/riotx/core/di/ViewModelModule.kt index a214073104..badfdd96c1 100644 --- a/vector/src/main/java/im/vector/riotx/core/di/ViewModelModule.kt +++ b/vector/src/main/java/im/vector/riotx/core/di/ViewModelModule.kt @@ -22,6 +22,7 @@ import dagger.Binds import dagger.Module import dagger.multibindings.IntoMap import im.vector.riotx.core.platform.ConfigurationViewModel +import im.vector.riotx.features.call.SharedActiveCallViewModel import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreFromKeyViewModel import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreFromPassphraseViewModel import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreSharedViewModel @@ -85,6 +86,11 @@ interface ViewModelModule { @ViewModelKey(ConfigurationViewModel::class) fun bindConfigurationViewModel(viewModel: ConfigurationViewModel): ViewModel + @Binds + @IntoMap + @ViewModelKey(SharedActiveCallViewModel::class) + fun bindSharedActiveCallViewModel(viewModel: SharedActiveCallViewModel): ViewModel + @Binds @IntoMap @ViewModelKey(UserDirectorySharedActionViewModel::class) diff --git a/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallView.kt b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallView.kt new file mode 100644 index 0000000000..9507a4daf8 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallView.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.core.ui.views + +import android.content.Context +import android.util.AttributeSet +import android.widget.RelativeLayout +import im.vector.riotx.R +import im.vector.riotx.features.themes.ThemeUtils + +class ActiveCallView @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null, + defStyleAttr: Int = 0 +) : RelativeLayout(context, attrs, defStyleAttr) { + + interface Callback { + fun onTapToReturnToCall() + } + + var callback: Callback? = null + + init { + setupView() + } + + private fun setupView() { + inflate(context, R.layout.view_active_call_view, this) + setBackgroundColor(ThemeUtils.getColor(context, R.attr.colorPrimary)) + setOnClickListener { callback?.onTapToReturnToCall() } + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index 4c7919acc2..6dbf339373 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -22,12 +22,14 @@ import android.view.ViewGroup import android.widget.ImageView import android.widget.LinearLayout import androidx.constraintlayout.widget.ConstraintLayout +import androidx.core.view.isInvisible import androidx.core.view.isVisible import butterknife.BindView import butterknife.ButterKnife import butterknife.OnClick import im.vector.matrix.android.api.session.call.CallState import im.vector.riotx.R +import kotlinx.android.synthetic.main.fragment_call_controls.view.* class CallControlsView @JvmOverloads constructor( context: Context, attrs: AttributeSet? = null, defStyleAttr: Int = 0 @@ -82,6 +84,12 @@ class CallControlsView @JvmOverloads constructor( interactionListener?.didTapToggleVideo() } + @OnClick(R.id.iv_leftMiniControl) + fun returnToChat() { + interactionListener?.returnToChat() + } + + fun updateForState(state: VectorCallViewState) { val callState = state.callState.invoke() muteIcon.setImageResource(if (state.isAudioMuted) R.drawable.ic_microphone_off else R.drawable.ic_microphone_on) @@ -106,6 +114,7 @@ class CallControlsView @JvmOverloads constructor( CallState.CONNECTED -> { ringingControls.isVisible = false connectedControls.isVisible = true + iv_video_toggle.isInvisible = !state.isVideoCall } CallState.TERMINATED, null -> { @@ -121,5 +130,6 @@ class CallControlsView @JvmOverloads constructor( fun didEndCall() fun didTapToggleMute() fun didTapToggleVideo() + fun returnToChat() } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt new file mode 100644 index 0000000000..efd8541e1c --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import androidx.lifecycle.MutableLiveData +import androidx.lifecycle.ViewModel +import im.vector.matrix.android.api.session.call.MxCall +import im.vector.riotx.core.platform.VectorSharedAction +import javax.inject.Inject + +sealed class CallActions : VectorSharedAction { + data class GoToCallActivity(val mxCall: MxCall) : CallActions() + data class ToggleVisibility(val visible: Boolean) : CallActions() +} + +class SharedActiveCallViewModel @Inject constructor( + private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager +) : ViewModel() { + + val activeCall: MutableLiveData = MutableLiveData() + + private val listener = object : WebRtcPeerConnectionManager.CurrentCallListener { + override fun onCurrentCallChange(call: MxCall?) { + activeCall.postValue(call) + } + } + + init { + activeCall.postValue(webRtcPeerConnectionManager.currentCall?.mxCall) + webRtcPeerConnectionManager.addCurrentCallListener(listener) + } + + override fun onCleared() { + webRtcPeerConnectionManager.removeCurrentCallListener(listener) + super.onCleared() + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 5108883fed..d201588bf9 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -171,59 +171,59 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callControlsView.updateForState(state) when (state.callState.invoke()) { CallState.IDLE, - CallState.DIALING -> { + CallState.DIALING -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.setText(R.string.call_ring) - state.otherUserMatrixItem.invoke()?.let { - avatarRenderer.render(it, otherMemberAvatar) - participantNameText.text = it.getBestName() - callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) - } + configureCallInfo(state) } - CallState.LOCAL_RINGING -> { + CallState.LOCAL_RINGING -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.text = null - state.otherUserMatrixItem.invoke()?.let { - avatarRenderer.render(it, otherMemberAvatar) - participantNameText.text = it.getBestName() - callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) - } + configureCallInfo(state) } - CallState.ANSWERING -> { + CallState.ANSWERING -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.setText(R.string.call_connecting) - state.otherUserMatrixItem.invoke()?.let { - avatarRenderer.render(it, otherMemberAvatar) - } + configureCallInfo(state) } - CallState.CONNECTING -> { + CallState.CONNECTING -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true + configureCallInfo(state) callStatusText.setText(R.string.call_connecting) } - CallState.CONNECTED -> { + CallState.CONNECTED -> { if (callArgs.isVideoCall) { callVideoGroup.isVisible = true callInfoGroup.isVisible = false } else { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true + configureCallInfo(state) callStatusText.text = null } } - CallState.TERMINATED -> { + CallState.TERMINATED -> { finish() } - null -> { + null -> { } } } + private fun configureCallInfo(state: VectorCallViewState) { + state.otherUserMatrixItem.invoke()?.let { + avatarRenderer.render(it, otherMemberAvatar) + participantNameText.text = it.getBestName() + callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) + } + } + private fun configureCallViews() { callControlsView.interactionListener = this // if (callArgs.isVideoCall) { @@ -337,4 +337,9 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis override fun didTapToggleVideo() { callViewModel.handle(VectorCallViewActions.ToggleVideo) } + + override fun returnToChat() { + // TODO, what if the room is not in backstack?? + finish() + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index bb7ca8f8f4..1a60fc97e4 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -18,6 +18,7 @@ package im.vector.riotx.features.call import android.content.Context import im.vector.matrix.android.api.MatrixCallback +import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils @@ -68,6 +69,19 @@ class WebRtcPeerConnectionManager @Inject constructor( private val sessionHolder: ActiveSessionHolder ) : CallsListener { + interface CurrentCallListener { + fun onCurrentCallChange(call: MxCall?) + } + + private val currentCallsListeners = emptyList().toMutableList() + fun addCurrentCallListener(listener: CurrentCallListener) { + currentCallsListeners.add(listener) + } + + fun removeCurrentCallListener(listener: CurrentCallListener) { + currentCallsListeners.remove(listener) + } + data class CallContext( val mxCall: MxCall, @@ -137,6 +151,12 @@ class WebRtcPeerConnectionManager @Inject constructor( var remoteSurfaceRenderer: WeakReference? = null var currentCall: CallContext? = null + set(value) { + field = value + currentCallsListeners.forEach { + tryThis { it.onCurrentCallChange(value?.mxCall) } + } + } init { // TODO do this lazyly @@ -569,10 +589,10 @@ class WebRtcPeerConnectionManager @Inject constructor( */ PeerConnection.PeerConnectionState.NEW, - /** - * One or more of the ICE transports are currently in the process of establishing a connection; - * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state - */ + /** + * One or more of the ICE transports are currently in the process of establishing a connection; + * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state + */ PeerConnection.PeerConnectionState.CONNECTING -> { callContext.mxCall.state = CallState.CONNECTING } diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index d0ec5a7894..c1e5e032b0 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -42,6 +42,7 @@ import androidx.core.util.Pair import androidx.core.view.ViewCompat import androidx.core.view.forEach import androidx.core.view.isVisible +import androidx.lifecycle.Observer import androidx.recyclerview.widget.ItemTouchHelper import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView @@ -100,6 +101,7 @@ import im.vector.riotx.core.glide.GlideApp import im.vector.riotx.core.intent.getMimeTypeFromUri import im.vector.riotx.core.platform.VectorBaseFragment import im.vector.riotx.core.resources.ColorProvider +import im.vector.riotx.core.ui.views.ActiveCallView import im.vector.riotx.core.ui.views.JumpToReadMarkerView import im.vector.riotx.core.ui.views.NotificationAreaView import im.vector.riotx.core.utils.Debouncer @@ -127,6 +129,8 @@ import im.vector.riotx.features.attachments.ContactAttachment import im.vector.riotx.features.attachments.preview.AttachmentsPreviewActivity import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs import im.vector.riotx.features.attachments.toGroupedContentAttachmentData +import im.vector.riotx.features.call.SharedActiveCallViewModel +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.command.Command import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity @@ -205,7 +209,8 @@ class RoomDetailFragment @Inject constructor( JumpToReadMarkerView.Callback, AttachmentTypeSelectorView.Callback, AttachmentsHelper.Callback, - RoomWidgetsBannerView.Callback { + RoomWidgetsBannerView.Callback, + ActiveCallView.Callback { companion object { @@ -245,6 +250,8 @@ class RoomDetailFragment @Inject constructor( override fun getMenuRes() = R.menu.menu_timeline private lateinit var sharedActionViewModel: MessageSharedActionViewModel + private lateinit var sharedCallActionViewModel: SharedActiveCallViewModel + private lateinit var layoutManager: LinearLayoutManager private lateinit var jumpToBottomViewVisibilityManager: JumpToBottomViewVisibilityManager private var modelBuildListener: OnModelBuildFinishedListener? = null @@ -261,6 +268,7 @@ class RoomDetailFragment @Inject constructor( override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) sharedActionViewModel = activityViewModelProvider.get(MessageSharedActionViewModel::class.java) + sharedCallActionViewModel = activityViewModelProvider.get(SharedActiveCallViewModel::class.java) attachmentsHelper = AttachmentsHelper(requireContext(), this).register() keyboardStateUtils = KeyboardStateUtils(requireActivity()) setupToolbar(roomToolbar) @@ -269,6 +277,7 @@ class RoomDetailFragment @Inject constructor( setupInviteView() setupNotificationView() setupJumpToReadMarkerView() + setupActiveCallView() setupJumpToBottomView() setupWidgetsBannerView() @@ -283,6 +292,13 @@ class RoomDetailFragment @Inject constructor( } .disposeOnDestroyView() + sharedCallActionViewModel + .activeCall + .observe(viewLifecycleOwner, Observer { + //TODO delay a bit if it's a new call to let call activity launch before .. + activeCallView.isVisible = it != null + }) + roomDetailViewModel.selectSubscribe(this, RoomDetailViewState::tombstoneEventHandling, uniqueOnly("tombstoneEventHandling")) { renderTombstoneEventHandling(it) } @@ -374,6 +390,7 @@ class RoomDetailFragment @Inject constructor( override fun onDestroyView() { timelineEventController.callback = null timelineEventController.removeModelBuildListener(modelBuildListener) + activeCallView.callback = null modelBuildListener = null autoCompleter.clear() debouncer.cancelAll() @@ -412,6 +429,10 @@ class RoomDetailFragment @Inject constructor( jumpToReadMarkerView.callback = this } + private fun setupActiveCallView() { + activeCallView.callback = this + } + private fun navigateToEvent(action: RoomDetailViewEvents.NavigateToEvent) { val scrollPosition = timelineEventController.searchPositionOfEvent(action.eventId) if (scrollPosition == null) { @@ -483,7 +504,19 @@ class RoomDetailFragment @Inject constructor( R.id.video_call -> { roomDetailViewModel.getOtherUserIds()?.firstOrNull()?.let { // TODO CALL We should check/ask for permission here first - webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) + val activeCall = sharedCallActionViewModel.activeCall.value + if (activeCall != null) { + // resume existing if same room, if not prompt to kill and then restart new call? + if (activeCall.roomId == roomDetailArgs.roomId) { + onTapToReturnToCall() + } else { + // TODO might not work well, and should prompt + webRtcPeerConnectionManager.endCall() + webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) + } + } else { + webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) + } } true } @@ -1479,4 +1512,21 @@ class RoomDetailFragment @Inject constructor( RoomWidgetsBottomSheet.newInstance() .show(childFragmentManager, "ROOM_WIDGETS_BOTTOM_SHEET") } + + override fun onTapToReturnToCall() { + sharedCallActionViewModel.activeCall.value?.let { call -> + VectorCallActivity.newIntent( + requireContext(), + call.callId, + call.roomId, + call.otherUserId, + !call.isOutgoing, + call.isVideoCall, + false, + null + ).let { + startActivity(it) + } + } + } } diff --git a/vector/src/main/res/layout/fragment_call_controls.xml b/vector/src/main/res/layout/fragment_call_controls.xml index a1c8c313a7..6f1861afec 100644 --- a/vector/src/main/res/layout/fragment_call_controls.xml +++ b/vector/src/main/res/layout/fragment_call_controls.xml @@ -11,8 +11,8 @@ android:id="@+id/ringingControls" android:layout_width="match_parent" android:layout_height="wrap_content" - android:padding="16dp" android:orientation="horizontal" + android:padding="16dp" tools:background="@color/password_strength_bar_ok" tools:visibility="visible"> @@ -55,8 +55,8 @@ android:id="@+id/connectedControls" android:layout_width="match_parent" android:layout_height="wrap_content" - android:padding="16dp" android:orientation="horizontal" + android:padding="16dp" android:visibility="gone" tools:background="@color/password_strength_bar_low" tools:visibility="visible"> @@ -66,11 +66,13 @@ android:layout_width="44dp" android:layout_height="44dp" android:layout_marginBottom="32dp" + android:background="@drawable/oval_positive" + android:backgroundTint="?attr/riotx_background" android:clickable="true" android:focusable="true" - android:padding="8dp" + android:padding="10dp" android:src="@drawable/ic_home_bottom_chat" - android:tint="?attr/riotx_background" + android:tint="?attr/riotx_text_primary" tools:ignore="MissingConstraints" /> @@ -85,9 +87,9 @@ android:focusable="true" android:padding="16dp" android:src="@drawable/ic_microphone_off" - tools:src="@drawable/ic_microphone_on" android:tint="?attr/riotx_text_primary" - tools:ignore="MissingConstraints" /> + tools:ignore="MissingConstraints" + tools:src="@drawable/ic_microphone_on" /> + + diff --git a/vector/src/main/res/layout/view_active_call_view.xml b/vector/src/main/res/layout/view_active_call_view.xml new file mode 100644 index 0000000000..a457c5164e --- /dev/null +++ b/vector/src/main/res/layout/view_active_call_view.xml @@ -0,0 +1,43 @@ + + + + + + + + diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index 931a7896eb..91a3388bd2 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -362,6 +362,8 @@ Incoming Voice Call Call In Progress… Video Call In Progress… + Active Call (%s) + Return to call The remote side failed to pick up. Media Connection Failed From 84b474d070c7980a437e9aa1ec4a99eccd057eff Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 12 Jun 2020 17:44:45 +0200 Subject: [PATCH 41/83] klint --- .../vector/matrix/android/api/session/call/CallState.kt | 3 --- .../im/vector/riotx/features/call/CallControlsView.kt | 1 - .../im/vector/riotx/features/call/VectorCallViewModel.kt | 5 +---- .../riotx/features/call/WebRtcPeerConnectionManager.kt | 9 +-------- .../features/home/room/detail/RoomDetailFragment.kt | 2 +- 5 files changed, 3 insertions(+), 17 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt index 25bbe39f03..1ab53854aa 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt @@ -21,11 +21,9 @@ enum class CallState { /** Idle, setting up objects */ IDLE, - /** Dialing. Outgoing call is signaling the remote peer */ DIALING, - /** Local ringing. Incoming call offer received */ LOCAL_RINGING, @@ -35,7 +33,6 @@ enum class CallState { /** Connecting. Incoming/Outgoing Offer and answer are known, Currently checking and testing pairs of ice candidates */ CONNECTING, - /** Connected. Incoming/Outgoing call, the call is connected */ CONNECTED, diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index 6dbf339373..f8ed0f0cc1 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -89,7 +89,6 @@ class CallControlsView @JvmOverloads constructor( interactionListener?.returnToChat() } - fun updateForState(state: VectorCallViewState) { val callState = state.callState.invoke() muteIcon.setImageResource(if (state.isAudioMuted) R.drawable.ic_microphone_off else R.drawable.ic_microphone_on) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index f400b6864c..ec09b8bc2c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -28,8 +28,6 @@ import com.squareup.inject.assisted.AssistedInject import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.MxCall -import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent -import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.util.MatrixItem import im.vector.matrix.android.api.util.toMatrixItem import im.vector.riotx.core.extensions.exhaustive @@ -103,7 +101,6 @@ class VectorCallViewModel @AssistedInject constructor( } } } - } override fun onCleared() { @@ -138,7 +135,7 @@ class VectorCallViewModel @AssistedInject constructor( } VectorCallViewActions.ToggleVideo -> { withState { - if(it.isVideoCall) { + if (it.isVideoCall) { val videoEnabled = it.isVideoEnabled webRtcPeerConnectionManager.enableVideo(!videoEnabled) setState { diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 1a60fc97e4..5911aedc15 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -607,7 +607,6 @@ class WebRtcPeerConnectionManager @Inject constructor( * "connecting", or "checking". */ PeerConnection.PeerConnectionState.DISCONNECTED -> { - } null -> { } @@ -630,13 +629,11 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onIceConnectionChange(newState: PeerConnection.IceConnectionState) { Timber.v("## VOIP StreamObserver onIceConnectionChange IceConnectionState:$newState") when (newState) { - /** * the ICE agent is gathering addresses or is waiting to be given remote candidates through * calls to RTCPeerConnection.addIceCandidate() (or both). */ PeerConnection.IceConnectionState.NEW -> { - } /** * The ICE agent has been given one or more remote candidates and is checking pairs of local and remote candidates @@ -644,7 +641,6 @@ class WebRtcPeerConnectionManager @Inject constructor( * the peer connection to be made. It's possible that gathering of candidates is also still underway. */ PeerConnection.IceConnectionState.CHECKING -> { - } /** @@ -654,7 +650,6 @@ class WebRtcPeerConnectionManager @Inject constructor( * candidates against one another looking for a better connection to use. */ PeerConnection.IceConnectionState.CONNECTED -> { - } /** * Checks to ensure that components are still connected failed for at least one component of the RTCPeerConnection. @@ -674,13 +669,11 @@ class WebRtcPeerConnectionManager @Inject constructor( * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. */ PeerConnection.IceConnectionState.COMPLETED -> { - } /** * The ICE agent for this RTCPeerConnection has shut down and is no longer handling requests. */ PeerConnection.IceConnectionState.CLOSED -> { - } } } @@ -691,7 +684,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // reportError("Weird-looking stream: " + stream); if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) { Timber.e("## VOIP StreamObserver weird looking stream: $stream") - //TODO maybe do something more?? + // TODO maybe do something more?? callContext.mxCall.hangUp() return@execute } diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index c1e5e032b0..2bb3fc5d1d 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -295,7 +295,7 @@ class RoomDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - //TODO delay a bit if it's a new call to let call activity launch before .. + // TODO delay a bit if it's a new call to let call activity launch before .. activeCallView.isVisible = it != null }) From c6100fc26c69854d35eb5746ce0a64f3b180fe6a Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 12 Jun 2020 17:50:17 +0200 Subject: [PATCH 42/83] Code cleaning --- .../features/call/PeerConnectionClient.java | 1374 ----------------- .../call/WebRtcPeerConnectionManager.kt | 15 +- .../call/service/CallHeadsUpService.kt | 180 --- .../notifications/NotificationUtils.kt | 3 +- 4 files changed, 11 insertions(+), 1561 deletions(-) delete mode 100644 vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java delete mode 100644 vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt diff --git a/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java deleted file mode 100644 index ee2594fd1a..0000000000 --- a/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java +++ /dev/null @@ -1,1374 +0,0 @@ -///* -// * Copyright (c) 2020 New Vector Ltd -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package im.vector.riotx.features.call; -// -//import android.content.Context; -//import android.os.Environment; -//import android.os.ParcelFileDescriptor; -//import android.util.Log; -// -//import org.appspot.apprtc.AppRTCClient.SignalingParameters; -//import org.webrtc.AudioSource; -//import org.webrtc.AudioTrack; -//import org.webrtc.CameraVideoCapturer; -//import org.webrtc.DataChannel; -//import org.webrtc.EglBase; -//import org.webrtc.IceCandidate; -//import org.webrtc.Logging; -//import org.webrtc.MediaConstraints; -//import org.webrtc.MediaStream; -//import org.webrtc.PeerConnection; -//import org.webrtc.PeerConnection.IceConnectionState; -//import org.webrtc.PeerConnectionFactory; -//import org.webrtc.RtpParameters; -//import org.webrtc.RtpReceiver; -//import org.webrtc.RtpSender; -//import org.webrtc.SdpObserver; -//import org.webrtc.SessionDescription; -//import org.webrtc.StatsObserver; -//import org.webrtc.StatsReport; -//import org.webrtc.VideoCapturer; -//import org.webrtc.VideoRenderer; -//import org.webrtc.VideoSink; -//import org.webrtc.VideoSource; -//import org.webrtc.VideoTrack; -//import org.webrtc.voiceengine.WebRtcAudioManager; -//import org.webrtc.voiceengine.WebRtcAudioRecord; -//import org.webrtc.voiceengine.WebRtcAudioRecord.AudioRecordStartErrorCode; -//import org.webrtc.voiceengine.WebRtcAudioRecord.WebRtcAudioRecordErrorCallback; -//import org.webrtc.voiceengine.WebRtcAudioTrack; -//import org.webrtc.voiceengine.WebRtcAudioTrack.WebRtcAudioTrackErrorCallback; -//import org.webrtc.voiceengine.WebRtcAudioUtils; -// -//import java.io.File; -//import java.io.IOException; -//import java.nio.ByteBuffer; -//import java.util.ArrayList; -//import java.util.Arrays; -//import java.util.Collections; -//import java.util.EnumSet; -//import java.util.Iterator; -//import java.util.LinkedList; -//import java.util.List; -//import java.util.Timer; -//import java.util.TimerTask; -//import java.util.concurrent.ExecutorService; -//import java.util.concurrent.Executors; -//import java.util.regex.Matcher; -//import java.util.regex.Pattern; -// -///** -// * Peer connection client implementation. -// * -// *

All public methods are routed to local looper thread. -// * All PeerConnectionEvents callbacks are invoked from the same looper thread. -// * This class is a singleton. -// */ -//public class PeerConnectionClient { -// public static final String VIDEO_TRACK_ID = "ARDAMSv0"; -// public static final String AUDIO_TRACK_ID = "ARDAMSa0"; -// public static final String VIDEO_TRACK_TYPE = "video"; -// private static final String TAG = "PCRTCClient"; -// private static final String VIDEO_CODEC_VP8 = "VP8"; -// private static final String VIDEO_CODEC_VP9 = "VP9"; -// private static final String VIDEO_CODEC_H264 = "H264"; -// private static final String VIDEO_CODEC_H264_BASELINE = "H264 Baseline"; -// private static final String VIDEO_CODEC_H264_HIGH = "H264 High"; -// private static final String AUDIO_CODEC_OPUS = "opus"; -// private static final String AUDIO_CODEC_ISAC = "ISAC"; -// private static final String VIDEO_CODEC_PARAM_START_BITRATE = "x-google-start-bitrate"; -// private static final String VIDEO_FLEXFEC_FIELDTRIAL = -// "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/"; -// private static final String VIDEO_VP8_INTEL_HW_ENCODER_FIELDTRIAL = "WebRTC-IntelVP8/Enabled/"; -// private static final String VIDEO_H264_HIGH_PROFILE_FIELDTRIAL = -// "WebRTC-H264HighProfile/Enabled/"; -// private static final String DISABLE_WEBRTC_AGC_FIELDTRIAL = -// "WebRTC-Audio-MinimizeResamplingOnMobile/Enabled/"; -// private static final String AUDIO_CODEC_PARAM_BITRATE = "maxaveragebitrate"; -// private static final String AUDIO_ECHO_CANCELLATION_CONSTRAINT = "googEchoCancellation"; -// private static final String AUDIO_AUTO_GAIN_CONTROL_CONSTRAINT = "googAutoGainControl"; -// private static final String AUDIO_HIGH_PASS_FILTER_CONSTRAINT = "googHighpassFilter"; -// private static final String AUDIO_NOISE_SUPPRESSION_CONSTRAINT = "googNoiseSuppression"; -// private static final String AUDIO_LEVEL_CONTROL_CONSTRAINT = "levelControl"; -// private static final String DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT = "DtlsSrtpKeyAgreement"; -// private static final int HD_VIDEO_WIDTH = 1280; -// private static final int HD_VIDEO_HEIGHT = 720; -// private static final int BPS_IN_KBPS = 1000; -// -// private static final PeerConnectionClient instance = new PeerConnectionClient(); -// private final PCObserver pcObserver = new PCObserver(); -// private final SDPObserver sdpObserver = new SDPObserver(); -// private final ExecutorService executor; -// -// private PeerConnectionFactory factory; -// private PeerConnection peerConnection; -// PeerConnectionFactory.Options options = null; -// private AudioSource audioSource; -// private VideoSource videoSource; -// private boolean videoCallEnabled; -// private boolean preferIsac; -// private String preferredVideoCodec; -// private boolean videoCapturerStopped; -// private boolean isError; -// private Timer statsTimer; -// private VideoSink.Callbacks localRender; -// private List remoteRenders; -// private SignalingParameters signalingParameters; -// private MediaConstraints pcConstraints; -// private int videoWidth; -// private int videoHeight; -// private int videoFps; -// private MediaConstraints audioConstraints; -// private ParcelFileDescriptor aecDumpFileDescriptor; -// private MediaConstraints sdpMediaConstraints; -// private PeerConnectionParameters peerConnectionParameters; -// // Queued remote ICE candidates are consumed only after both local and -// // remote descriptions are set. Similarly local ICE candidates are sent to -// // remote peer after both local and remote description are set. -// private LinkedList queuedRemoteCandidates; -// private PeerConnectionEvents events; -// private boolean isInitiator; -// private SessionDescription localSdp; // either offer or answer SDP -// private MediaStream mediaStream; -// private VideoCapturer videoCapturer; -// // enableVideo is set to true if video should be rendered and sent. -// private boolean renderVideo; -// private VideoTrack localVideoTrack; -// private VideoTrack remoteVideoTrack; -// private RtpSender localVideoSender; -// // enableAudio is set to true if audio should be sent. -// private boolean enableAudio; -// private AudioTrack localAudioTrack; -// private DataChannel dataChannel; -// private boolean dataChannelEnabled; -// -// /** -// * Peer connection parameters. -// */ -// public static class DataChannelParameters { -// public final boolean ordered; -// public final int maxRetransmitTimeMs; -// public final int maxRetransmits; -// public final String protocol; -// public final boolean negotiated; -// public final int id; -// -// public DataChannelParameters(boolean ordered, int maxRetransmitTimeMs, int maxRetransmits, -// String protocol, boolean negotiated, int id) { -// this.ordered = ordered; -// this.maxRetransmitTimeMs = maxRetransmitTimeMs; -// this.maxRetransmits = maxRetransmits; -// this.protocol = protocol; -// this.negotiated = negotiated; -// this.id = id; -// } -// } -// -// /** -// * Peer connection parameters. -// */ -// public static class PeerConnectionParameters { -// public final boolean videoCallEnabled; -// public final boolean loopback; -// public final boolean tracing; -// public final int videoWidth; -// public final int videoHeight; -// public final int videoFps; -// public final int videoMaxBitrate; -// public final String videoCodec; -// public final boolean videoCodecHwAcceleration; -// public final boolean videoFlexfecEnabled; -// public final int audioStartBitrate; -// public final String audioCodec; -// public final boolean noAudioProcessing; -// public final boolean aecDump; -// public final boolean useOpenSLES; -// public final boolean disableBuiltInAEC; -// public final boolean disableBuiltInAGC; -// public final boolean disableBuiltInNS; -// public final boolean enableLevelControl; -// public final boolean disableWebRtcAGCAndHPF; -// private final DataChannelParameters dataChannelParameters; -// -// public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing, -// int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec, -// boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate, -// String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES, -// boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS, -// boolean enableLevelControl, boolean disableWebRtcAGCAndHPF) { -// this(videoCallEnabled, loopback, tracing, videoWidth, videoHeight, videoFps, videoMaxBitrate, -// videoCodec, videoCodecHwAcceleration, videoFlexfecEnabled, audioStartBitrate, audioCodec, -// noAudioProcessing, aecDump, useOpenSLES, disableBuiltInAEC, disableBuiltInAGC, -// disableBuiltInNS, enableLevelControl, disableWebRtcAGCAndHPF, null); -// } -// -// public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing, -// int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec, -// boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate, -// String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES, -// boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS, -// boolean enableLevelControl, boolean disableWebRtcAGCAndHPF, -// DataChannelParameters dataChannelParameters) { -// this.videoCallEnabled = videoCallEnabled; -// this.loopback = loopback; -// this.tracing = tracing; -// this.videoWidth = videoWidth; -// this.videoHeight = videoHeight; -// this.videoFps = videoFps; -// this.videoMaxBitrate = videoMaxBitrate; -// this.videoCodec = videoCodec; -// this.videoFlexfecEnabled = videoFlexfecEnabled; -// this.videoCodecHwAcceleration = videoCodecHwAcceleration; -// this.audioStartBitrate = audioStartBitrate; -// this.audioCodec = audioCodec; -// this.noAudioProcessing = noAudioProcessing; -// this.aecDump = aecDump; -// this.useOpenSLES = useOpenSLES; -// this.disableBuiltInAEC = disableBuiltInAEC; -// this.disableBuiltInAGC = disableBuiltInAGC; -// this.disableBuiltInNS = disableBuiltInNS; -// this.enableLevelControl = enableLevelControl; -// this.disableWebRtcAGCAndHPF = disableWebRtcAGCAndHPF; -// this.dataChannelParameters = dataChannelParameters; -// } -// } -// -// /** -// * Peer connection events. -// */ -// public interface PeerConnectionEvents { -// /** -// * Callback fired once local SDP is created and set. -// */ -// void onLocalDescription(final SessionDescription sdp); -// -// /** -// * Callback fired once local Ice candidate is generated. -// */ -// void onIceCandidate(final IceCandidate candidate); -// -// /** -// * Callback fired once local ICE candidates are removed. -// */ -// void onIceCandidatesRemoved(final IceCandidate[] candidates); -// -// /** -// * Callback fired once connection is established (IceConnectionState is -// * CONNECTED). -// */ -// void onIceConnected(); -// -// /** -// * Callback fired once connection is closed (IceConnectionState is -// * DISCONNECTED). -// */ -// void onIceDisconnected(); -// -// /** -// * Callback fired once peer connection is closed. -// */ -// void onPeerConnectionClosed(); -// -// /** -// * Callback fired once peer connection statistics is ready. -// */ -// void onPeerConnectionStatsReady(final StatsReport[] reports); -// -// /** -// * Callback fired once peer connection error happened. -// */ -// void onPeerConnectionError(final String description); -// } -// -// private PeerConnectionClient() { -// // Executor thread is started once in private ctor and is used for all -// // peer connection API calls to ensure new peer connection factory is -// // created on the same thread as previously destroyed factory. -// executor = Executors.newSingleThreadExecutor(); -// } -// -// public static PeerConnectionClient getInstance() { -// return instance; -// } -// -// public void setPeerConnectionFactoryOptions(PeerConnectionFactory.Options options) { -// this.options = options; -// } -// -// public void createPeerConnectionFactory(final Context context, -// final PeerConnectionParameters peerConnectionParameters, final PeerConnectionEvents events) { -// this.peerConnectionParameters = peerConnectionParameters; -// this.events = events; -// videoCallEnabled = peerConnectionParameters.videoCallEnabled; -// dataChannelEnabled = peerConnectionParameters.dataChannelParameters != null; -// // Reset variables to initial states. -// factory = null; -// peerConnection = null; -// preferIsac = false; -// videoCapturerStopped = false; -// isError = false; -// queuedRemoteCandidates = null; -// localSdp = null; // either offer or answer SDP -// mediaStream = null; -// videoCapturer = null; -// renderVideo = true; -// localVideoTrack = null; -// remoteVideoTrack = null; -// localVideoSender = null; -// enableAudio = true; -// localAudioTrack = null; -// statsTimer = new Timer(); -// -// executor.execute(new Runnable() { -// @Override -// public void run() { -// createPeerConnectionFactoryInternal(context); -// } -// }); -// } -// -// public void createPeerConnection(final EglBase.Context renderEGLContext, -// final VideoRenderer.Callbacks localRender, final VideoRenderer.Callbacks remoteRender, -// final VideoCapturer videoCapturer, final SignalingParameters signalingParameters) { -// createPeerConnection(renderEGLContext, localRender, Collections.singletonList(remoteRender), -// videoCapturer, signalingParameters); -// } -// -// public void createPeerConnection(final EglBase.Context renderEGLContext, -// final VideoRenderer.Callbacks localRender, final List remoteRenders, -// final VideoCapturer videoCapturer, final SignalingParameters signalingParameters) { -// if (peerConnectionParameters == null) { -// Log.e(TAG, "Creating peer connection without initializing factory."); -// return; -// } -// this.localRender = localRender; -// this.remoteRenders = remoteRenders; -// this.videoCapturer = videoCapturer; -// this.signalingParameters = signalingParameters; -// executor.execute(new Runnable() { -// @Override -// public void run() { -// try { -// createMediaConstraintsInternal(); -// createPeerConnectionInternal(renderEGLContext); -// } catch (Exception e) { -// reportError("Failed to create peer connection: " + e.getMessage()); -// throw e; -// } -// } -// }); -// } -// -// public void close() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// closeInternal(); -// } -// }); -// } -// -// public boolean isVideoCallEnabled() { -// return videoCallEnabled; -// } -// -// private void createPeerConnectionFactoryInternal(Context context) { -// PeerConnectionFactory.initializeInternalTracer(); -// if (peerConnectionParameters.tracing) { -// PeerConnectionFactory.startInternalTracingCapture( -// Environment.getExternalStorageDirectory().getAbsolutePath() + File.separator -// + "webrtc-trace.txt"); -// } -// Log.d(TAG, -// "Create peer connection factory. Use video: " + peerConnectionParameters.videoCallEnabled); -// isError = false; -// -// // Initialize field trials. -// String fieldTrials = ""; -// if (peerConnectionParameters.videoFlexfecEnabled) { -// fieldTrials += VIDEO_FLEXFEC_FIELDTRIAL; -// Log.d(TAG, "Enable FlexFEC field trial."); -// } -// fieldTrials += VIDEO_VP8_INTEL_HW_ENCODER_FIELDTRIAL; -// if (peerConnectionParameters.disableWebRtcAGCAndHPF) { -// fieldTrials += DISABLE_WEBRTC_AGC_FIELDTRIAL; -// Log.d(TAG, "Disable WebRTC AGC field trial."); -// } -// -// // Check preferred video codec. -// preferredVideoCodec = VIDEO_CODEC_VP8; -// if (videoCallEnabled && peerConnectionParameters.videoCodec != null) { -// switch (peerConnectionParameters.videoCodec) { -// case VIDEO_CODEC_VP8: -// preferredVideoCodec = VIDEO_CODEC_VP8; -// break; -// case VIDEO_CODEC_VP9: -// preferredVideoCodec = VIDEO_CODEC_VP9; -// break; -// case VIDEO_CODEC_H264_BASELINE: -// preferredVideoCodec = VIDEO_CODEC_H264; -// break; -// case VIDEO_CODEC_H264_HIGH: -// // TODO(magjed): Strip High from SDP when selecting Baseline instead of using field trial. -// fieldTrials += VIDEO_H264_HIGH_PROFILE_FIELDTRIAL; -// preferredVideoCodec = VIDEO_CODEC_H264; -// break; -// default: -// preferredVideoCodec = VIDEO_CODEC_VP8; -// } -// } -// Log.d(TAG, "Preferred video codec: " + preferredVideoCodec); -// PeerConnectionFactory.initializeFieldTrials(fieldTrials); -// Log.d(TAG, "Field trials: " + fieldTrials); -// -// // Check if ISAC is used by default. -// preferIsac = peerConnectionParameters.audioCodec != null -// && peerConnectionParameters.audioCodec.equals(AUDIO_CODEC_ISAC); -// -// // Enable/disable OpenSL ES playback. -// if (!peerConnectionParameters.useOpenSLES) { -// Log.d(TAG, "Disable OpenSL ES audio even if device supports it"); -// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true /* enable */); -// } else { -// Log.d(TAG, "Allow OpenSL ES audio if device supports it"); -// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(false); -// } -// -// if (peerConnectionParameters.disableBuiltInAEC) { -// Log.d(TAG, "Disable built-in AEC even if device supports it"); -// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true); -// } else { -// Log.d(TAG, "Enable built-in AEC if device supports it"); -// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(false); -// } -// -// if (peerConnectionParameters.disableBuiltInAGC) { -// Log.d(TAG, "Disable built-in AGC even if device supports it"); -// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(true); -// } else { -// Log.d(TAG, "Enable built-in AGC if device supports it"); -// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(false); -// } -// -// if (peerConnectionParameters.disableBuiltInNS) { -// Log.d(TAG, "Disable built-in NS even if device supports it"); -// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true); -// } else { -// Log.d(TAG, "Enable built-in NS if device supports it"); -// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(false); -// } -// -// // Set audio record error callbacks. -// WebRtcAudioRecord.setErrorCallback(new WebRtcAudioRecordErrorCallback() { -// @Override -// public void onWebRtcAudioRecordInitError(String errorMessage) { -// Log.e(TAG, "onWebRtcAudioRecordInitError: " + errorMessage); -// reportError(errorMessage); -// } -// -// @Override -// public void onWebRtcAudioRecordStartError( -// AudioRecordStartErrorCode errorCode, String errorMessage) { -// Log.e(TAG, "onWebRtcAudioRecordStartError: " + errorCode + ". " + errorMessage); -// reportError(errorMessage); -// } -// -// @Override -// public void onWebRtcAudioRecordError(String errorMessage) { -// Log.e(TAG, "onWebRtcAudioRecordError: " + errorMessage); -// reportError(errorMessage); -// } -// }); -// -// WebRtcAudioTrack.setErrorCallback(new WebRtcAudioTrackErrorCallback() { -// @Override -// public void onWebRtcAudioTrackInitError(String errorMessage) { -// reportError(errorMessage); -// } -// -// @Override -// public void onWebRtcAudioTrackStartError(String errorMessage) { -// reportError(errorMessage); -// } -// -// @Override -// public void onWebRtcAudioTrackError(String errorMessage) { -// reportError(errorMessage); -// } -// }); -// -// // Create peer connection factory. -// PeerConnectionFactory.initializeAndroidGlobals( -// context, peerConnectionParameters.videoCodecHwAcceleration); -// if (options != null) { -// Log.d(TAG, "Factory networkIgnoreMask option: " + options.networkIgnoreMask); -// } -// factory = new PeerConnectionFactory(options); -// Log.d(TAG, "Peer connection factory created."); -// } -// -// private void createMediaConstraintsInternal() { -// // Create peer connection constraints. -// pcConstraints = new MediaConstraints(); -// // Enable DTLS for normal calls and disable for loopback calls. -// if (peerConnectionParameters.loopback) { -// pcConstraints.optional.add( -// new MediaConstraints.KeyValuePair(DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT, "false")); -// } else { -// pcConstraints.optional.add( -// new MediaConstraints.KeyValuePair(DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT, "true")); -// } -// -// // Check if there is a camera on device and disable video call if not. -// if (videoCapturer == null) { -// Log.w(TAG, "No camera on device. Switch to audio only call."); -// videoCallEnabled = false; -// } -// // Create video constraints if video call is enabled. -// if (videoCallEnabled) { -// videoWidth = peerConnectionParameters.videoWidth; -// videoHeight = peerConnectionParameters.videoHeight; -// videoFps = peerConnectionParameters.videoFps; -// -// // If video resolution is not specified, default to HD. -// if (videoWidth == 0 || videoHeight == 0) { -// videoWidth = HD_VIDEO_WIDTH; -// videoHeight = HD_VIDEO_HEIGHT; -// } -// -// // If fps is not specified, default to 30. -// if (videoFps == 0) { -// videoFps = 30; -// } -// Logging.d(TAG, "Capturing format: " + videoWidth + "x" + videoHeight + "@" + videoFps); -// } -// -// // Create audio constraints. -// audioConstraints = new MediaConstraints(); -// // added for audio performance measurements -// if (peerConnectionParameters.noAudioProcessing) { -// Log.d(TAG, "Disabling audio processing"); -// audioConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair(AUDIO_ECHO_CANCELLATION_CONSTRAINT, "false")); -// audioConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair(AUDIO_AUTO_GAIN_CONTROL_CONSTRAINT, "false")); -// audioConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair(AUDIO_HIGH_PASS_FILTER_CONSTRAINT, "false")); -// audioConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair(AUDIO_NOISE_SUPPRESSION_CONSTRAINT, "false")); -// } -// if (peerConnectionParameters.enableLevelControl) { -// Log.d(TAG, "Enabling level control."); -// audioConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair(AUDIO_LEVEL_CONTROL_CONSTRAINT, "true")); -// } -// // Create SDP constraints. -// sdpMediaConstraints = new MediaConstraints(); -// sdpMediaConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); -// if (videoCallEnabled || peerConnectionParameters.loopback) { -// sdpMediaConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); -// } else { -// sdpMediaConstraints.mandatory.add( -// new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "false")); -// } -// } -// -// private void createPeerConnectionInternal(EglBase.Context renderEGLContext) { -// if (factory == null || isError) { -// Log.e(TAG, "Peerconnection factory is not created"); -// return; -// } -// Log.d(TAG, "Create peer connection."); -// -// Log.d(TAG, "PCConstraints: " + pcConstraints.toString()); -// queuedRemoteCandidates = new LinkedList(); -// -// if (videoCallEnabled) { -// Log.d(TAG, "EGLContext: " + renderEGLContext); -// factory.setVideoHwAccelerationOptions(renderEGLContext, renderEGLContext); -// } -// -// PeerConnection.RTCConfiguration rtcConfig = -// new PeerConnection.RTCConfiguration(signalingParameters.iceServers); -// // TCP candidates are only useful when connecting to a server that supports -// // ICE-TCP. -// rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; -// rtcConfig.bundlePolicy = PeerConnection.BundlePolicy.MAXBUNDLE; -// rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.REQUIRE; -// rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; -// // Use ECDSA encryption. -// rtcConfig.keyType = PeerConnection.KeyType.ECDSA; -// -// peerConnection = factory.createPeerConnection(rtcConfig, pcConstraints, pcObserver); -// -// if (dataChannelEnabled) { -// DataChannel.Init init = new DataChannel.Init(); -// init.ordered = peerConnectionParameters.dataChannelParameters.ordered; -// init.negotiated = peerConnectionParameters.dataChannelParameters.negotiated; -// init.maxRetransmits = peerConnectionParameters.dataChannelParameters.maxRetransmits; -// init.maxRetransmitTimeMs = peerConnectionParameters.dataChannelParameters.maxRetransmitTimeMs; -// init.id = peerConnectionParameters.dataChannelParameters.id; -// init.protocol = peerConnectionParameters.dataChannelParameters.protocol; -// dataChannel = peerConnection.createDataChannel("ApprtcDemo data", init); -// } -// isInitiator = false; -// -// // Set default WebRTC tracing and INFO libjingle logging. -// // NOTE: this _must_ happen while |factory| is alive! -// Logging.enableTracing("logcat:", EnumSet.of(Logging.TraceLevel.TRACE_DEFAULT)); -// Logging.enableLogToDebugOutput(Logging.Severity.LS_INFO); -// -// mediaStream = factory.createLocalMediaStream("ARDAMS"); -// if (videoCallEnabled) { -// mediaStream.addTrack(createVideoTrack(videoCapturer)); -// } -// -// mediaStream.addTrack(createAudioTrack()); -// peerConnection.addStream(mediaStream); -// if (videoCallEnabled) { -// findVideoSender(); -// } -// -// if (peerConnectionParameters.aecDump) { -// try { -// aecDumpFileDescriptor = -// ParcelFileDescriptor.open(new File(Environment.getExternalStorageDirectory().getPath() -// + File.separator + "Download/audio.aecdump"), -// ParcelFileDescriptor.MODE_READ_WRITE | ParcelFileDescriptor.MODE_CREATE -// | ParcelFileDescriptor.MODE_TRUNCATE); -// factory.startAecDump(aecDumpFileDescriptor.getFd(), -1); -// } catch (IOException e) { -// Log.e(TAG, "Can not open aecdump file", e); -// } -// } -// -// Log.d(TAG, "Peer connection created."); -// } -// -// private void closeInternal() { -// if (factory != null && peerConnectionParameters.aecDump) { -// factory.stopAecDump(); -// } -// Log.d(TAG, "Closing peer connection."); -// statsTimer.cancel(); -// if (dataChannel != null) { -// dataChannel.dispose(); -// dataChannel = null; -// } -// if (peerConnection != null) { -// peerConnection.dispose(); -// peerConnection = null; -// } -// Log.d(TAG, "Closing audio source."); -// if (audioSource != null) { -// audioSource.dispose(); -// audioSource = null; -// } -// Log.d(TAG, "Stopping capture."); -// if (videoCapturer != null) { -// try { -// videoCapturer.stopCapture(); -// } catch (InterruptedException e) { -// throw new RuntimeException(e); -// } -// videoCapturerStopped = true; -// videoCapturer.dispose(); -// videoCapturer = null; -// } -// Log.d(TAG, "Closing video source."); -// if (videoSource != null) { -// videoSource.dispose(); -// videoSource = null; -// } -// localRender = null; -// remoteRenders = null; -// Log.d(TAG, "Closing peer connection factory."); -// if (factory != null) { -// factory.dispose(); -// factory = null; -// } -// options = null; -// Log.d(TAG, "Closing peer connection done."); -// events.onPeerConnectionClosed(); -// PeerConnectionFactory.stopInternalTracingCapture(); -// PeerConnectionFactory.shutdownInternalTracer(); -// events = null; -// } -// -// public boolean isHDVideo() { -// if (!videoCallEnabled) { -// return false; -// } -// -// return videoWidth * videoHeight >= 1280 * 720; -// } -// -// private void getStats() { -// if (peerConnection == null || isError) { -// return; -// } -// boolean success = peerConnection.getStats(new StatsObserver() { -// @Override -// public void onComplete(final StatsReport[] reports) { -// events.onPeerConnectionStatsReady(reports); -// } -// }, null); -// if (!success) { -// Log.e(TAG, "getStats() returns false!"); -// } -// } -// -// public void enableStatsEvents(boolean enable, int periodMs) { -// if (enable) { -// try { -// statsTimer.schedule(new TimerTask() { -// @Override -// public void run() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// getStats(); -// } -// }); -// } -// }, 0, periodMs); -// } catch (Exception e) { -// Log.e(TAG, "Can not schedule statistics timer", e); -// } -// } else { -// statsTimer.cancel(); -// } -// } -// -// public void setAudioEnabled(final boolean enable) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// enableAudio = enable; -// if (localAudioTrack != null) { -// localAudioTrack.setEnabled(enableAudio); -// } -// } -// }); -// } -// -// public void setVideoEnabled(final boolean enable) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// renderVideo = enable; -// if (localVideoTrack != null) { -// localVideoTrack.setEnabled(renderVideo); -// } -// if (remoteVideoTrack != null) { -// remoteVideoTrack.setEnabled(renderVideo); -// } -// } -// }); -// } -// -// public void createOffer() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection != null && !isError) { -// Log.d(TAG, "PC Create OFFER"); -// isInitiator = true; -// peerConnection.createOffer(sdpObserver, sdpMediaConstraints); -// } -// } -// }); -// } -// -// public void createAnswer() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection != null && !isError) { -// Log.d(TAG, "PC create ANSWER"); -// isInitiator = false; -// peerConnection.createAnswer(sdpObserver, sdpMediaConstraints); -// } -// } -// }); -// } -// -// public void addRemoteIceCandidate(final IceCandidate candidate) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection != null && !isError) { -// if (queuedRemoteCandidates != null) { -// queuedRemoteCandidates.add(candidate); -// } else { -// peerConnection.addIceCandidate(candidate); -// } -// } -// } -// }); -// } -// -// public void removeRemoteIceCandidates(final IceCandidate[] candidates) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection == null || isError) { -// return; -// } -// // Drain the queued remote candidates if there is any so that -// // they are processed in the proper order. -// drainCandidates(); -// peerConnection.removeIceCandidates(candidates); -// } -// }); -// } -// -// public void setRemoteDescription(final SessionDescription sdp) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection == null || isError) { -// return; -// } -// String sdpDescription = sdp.description; -// if (preferIsac) { -// sdpDescription = preferCodec(sdpDescription, AUDIO_CODEC_ISAC, true); -// } -// if (videoCallEnabled) { -// sdpDescription = preferCodec(sdpDescription, preferredVideoCodec, false); -// } -// if (peerConnectionParameters.audioStartBitrate > 0) { -// sdpDescription = setStartBitrate( -// AUDIO_CODEC_OPUS, false, sdpDescription, peerConnectionParameters.audioStartBitrate); -// } -// Log.d(TAG, "Set remote SDP."); -// SessionDescription sdpRemote = new SessionDescription(sdp.type, sdpDescription); -// peerConnection.setRemoteDescription(sdpObserver, sdpRemote); -// } -// }); -// } -// -// public void stopVideoSource() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (videoCapturer != null && !videoCapturerStopped) { -// Log.d(TAG, "Stop video source."); -// try { -// videoCapturer.stopCapture(); -// } catch (InterruptedException e) { -// } -// videoCapturerStopped = true; -// } -// } -// }); -// } -// -// public void startVideoSource() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (videoCapturer != null && videoCapturerStopped) { -// Log.d(TAG, "Restart video source."); -// videoCapturer.startCapture(videoWidth, videoHeight, videoFps); -// videoCapturerStopped = false; -// } -// } -// }); -// } -// -// public void setVideoMaxBitrate(final Integer maxBitrateKbps) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection == null || localVideoSender == null || isError) { -// return; -// } -// Log.d(TAG, "Requested max video bitrate: " + maxBitrateKbps); -// if (localVideoSender == null) { -// Log.w(TAG, "Sender is not ready."); -// return; -// } -// -// RtpParameters parameters = localVideoSender.getParameters(); -// if (parameters.encodings.size() == 0) { -// Log.w(TAG, "RtpParameters are not ready."); -// return; -// } -// -// for (RtpParameters.Encoding encoding : parameters.encodings) { -// // Null value means no limit. -// encoding.maxBitrateBps = maxBitrateKbps == null ? null : maxBitrateKbps * BPS_IN_KBPS; -// } -// if (!localVideoSender.setParameters(parameters)) { -// Log.e(TAG, "RtpSender.setParameters failed."); -// } -// Log.d(TAG, "Configured max video bitrate to: " + maxBitrateKbps); -// } -// }); -// } -// -// private void reportError(final String errorMessage) { -// Log.e(TAG, "Peerconnection error: " + errorMessage); -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (!isError) { -// events.onPeerConnectionError(errorMessage); -// isError = true; -// } -// } -// }); -// } -// -// private AudioTrack createAudioTrack() { -// audioSource = factory.createAudioSource(audioConstraints); -// localAudioTrack = factory.createAudioTrack(AUDIO_TRACK_ID, audioSource); -// localAudioTrack.setEnabled(enableAudio); -// return localAudioTrack; -// } -//æ -// private VideoTrack createVideoTrack(VideoCapturer capturer) { -// videoSource = factory.createVideoSource(capturer); -// capturer.startCapture(videoWidth, videoHeight, videoFps); -// -// localVideoTrack = factory.createVideoTrack(VIDEO_TRACK_ID, videoSource); -// localVideoTrack.setEnabled(renderVideo); -// localVideoTrack.addRenderer(new VideoRenderer(localRender)); -// return localVideoTrack; -// } -// -// private void findVideoSender() { -// for (RtpSender sender : peerConnection.getSenders()) { -// if (sender.track() != null) { -// String trackType = sender.track().kind(); -// if (trackType.equals(VIDEO_TRACK_TYPE)) { -// Log.d(TAG, "Found video sender."); -// localVideoSender = sender; -// } -// } -// } -// } -// -// private static String setStartBitrate( -// String codec, boolean isVideoCodec, String sdpDescription, int bitrateKbps) { -// String[] lines = sdpDescription.split("\r\n"); -// int rtpmapLineIndex = -1; -// boolean sdpFormatUpdated = false; -// String codecRtpMap = null; -// // Search for codec rtpmap in format -// // a=rtpmap: / [/] -// String regex = "^a=rtpmap:(\\d+) " + codec + "(/\\d+)+[\r]?$"; -// Pattern codecPattern = Pattern.compile(regex); -// for (int i = 0; i < lines.length; i++) { -// Matcher codecMatcher = codecPattern.matcher(lines[i]); -// if (codecMatcher.matches()) { -// codecRtpMap = codecMatcher.group(1); -// rtpmapLineIndex = i; -// break; -// } -// } -// if (codecRtpMap == null) { -// Log.w(TAG, "No rtpmap for " + codec + " codec"); -// return sdpDescription; -// } -// Log.d(TAG, "Found " + codec + " rtpmap " + codecRtpMap + " at " + lines[rtpmapLineIndex]); -// -// // Check if a=fmtp string already exist in remote SDP for this codec and -// // update it with new bitrate parameter. -// regex = "^a=fmtp:" + codecRtpMap + " \\w+=\\d+.*[\r]?$"; -// codecPattern = Pattern.compile(regex); -// for (int i = 0; i < lines.length; i++) { -// Matcher codecMatcher = codecPattern.matcher(lines[i]); -// if (codecMatcher.matches()) { -// Log.d(TAG, "Found " + codec + " " + lines[i]); -// if (isVideoCodec) { -// lines[i] += "; " + VIDEO_CODEC_PARAM_START_BITRATE + "=" + bitrateKbps; -// } else { -// lines[i] += "; " + AUDIO_CODEC_PARAM_BITRATE + "=" + (bitrateKbps * 1000); -// } -// Log.d(TAG, "Update remote SDP line: " + lines[i]); -// sdpFormatUpdated = true; -// break; -// } -// } -// -// StringBuilder newSdpDescription = new StringBuilder(); -// for (int i = 0; i < lines.length; i++) { -// newSdpDescription.append(lines[i]).append("\r\n"); -// // Append new a=fmtp line if no such line exist for a codec. -// if (!sdpFormatUpdated && i == rtpmapLineIndex) { -// String bitrateSet; -// if (isVideoCodec) { -// bitrateSet = -// "a=fmtp:" + codecRtpMap + " " + VIDEO_CODEC_PARAM_START_BITRATE + "=" + bitrateKbps; -// } else { -// bitrateSet = "a=fmtp:" + codecRtpMap + " " + AUDIO_CODEC_PARAM_BITRATE + "=" -// + (bitrateKbps * 1000); -// } -// Log.d(TAG, "Add remote SDP line: " + bitrateSet); -// newSdpDescription.append(bitrateSet).append("\r\n"); -// } -// } -// return newSdpDescription.toString(); -// } -// -// /** -// * Returns the line number containing "m=audio|video", or -1 if no such line exists. -// */ -// private static int findMediaDescriptionLine(boolean isAudio, String[] sdpLines) { -// final String mediaDescription = isAudio ? "m=audio " : "m=video "; -// for (int i = 0; i < sdpLines.length; ++i) { -// if (sdpLines[i].startsWith(mediaDescription)) { -// return i; -// } -// } -// return -1; -// } -// -// private static String joinString( -// Iterable s, String delimiter, boolean delimiterAtEnd) { -// Iterator iter = s.iterator(); -// if (!iter.hasNext()) { -// return ""; -// } -// StringBuilder buffer = new StringBuilder(iter.next()); -// while (iter.hasNext()) { -// buffer.append(delimiter).append(iter.next()); -// } -// if (delimiterAtEnd) { -// buffer.append(delimiter); -// } -// return buffer.toString(); -// } -// -// private static String movePayloadTypesToFront(List preferredPayloadTypes, String mLine) { -// // The format of the media description line should be: m= ... -// final List origLineParts = Arrays.asList(mLine.split(" ")); -// if (origLineParts.size() <= 3) { -// Log.e(TAG, "Wrong SDP media description format: " + mLine); -// return null; -// } -// final List header = origLineParts.subList(0, 3); -// final List unpreferredPayloadTypes = -// new ArrayList(origLineParts.subList(3, origLineParts.size())); -// unpreferredPayloadTypes.removeAll(preferredPayloadTypes); -// // Reconstruct the line with |preferredPayloadTypes| moved to the beginning of the payload -// // types. -// final List newLineParts = new ArrayList(); -// newLineParts.addAll(header); -// newLineParts.addAll(preferredPayloadTypes); -// newLineParts.addAll(unpreferredPayloadTypes); -// return joinString(newLineParts, " ", false /* delimiterAtEnd */); -// } -// -// private static String preferCodec(String sdpDescription, String codec, boolean isAudio) { -// final String[] lines = sdpDescription.split("\r\n"); -// final int mLineIndex = findMediaDescriptionLine(isAudio, lines); -// if (mLineIndex == -1) { -// Log.w(TAG, "No mediaDescription line, so can't prefer " + codec); -// return sdpDescription; -// } -// // A list with all the payload types with name |codec|. The payload types are integers in the -// // range 96-127, but they are stored as strings here. -// final List codecPayloadTypes = new ArrayList(); -// // a=rtpmap: / [/] -// final Pattern codecPattern = Pattern.compile("^a=rtpmap:(\\d+) " + codec + "(/\\d+)+[\r]?$"); -// for (int i = 0; i < lines.length; ++i) { -// Matcher codecMatcher = codecPattern.matcher(lines[i]); -// if (codecMatcher.matches()) { -// codecPayloadTypes.add(codecMatcher.group(1)); -// } -// } -// if (codecPayloadTypes.isEmpty()) { -// Log.w(TAG, "No payload types with name " + codec); -// return sdpDescription; -// } -// -// final String newMLine = movePayloadTypesToFront(codecPayloadTypes, lines[mLineIndex]); -// if (newMLine == null) { -// return sdpDescription; -// } -// Log.d(TAG, "Change media description from: " + lines[mLineIndex] + " to " + newMLine); -// lines[mLineIndex] = newMLine; -// return joinString(Arrays.asList(lines), "\r\n", true /* delimiterAtEnd */); -// } -// -// private void drainCandidates() { -// if (queuedRemoteCandidates != null) { -// Log.d(TAG, "Add " + queuedRemoteCandidates.size() + " remote candidates"); -// for (IceCandidate candidate : queuedRemoteCandidates) { -// peerConnection.addIceCandidate(candidate); -// } -// queuedRemoteCandidates = null; -// } -// } -// -// private void switchCameraInternal() { -// if (videoCapturer instanceof CameraVideoCapturer) { -// if (!videoCallEnabled || isError || videoCapturer == null) { -// Log.e(TAG, "Failed to switch camera. Video: " + videoCallEnabled + ". Error : " + isError); -// return; // No video is sent or only one camera is available or error happened. -// } -// Log.d(TAG, "Switch camera"); -// CameraVideoCapturer cameraVideoCapturer = (CameraVideoCapturer) videoCapturer; -// cameraVideoCapturer.switchCamera(null); -// } else { -// Log.d(TAG, "Will not switch camera, video caputurer is not a camera"); -// } -// } -// -// public void switchCamera() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// switchCameraInternal(); -// } -// }); -// } -// -// public void changeCaptureFormat(final int width, final int height, final int framerate) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// changeCaptureFormatInternal(width, height, framerate); -// } -// }); -// } -// -// private void changeCaptureFormatInternal(int width, int height, int framerate) { -// if (!videoCallEnabled || isError || videoCapturer == null) { -// Log.e(TAG, -// "Failed to change capture format. Video: " + videoCallEnabled + ". Error : " + isError); -// return; -// } -// Log.d(TAG, "changeCaptureFormat: " + width + "x" + height + "@" + framerate); -// videoSource.adaptOutputFormat(width, height, framerate); -// } -// -// // Implementation detail: observe ICE & stream changes and react accordingly. -// private class PCObserver implements PeerConnection.Observer { -// @Override -// public void onIceCandidate(final IceCandidate candidate) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// events.onIceCandidate(candidate); -// } -// }); -// } -// -// @Override -// public void onIceCandidatesRemoved(final IceCandidate[] candidates) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// events.onIceCandidatesRemoved(candidates); -// } -// }); -// } -// -// @Override -// public void onSignalingChange(PeerConnection.SignalingState newState) { -// Log.d(TAG, "SignalingState: " + newState); -// } -// -// @Override -// public void onIceConnectionChange(final PeerConnection.IceConnectionState newState) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// Log.d(TAG, "IceConnectionState: " + newState); -// if (newState == IceConnectionState.CONNECTED) { -// events.onIceConnected(); -// } else if (newState == IceConnectionState.DISCONNECTED) { -// events.onIceDisconnected(); -// } else if (newState == IceConnectionState.FAILED) { -// reportError("ICE connection failed."); -// } -// } -// }); -// } -// -// @Override -// public void onIceGatheringChange(PeerConnection.IceGatheringState newState) { -// Log.d(TAG, "IceGatheringState: " + newState); -// } -// -// @Override -// public void onIceConnectionReceivingChange(boolean receiving) { -// Log.d(TAG, "IceConnectionReceiving changed to " + receiving); -// } -// -// @Override -// public void onAddStream(final MediaStream stream) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection == null || isError) { -// return; -// } -// if (stream.audioTracks.size() > 1 || stream.videoTracks.size() > 1) { -// reportError("Weird-looking stream: " + stream); -// return; -// } -// if (stream.videoTracks.size() == 1) { -// remoteVideoTrack = stream.videoTracks.get(0); -// remoteVideoTrack.setEnabled(renderVideo); -// for (VideoRenderer.Callbacks remoteRender : remoteRenders) { -// remoteVideoTrack.addRenderer(new VideoRenderer(remoteRender)); -// } -// } -// } -// }); -// } -// -// @Override -// public void onRemoveStream(final MediaStream stream) { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// remoteVideoTrack = null; -// } -// }); -// } -// -// @Override -// public void onDataChannel(final DataChannel dc) { -// Log.d(TAG, "New Data channel " + dc.label()); -// -// if (!dataChannelEnabled) -// return; -// -// dc.registerObserver(new DataChannel.Observer() { -// public void onBufferedAmountChange(long previousAmount) { -// Log.d(TAG, "Data channel buffered amount changed: " + dc.label() + ": " + dc.state()); -// } -// -// @Override -// public void onStateChange() { -// Log.d(TAG, "Data channel state changed: " + dc.label() + ": " + dc.state()); -// } -// -// @Override -// public void onMessage(final DataChannel.Buffer buffer) { -// if (buffer.binary) { -// Log.d(TAG, "Received binary msg over " + dc); -// return; -// } -// ByteBuffer data = buffer.data; -// final byte[] bytes = new byte[data.capacity()]; -// data.get(bytes); -// String strData = new String(bytes); -// Log.d(TAG, "Got msg: " + strData + " over " + dc); -// } -// }); -// } -// -// @Override -// public void onRenegotiationNeeded() { -// // No need to do anything; AppRTC follows a pre-agreed-upon -// // signaling/negotiation protocol. -// } -// -// @Override -// public void onAddTrack(final RtpReceiver receiver, final MediaStream[] mediaStreams) { -// } -// } -// -// // Implementation detail: handle offer creation/signaling and answer setting, -// // as well as adding remote ICE candidates once the answer SDP is set. -// private class SDPObserver implements SdpObserver { -// @Override -// public void onCreateSuccess(final SessionDescription origSdp) { -// if (localSdp != null) { -// reportError("Multiple SDP create."); -// return; -// } -// String sdpDescription = origSdp.description; -// if (preferIsac) { -// sdpDescription = preferCodec(sdpDescription, AUDIO_CODEC_ISAC, true); -// } -// if (videoCallEnabled) { -// sdpDescription = preferCodec(sdpDescription, preferredVideoCodec, false); -// } -// final SessionDescription sdp = new SessionDescription(origSdp.type, sdpDescription); -// localSdp = sdp; -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection != null && !isError) { -// Log.d(TAG, "Set local SDP from " + sdp.type); -// peerConnection.setLocalDescription(sdpObserver, sdp); -// } -// } -// }); -// } -// -// @Override -// public void onSetSuccess() { -// executor.execute(new Runnable() { -// @Override -// public void run() { -// if (peerConnection == null || isError) { -// return; -// } -// if (isInitiator) { -// // For offering peer connection we first create offer and set -// // local SDP, then after receiving answer set remote SDP. -// if (peerConnection.getRemoteDescription() == null) { -// // We've just set our local SDP so time to send it. -// Log.d(TAG, "Local SDP set succesfully"); -// events.onLocalDescription(localSdp); -// } else { -// // We've just set remote description, so drain remote -// // and send local ICE candidates. -// Log.d(TAG, "Remote SDP set succesfully"); -// drainCandidates(); -// } -// } else { -// // For answering peer connection we set remote SDP and then -// // create answer and set local SDP. -// if (peerConnection.getLocalDescription() != null) { -// // We've just set our local SDP so time to send it, drain -// // remote and send local ICE candidates. -// Log.d(TAG, "Local SDP set succesfully"); -// events.onLocalDescription(localSdp); -// drainCandidates(); -// } else { -// // We've just set remote SDP - do nothing for now - -// // answer will be created soon. -// Log.d(TAG, "Remote SDP set succesfully"); -// } -// } -// } -// }); -// } -// -// @Override -// public void onCreateFailure(final String error) { -// reportError("createSDP error: " + error); -// } -// -// @Override -// public void onSetFailure(final String error) { -// reportError("setSDP error: " + error); -// } -// } -//} diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 5911aedc15..693b6168a6 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -601,11 +601,13 @@ class WebRtcPeerConnectionManager @Inject constructor( * This value was in the RTCSignalingState enum (and therefore found by reading the value of the signalingState) * property until the May 13, 2016 draft of the specification. */ - PeerConnection.PeerConnectionState.CLOSED, - /** - * At least one of the ICE transports for the connection is in the "disconnected" state and none of the other transports are in the state "failed", - * "connecting", or "checking". - */ + PeerConnection.PeerConnectionState.CLOSED -> { + + } + /** + * At least one of the ICE transports for the connection is in the "disconnected" state and none of + * the other transports are in the state "failed", "connecting", or "checking". + */ PeerConnection.PeerConnectionState.DISCONNECTED -> { } null -> { @@ -659,7 +661,8 @@ class WebRtcPeerConnectionManager @Inject constructor( PeerConnection.IceConnectionState.DISCONNECTED -> { } /** - * The ICE candidate has checked all candidates pairs against one another and has failed to find compatible matches for all components of the connection. + * The ICE candidate has checked all candidates pairs against one another and has failed to find + * compatible matches for all components of the connection. * It is, however, possible that the ICE agent did find compatible connections for some components. */ PeerConnection.IceConnectionState.FAILED -> { diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt deleted file mode 100644 index c68b22fabb..0000000000 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpService.kt +++ /dev/null @@ -1,180 +0,0 @@ -// /* -// * Copyright (c) 2020 New Vector Ltd -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// package im.vector.riotx.features.call.service -// -// import android.app.Notification -// import android.app.NotificationChannel -// import android.app.NotificationManager -// import android.app.PendingIntent -// import android.app.Service -// import android.content.ContentResolver.SCHEME_ANDROID_RESOURCE -// import android.content.Context -// import android.content.Intent -// import android.media.AudioAttributes -// import android.net.Uri -// import android.os.Binder -// import android.os.Build -// import android.os.IBinder -// import androidx.core.app.NotificationCompat -// import androidx.core.content.ContextCompat -// import androidx.core.graphics.drawable.IconCompat -// import im.vector.matrix.android.api.session.call.MxCallDetail -// import im.vector.riotx.R -// import im.vector.riotx.core.extensions.vectorComponent -// import im.vector.riotx.features.call.VectorCallActivity -// import im.vector.riotx.features.notifications.NotificationUtils -// import im.vector.riotx.features.themes.ThemeUtils -// -// class CallHeadsUpService : Service() { -// // -// // private val CHANNEL_ID = "CallChannel" -// // private val CHANNEL_NAME = "Call Channel" -// // private val CHANNEL_DESCRIPTION = "Call Notifications" -// -// lateinit var notificationUtils: NotificationUtils -// private val binder: IBinder = CallHeadsUpServiceBinder() -// -// override fun onBind(intent: Intent): IBinder? { -// return binder -// } -// -// override fun onCreate() { -// super.onCreate() -// notificationUtils = vectorComponent().notificationUtils() -// } -// override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { -// val callHeadsUpServiceArgs: CallHeadsUpServiceArgs? = intent?.extras?.getParcelable(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS) -// -// // createNotificationChannel() -// -// // val title = callHeadsUpServiceArgs?.otherUserId ?: "" -// // val description = when { -// // callHeadsUpServiceArgs?.isIncomingCall == false -> getString(R.string.call_ring) -// // callHeadsUpServiceArgs?.isVideoCall == true -> getString(R.string.incoming_video_call) -// // else -> getString(R.string.incoming_voice_call) -// // } -// -// // val actions = if (callHeadsUpServiceArgs?.isIncomingCall == true) createAnswerAndRejectActions() else emptyList() -// -// notificationUtils.buildIncomingCallNotification( -// callHeadsUpServiceArgs?.isVideoCall ?: false, -// callHeadsUpServiceArgs?.otherUserId ?: "", -// callHeadsUpServiceArgs?.roomId ?: "", -// callHeadsUpServiceArgs?.callId ?: "" -// ).let { -// startForeground(NOTIFICATION_ID, it) -// } -// // createNotification(title, description, actions).also { -// // startForeground(NOTIFICATION_ID, it) -// // } -// -// return START_STICKY -// } -// -// // private fun createNotification(title: String, content: String, actions: List): Notification { -// // val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { -// // putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) -// // }.let { -// // PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, it, PendingIntent.FLAG_UPDATE_CURRENT) -// // } -// // return NotificationCompat -// // .Builder(applicationContext, CHANNEL_ID) -// // .setContentTitle(title) -// // .setContentText(content) -// // .setSmallIcon(R.drawable.ic_call) -// // .setPriority(NotificationCompat.PRIORITY_MAX) -// // .setWhen(0) -// // .setCategory(NotificationCompat.CATEGORY_CALL) -// // .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) -// // .setDefaults(NotificationCompat.DEFAULT_SOUND or NotificationCompat.DEFAULT_VIBRATE) -// // .setSound(Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg")) -// // .setVibrate(longArrayOf(1000, 1000)) -// // .setFullScreenIntent(answerCallActionReceiver, true) -// // .setOngoing(true) -// // //.setStyle(NotificationCompat.BigTextStyle()) -// // .setAutoCancel(true) -// // .apply { actions.forEach { addAction(it) } } -// // .build() -// // } -// -// // private fun createAnswerAndRejectActions(): List { -// // val answerCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { -// // putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_ANSWER) -// // } -// // val rejectCallActionReceiver = Intent(applicationContext, CallHeadsUpActionReceiver::class.java).apply { -// // putExtra(EXTRA_CALL_ACTION_KEY, CALL_ACTION_REJECT) -// // } -// // val answerCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_ANSWER, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) -// // val rejectCallPendingIntent = PendingIntent.getBroadcast(applicationContext, CALL_ACTION_REJECT, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) -// // -// // return listOf( -// // NotificationCompat.Action( -// // R.drawable.ic_call, -// // //IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), -// // getString(R.string.call_notification_answer), -// // answerCallPendingIntent -// // ), -// // NotificationCompat.Action( -// // IconCompat.createWithResource(applicationContext, R.drawable.ic_call_end).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_notice)), -// // getString(R.string.call_notification_reject), -// // rejectCallPendingIntent) -// // ) -// // } -// -// // private fun createNotificationChannel() { -// // if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return -// // -// // val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH).apply { -// // description = CHANNEL_DESCRIPTION -// // setSound( -// // Uri.parse(SCHEME_ANDROID_RESOURCE + "://" + applicationContext.packageName + "/raw/ring.ogg"), -// // AudioAttributes -// // .Builder() -// // .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) -// // .setUsage(AudioAttributes.USAGE_NOTIFICATION_RINGTONE) -// // .build() -// // ) -// // lockscreenVisibility = NotificationCompat.VISIBILITY_PUBLIC -// // enableVibration(true) -// // enableLights(true) -// // } -// // applicationContext.getSystemService(NotificationManager::class.java)?.createNotificationChannel(channel) -// // } -// -// inner class CallHeadsUpServiceBinder : Binder() { -// -// fun getService() = this@CallHeadsUpService -// } -// -// -// companion object { -// private const val EXTRA_CALL_HEADS_UP_SERVICE_PARAMS = "EXTRA_CALL_PARAMS" -// -// const val EXTRA_CALL_ACTION_KEY = "EXTRA_CALL_ACTION_KEY" -// // const val CALL_ACTION_ANSWER = 100 -// const val CALL_ACTION_REJECT = 101 -// -// private const val NOTIFICATION_ID = 999 -// -// fun newInstance(context: Context, mxCall: MxCallDetail): Intent { -// val args = CallHeadsUpServiceArgs(mxCall.callId, mxCall.roomId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall) -// return Intent(context, CallHeadsUpService::class.java).apply { -// putExtra(EXTRA_CALL_HEADS_UP_SERVICE_PARAMS, args) -// } -// } -// } -// } diff --git a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt index dce8c72101..72471ccf1d 100755 --- a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt +++ b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt @@ -326,7 +326,8 @@ class NotificationUtils @Inject constructor(private val context: Context, builder.addAction( NotificationCompat.Action( R.drawable.ic_call, - // IconCompat.createWithResource(applicationContext, R.drawable.ic_call).setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), + // IconCompat.createWithResource(applicationContext, R.drawable.ic_call) + // .setTint(ContextCompat.getColor(applicationContext, R.color.riotx_positive_accent)), context.getString(R.string.call_notification_answer), answerCallPendingIntent ) From 39f3a1c6972f108c7656bdb07008be17a1be58af Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 12 Jun 2020 19:10:04 +0200 Subject: [PATCH 43/83] Fix glitch when opening timeline first time --- vector/src/main/res/layout/fragment_room_detail.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vector/src/main/res/layout/fragment_room_detail.xml b/vector/src/main/res/layout/fragment_room_detail.xml index 132dc1e686..f2d3bdc009 100644 --- a/vector/src/main/res/layout/fragment_room_detail.xml +++ b/vector/src/main/res/layout/fragment_room_detail.xml @@ -98,6 +98,8 @@ app:layout_constraintTop_toBottomOf="@id/roomToolbar" /> + app:layout_constraintTop_toBottomOf="@id/activeCallView"> Date: Fri, 12 Jun 2020 19:10:20 +0200 Subject: [PATCH 44/83] Very basic audio speaker support --- vector/src/main/AndroidManifest.xml | 2 + .../riotx/features/call/CallAudioManager.kt | 115 ++++++++++++++++++ .../call/WebRtcPeerConnectionManager.kt | 8 +- 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 50c9eabadb..6b0253c5fc 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -14,6 +14,8 @@ + + diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt new file mode 100644 index 0000000000..a5c3069367 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.content.Context +import android.content.pm.PackageManager +import android.media.AudioManager +import im.vector.matrix.android.api.session.call.MxCall +import timber.log.Timber + +class CallAudioManager( + val applicationContext: Context +) { + + private val audioManager: AudioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + private var savedIsSpeakerPhoneOn = false + private var savedIsMicrophoneMute = false + private var savedAudioMode = AudioManager.MODE_INVALID + + private val audioFocusChangeListener = AudioManager.OnAudioFocusChangeListener { focusChange -> + + // Called on the listener to notify if the audio focus for this listener has been changed. + // The |focusChange| value indicates whether the focus was gained, whether the focus was lost, + // and whether that loss is transient, or whether the new focus holder will hold it for an + // unknown amount of time. + Timber.v("## VOIP: Audio focus change $focusChange") + } + + fun startForCall(mxCall: MxCall) { + Timber.v("## VOIP: AudioManager startForCall ${mxCall.callId}") + savedIsSpeakerPhoneOn = audioManager.isSpeakerphoneOn + savedIsMicrophoneMute = audioManager.isMicrophoneMute + savedAudioMode = audioManager.mode + + // Request audio playout focus (without ducking) and install listener for changes in focus. + + // Remove the deprecation forces us to use 2 different method depending on API level + @Suppress("DEPRECATION") val result = audioManager.requestAudioFocus(audioFocusChangeListener, + AudioManager.STREAM_VOICE_CALL, AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) + if (result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + Timber.d("## VOIP Audio focus request granted for VOICE_CALL streams") + } else { + Timber.d("## VOIP Audio focus request failed") + } + + // Start by setting MODE_IN_COMMUNICATION as default audio mode. It is + // required to be in this mode when playout and/or recording starts for + // best possible VoIP performance. + audioManager.mode = AudioManager.MODE_IN_COMMUNICATION + + // Always disable microphone mute during a WebRTC call. + setMicrophoneMute(false) + + // TODO check if there are headsets? + if (mxCall.isVideoCall) { + setSpeakerphoneOn(true) + } + } + + fun stop() { + Timber.v("## VOIP: AudioManager stopCall") + + // Restore previously stored audio states. + setSpeakerphoneOn(savedIsSpeakerPhoneOn) + setMicrophoneMute(savedIsMicrophoneMute) + audioManager.mode = savedAudioMode + + @Suppress("DEPRECATION") + audioManager.abandonAudioFocus(audioFocusChangeListener) + } + + /** Sets the speaker phone mode. */ + private fun setSpeakerphoneOn(on: Boolean) { + Timber.v("## VOIP: AudioManager setSpeakerphoneOn $on") + val wasOn = audioManager.isSpeakerphoneOn + if (wasOn == on) { + return + } + audioManager.isSpeakerphoneOn = on + } + + /** Sets the microphone mute state. */ + private fun setMicrophoneMute(on: Boolean) { + Timber.v("## VOIP: AudioManager setMicrophoneMute $on") + val wasMuted = audioManager.isMicrophoneMute + if (wasMuted == on) { + return + } + audioManager.isMicrophoneMute = on + + audioManager.isMusicActive + } + + /** true if the device has a telephony radio with data + * communication support. */ + private fun isThisPhone(): Boolean { + return applicationContext.packageManager.hasSystemFeature( + PackageManager.FEATURE_TELEPHONY) + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 693b6168a6..f40d02d2f8 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -82,6 +82,8 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCallsListeners.remove(listener) } + val audioManager = CallAudioManager(context.applicationContext) + data class CallContext( val mxCall: MxCall, @@ -457,6 +459,8 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall") val createdCall = sessionHolder.getSafeActiveSession()?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return val callContext = CallContext(createdCall) + + audioManager.startForCall(createdCall) currentCall = callContext executor.execute { @@ -489,11 +493,13 @@ class WebRtcPeerConnectionManager @Inject constructor( if (currentCall != null) { Timber.w("## VOIP TODO: Automatically reject incoming call?") mxCall.hangUp() + audioManager.stop() return } val callContext = CallContext(mxCall) currentCall = callContext + audioManager.startForCall(mxCall) executor.execute { callContext.remoteCandidateSource = ReplaySubject.create() } @@ -538,6 +544,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun endCall() { currentCall?.mxCall?.hangUp() currentCall = null + audioManager.stop() close() } @@ -602,7 +609,6 @@ class WebRtcPeerConnectionManager @Inject constructor( * property until the May 13, 2016 draft of the specification. */ PeerConnection.PeerConnectionState.CLOSED -> { - } /** * At least one of the ICE transports for the connection is in the "disconnected" state and none of From 0f625c27a1863016c8bd2d70f21568681e1716f1 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 15 Jun 2020 15:46:54 +0200 Subject: [PATCH 45/83] Simple menu to select sound device --- .../vector/riotx/core/di/ScreenComponent.kt | 2 + .../VectorBaseBottomSheetDialogFragment.kt | 14 ++++ .../riotx/features/call/CallAudioManager.kt | 22 ++++++ .../features/call/CallControlsBottomSheet.kt | 78 +++++++++++++++++++ .../riotx/features/call/CallControlsView.kt | 9 ++- .../riotx/features/call/VectorCallActivity.kt | 71 ++++++++++++++++- .../features/call/VectorCallViewModel.kt | 24 ++++-- vector/src/main/res/layout/activity_call.xml | 1 + .../res/layout/bottom_sheet_call_controls.xml | 19 +++++ .../res/layout/fragment_call_controls.xml | 4 +- .../main/res/layout/fragment_room_detail.xml | 2 + .../main/res/layout/view_active_call_view.xml | 3 + vector/src/main/res/values/strings.xml | 4 + 13 files changed, 242 insertions(+), 11 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt create mode 100644 vector/src/main/res/layout/bottom_sheet_call_controls.xml diff --git a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt index ce00162e5c..c5fdb89ea4 100644 --- a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt +++ b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt @@ -24,6 +24,7 @@ import dagger.Component import im.vector.riotx.core.error.ErrorFormatter import im.vector.riotx.core.preference.UserAvatarPreference import im.vector.riotx.features.MainActivity +import im.vector.riotx.features.call.CallControlsBottomSheet import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.createdirect.CreateDirectRoomActivity import im.vector.riotx.features.crypto.keysbackup.settings.KeysBackupManageActivity @@ -148,6 +149,7 @@ interface ScreenComponent { fun inject(bottomSheet: BootstrapBottomSheet) fun inject(bottomSheet: RoomWidgetPermissionBottomSheet) fun inject(bottomSheet: RoomWidgetsBottomSheet) + fun inject(callControlsBottomSheet: CallControlsBottomSheet) /* ========================================================================================== * Others diff --git a/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseBottomSheetDialogFragment.kt b/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseBottomSheetDialogFragment.kt index fbc4bc5292..393642139e 100644 --- a/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseBottomSheetDialogFragment.kt +++ b/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseBottomSheetDialogFragment.kt @@ -34,6 +34,7 @@ import com.airbnb.mvrx.MvRxViewId import com.google.android.material.bottomsheet.BottomSheetBehavior import com.google.android.material.bottomsheet.BottomSheetDialog import com.google.android.material.bottomsheet.BottomSheetDialogFragment +import com.jakewharton.rxbinding3.view.clicks import im.vector.riotx.core.di.DaggerScreenComponent import im.vector.riotx.core.di.ScreenComponent import im.vector.riotx.core.utils.DimensionConverter @@ -41,6 +42,7 @@ import io.reactivex.android.schedulers.AndroidSchedulers import io.reactivex.disposables.CompositeDisposable import io.reactivex.disposables.Disposable import timber.log.Timber +import java.util.concurrent.TimeUnit /** * Add MvRx capabilities to bottomsheetdialog (like BaseMvRxFragment) @@ -169,6 +171,18 @@ abstract class VectorBaseBottomSheetDialogFragment : BottomSheetDialogFragment() return this } + /* ========================================================================================== + * Views + * ========================================================================================== */ + + protected fun View.debouncedClicks(onClicked: () -> Unit) { + clicks() + .throttleFirst(300, TimeUnit.MILLISECONDS) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { onClicked() } + .disposeOnDestroyView() + } + /* ========================================================================================== * ViewEvents * ========================================================================================== */ diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt index a5c3069367..b3bdc23707 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -26,6 +26,11 @@ class CallAudioManager( val applicationContext: Context ) { + enum class SoundDevice { + PHONE, + SPEAKER + } + private val audioManager: AudioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager private var savedIsSpeakerPhoneOn = false @@ -69,6 +74,8 @@ class CallAudioManager( // TODO check if there are headsets? if (mxCall.isVideoCall) { setSpeakerphoneOn(true) + } else { + setSpeakerphoneOn(false) } } @@ -84,6 +91,21 @@ class CallAudioManager( audioManager.abandonAudioFocus(audioFocusChangeListener) } + fun getCurrentSoundDevice() : SoundDevice { + if (audioManager.isSpeakerphoneOn) { + return SoundDevice.SPEAKER + } else { + return SoundDevice.PHONE + } + } + + fun setCurrentSoundDevice(device: SoundDevice) { + when (device) { + SoundDevice.PHONE -> setSpeakerphoneOn(false) + SoundDevice.SPEAKER -> setSpeakerphoneOn(true) + } + } + /** Sets the speaker phone mode. */ private fun setSpeakerphoneOn(on: Boolean) { Timber.v("## VOIP: AudioManager setSpeakerphoneOn $on") diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt new file mode 100644 index 0000000000..6e4a304932 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.os.Bundle +import android.view.View +import androidx.appcompat.app.AlertDialog +import com.airbnb.mvrx.activityViewModel +import im.vector.riotx.R +import im.vector.riotx.core.platform.VectorBaseBottomSheetDialogFragment +import kotlinx.android.synthetic.main.bottom_sheet_call_controls.* + +class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { + override fun getLayoutResId() = R.layout.bottom_sheet_call_controls + + private val callViewModel: VectorCallViewModel by activityViewModel() + + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + super.onViewCreated(view, savedInstanceState) + + callViewModel.subscribe(this) { + renderState(it) + } + + callControlsSoundDevice.clickableView.debouncedClicks { + val soundDevices = listOf( + getString(R.string.sound_device_phone), + getString(R.string.sound_device_speaker) + ) + AlertDialog.Builder(requireContext()) + .setItems(soundDevices.toTypedArray()) { d, n -> + d.cancel() + when (soundDevices[n]) { + getString(R.string.sound_device_phone) -> { + callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.PHONE)) + } + getString(R.string.sound_device_speaker) -> { + callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.SPEAKER)) + } + } + } + .setNegativeButton(R.string.cancel, null) + .show() + } + } + +// override fun onCreateDialog(savedInstanceState: Bundle?): Dialog { +// return super.onCreateDialog(savedInstanceState).apply { +// window?.addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN) +// window?.decorView?.systemUiVisibility = +// View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or +// View.SYSTEM_UI_FLAG_FULLSCREEN or +// View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY +// } +// } + + private fun renderState(state: VectorCallViewState) { + callControlsSoundDevice.title = getString(R.string.call_select_sound_device) + callControlsSoundDevice.subTitle = when (state.soundDevice) { + CallAudioManager.SoundDevice.PHONE -> getString(R.string.sound_device_phone) + CallAudioManager.SoundDevice.SPEAKER -> getString(R.string.sound_device_speaker) + } + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index f8ed0f0cc1..9c4bf5f6e1 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -22,7 +22,6 @@ import android.view.ViewGroup import android.widget.ImageView import android.widget.LinearLayout import androidx.constraintlayout.widget.ConstraintLayout -import androidx.core.view.isInvisible import androidx.core.view.isVisible import butterknife.BindView import butterknife.ButterKnife @@ -89,6 +88,11 @@ class CallControlsView @JvmOverloads constructor( interactionListener?.returnToChat() } + @OnClick(R.id.iv_more) + fun moreControlOption() { + interactionListener?.didTapMore() + } + fun updateForState(state: VectorCallViewState) { val callState = state.callState.invoke() muteIcon.setImageResource(if (state.isAudioMuted) R.drawable.ic_microphone_off else R.drawable.ic_microphone_on) @@ -113,7 +117,7 @@ class CallControlsView @JvmOverloads constructor( CallState.CONNECTED -> { ringingControls.isVisible = false connectedControls.isVisible = true - iv_video_toggle.isInvisible = !state.isVideoCall + iv_video_toggle.isVisible = state.isVideoCall } CallState.TERMINATED, null -> { @@ -130,5 +134,6 @@ class CallControlsView @JvmOverloads constructor( fun didTapToggleMute() fun didTapToggleVideo() fun returnToChat() + fun didTapMore() } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index d201588bf9..5bb1d8d93c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -26,11 +26,14 @@ import android.os.Parcelable import android.view.View import android.view.Window import android.view.WindowManager +import androidx.core.view.ViewCompat import androidx.core.view.isInvisible import androidx.core.view.isVisible +import androidx.core.view.updatePadding import butterknife.BindView import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel +import com.jakewharton.rxbinding3.view.clicks import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCallDetail @@ -46,10 +49,12 @@ import im.vector.riotx.features.home.AvatarRenderer import io.reactivex.android.schedulers.AndroidSchedulers import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* +import kotlinx.android.synthetic.main.fragment_attachments_preview.* import org.webrtc.EglBase import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer import timber.log.Timber +import java.util.concurrent.TimeUnit import javax.inject.Inject @Parcelize @@ -91,6 +96,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private var rootEglBase: EglBase? = null + var systemUiVisibility = false + override fun doBeforeSetContentView() { // Set window styles for fullscreen-window size. Needs to be done before adding content. requestWindowFeature(Window.FEATURE_NO_TITLE) @@ -108,14 +115,64 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis ) } - window.decorView.systemUiVisibility = - View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY + hideSystemUI() setContentView(R.layout.activity_call) } + private fun hideSystemUI() { + systemUiVisibility = false + // Enables regular immersive mode. + // For "lean back" mode, remove SYSTEM_UI_FLAG_IMMERSIVE. + // Or for "sticky immersive," replace it with SYSTEM_UI_FLAG_IMMERSIVE_STICKY + window.decorView.systemUiVisibility = (View.SYSTEM_UI_FLAG_IMMERSIVE + // Set the content to appear under the system bars so that the + // content doesn't resize when the system bars hide and show. + or View.SYSTEM_UI_FLAG_LAYOUT_STABLE + or View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION + or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN + // Hide the nav bar and status bar + or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION + or View.SYSTEM_UI_FLAG_FULLSCREEN) + } + + // Shows the system bars by removing all the flags +// except for the ones that make the content appear under the system bars. + private fun showSystemUI() { + systemUiVisibility = true + window.decorView.systemUiVisibility = (View.SYSTEM_UI_FLAG_LAYOUT_STABLE + or View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION + or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN) + } + + private fun toggleUiSystemVisibility() { + if (systemUiVisibility) { + hideSystemUI() + } else { + showSystemUI() + } + } + + override fun onWindowFocusChanged(hasFocus: Boolean) { + super.onWindowFocusChanged(hasFocus) + // Rehide when bottom sheet is dismissed + if (hasFocus) { + hideSystemUI() + } + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + // This will need to be refined + ViewCompat.setOnApplyWindowInsetsListener(constraintLayout) { v, insets -> + v.updatePadding(bottom = if (systemUiVisibility) insets.systemWindowInsetBottom else 0) + insets + } +// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { +// // window.navigationBarColor = ContextCompat.getColor(this, R.color.riotx_background_light) +// // } + + // for content intent when screen is locked // window.addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED); // window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); @@ -125,6 +182,12 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis finish() } + constraintLayout.clicks() + .throttleFirst(300, TimeUnit.MILLISECONDS) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { toggleUiSystemVisibility() } + .disposeOnDestroy() + if (isFirstCreation()) { // Reduce priority of notification as the activity is on screen CallService.onPendingCall( @@ -342,4 +405,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis // TODO, what if the room is not in backstack?? finish() } + + override fun didTapMore() { + CallControlsBottomSheet().show(supportFragmentManager, "Controls") + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index ec09b8bc2c..809df48517 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -41,6 +41,7 @@ data class VectorCallViewState( val isVideoCall: Boolean, val isAudioMuted: Boolean = false, val isVideoEnabled: Boolean = true, + val soundDevice: CallAudioManager.SoundDevice = CallAudioManager.SoundDevice.PHONE, val otherUserMatrixItem: Async = Uninitialized, val callState: Async = Uninitialized ) : MvRxState @@ -51,6 +52,7 @@ sealed class VectorCallViewActions : VectorViewModelAction { object DeclineCall : VectorCallViewActions() object ToggleMute : VectorCallViewActions() object ToggleVideo : VectorCallViewActions() + data class ChangeAudioDevice(val device: CallAudioManager.SoundDevice) : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { @@ -86,6 +88,7 @@ class VectorCallViewModel @AssistedInject constructor( autoReplyIfNeeded = args.autoAccept initialState.callId?.let { + session.callSignalingService().getCallWithId(it)?.let { mxCall -> this.call = mxCall mxCall.otherUserId @@ -96,7 +99,8 @@ class VectorCallViewModel @AssistedInject constructor( copy( isVideoCall = mxCall.isVideoCall, callState = Success(mxCall.state), - otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized + otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized, + soundDevice = webRtcPeerConnectionManager.audioManager.getCurrentSoundDevice() ) } } @@ -111,20 +115,20 @@ class VectorCallViewModel @AssistedInject constructor( override fun handle(action: VectorCallViewActions) = withState { when (action) { - VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() - VectorCallViewActions.AcceptCall -> { + VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() + VectorCallViewActions.AcceptCall -> { setState { copy(callState = Loading()) } webRtcPeerConnectionManager.acceptIncomingCall() } - VectorCallViewActions.DeclineCall -> { + VectorCallViewActions.DeclineCall -> { setState { copy(callState = Loading()) } webRtcPeerConnectionManager.endCall() } - VectorCallViewActions.ToggleMute -> { + VectorCallViewActions.ToggleMute -> { withState { val muted = it.isAudioMuted webRtcPeerConnectionManager.muteCall(!muted) @@ -133,7 +137,7 @@ class VectorCallViewModel @AssistedInject constructor( } } } - VectorCallViewActions.ToggleVideo -> { + VectorCallViewActions.ToggleVideo -> { withState { if (it.isVideoCall) { val videoEnabled = it.isVideoEnabled @@ -144,6 +148,14 @@ class VectorCallViewModel @AssistedInject constructor( } } } + is VectorCallViewActions.ChangeAudioDevice -> { + webRtcPeerConnectionManager.audioManager.setCurrentSoundDevice(action.device) + setState { + copy( + soundDevice = webRtcPeerConnectionManager.audioManager.getCurrentSoundDevice() + ) + } + } }.exhaustive } diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index 6448a1336d..9b27bef6ac 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -5,6 +5,7 @@ xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" + android:id="@+id/constraintLayout" android:background="?riotx_background" tools:ignore="MergeRootFrame"> diff --git a/vector/src/main/res/layout/bottom_sheet_call_controls.xml b/vector/src/main/res/layout/bottom_sheet_call_controls.xml new file mode 100644 index 0000000000..0c17bcb049 --- /dev/null +++ b/vector/src/main/res/layout/bottom_sheet_call_controls.xml @@ -0,0 +1,19 @@ + + + + + + diff --git a/vector/src/main/res/layout/fragment_call_controls.xml b/vector/src/main/res/layout/fragment_call_controls.xml index 6f1861afec..74da380f9e 100644 --- a/vector/src/main/res/layout/fragment_call_controls.xml +++ b/vector/src/main/res/layout/fragment_call_controls.xml @@ -123,11 +123,13 @@ android:layout_width="44dp" android:layout_height="44dp" android:layout_marginBottom="32dp" + android:background="@drawable/oval_positive" + android:backgroundTint="?attr/riotx_background" android:clickable="true" android:focusable="true" android:padding="8dp" android:src="@drawable/ic_more_vertical" - android:tint="?attr/riotx_background" + android:tint="?attr/riotx_text_primary" tools:ignore="MissingConstraints" /> Please ask the administrator of your homeserver (%1$s) to configure a TURN server in order for calls to work reliably.\n\nAlternatively, you can try to use the public server at %2$s, but this will not be as reliable, and it will share your IP address with that server. You can also manage this in Settings." Try using %s Do not ask me again + Select Sound Device + Phone + Speaker + Send files Send sticker From 4966bef9c3d782a757c579452932686c05bdf5f4 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 15 Jun 2020 15:47:30 +0200 Subject: [PATCH 46/83] Quick signaling doc --- docs/voip_signaling.md | 420 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 420 insertions(+) create mode 100644 docs/voip_signaling.md diff --git a/docs/voip_signaling.md b/docs/voip_signaling.md new file mode 100644 index 0000000000..ba72e1b218 --- /dev/null +++ b/docs/voip_signaling.md @@ -0,0 +1,420 @@ + ╔════════════════════════════════════════════════╗ + ║ ║ + ║A] Placing a call offer ║ + ║ ║ + ╚════════════════════════════════════════════════╝ + + + + ┌───────────────┐ + │ Matrix │ + ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + │ + │ + │ + ┌─────────────────┐ │ ┌───────────────────────────┐ ┌─────────────────┐ + │ Caller │ │ Signaling Room │ │ │ Callee │ + └─────────────────┘ │ ├───────────────────────────┤ └─────────────────┘ + ┌────┐ │ │ │ + │ 3 │ │ │ ┌────────────────────┐ │ + ┌─────────────────┐──────┴────┴──────────────────────────┼─▶│ m.call.invite │ │ │ ┌─────────────────┐ + │ │ │ │ │ mx event │ │ │ │ + │ │ │ └────────────────────┘ │ │ │ │ + │ │ │ │ │ │ │ + │ Riot.im │ │ │ │ │ Riot.im │ + ┌──│ App │ │ │ │ │ App │ + │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ │ + │ └─────────────────┘ │ │ │ └─────────────────┘ + ┌────┤ ▲ │ │ │ + │ 1 │ ├────┐ │ └───────────────────────────┘ + └────┤ │ 2 │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + │ ┌──┴────┴─────────┐ ┌─────────────────┐ + │ │ │ │ │ + │ │ │ │ │ + │ │ WebRtc │ │ WebRtc │ + └─▶│ │ │ │ + │ │ │ │ + │ │ │ │ + └─────────────────┘ └─────────────────┘ + + + + + + + + + + ┌────┐ + │ 1 │ The Caller app get access to system resources (camera, mic), eventually stun/turn servers, define some + └────┘ constrains (video quality, format) and pass it to WebRtc in order to create a Peer Call offer + + ┌────┐ + │ 2 │ The WebRtc layer creates a call Offer (sdp) that needs to be sent to callee + └────┘ + + ┌────┐ The app layer, takes the webrtc offer, encapsulate it in a matrix event adds a callId and send it to the other + │ 3 │ user via the room + └────┘ + ┌──────────────┐ + │ mx event │ + ├──────────────┴────────┐ + │ type: m.call.invite │ + │ + callId │ + │ │ + │ ┌──────────────────┐ │ + │ │ webrtc sdp │ │ + │ └──────────────────┘ │ + └───────────────────────┘ + + + + + + + + ╔════════════════════════════════════════════════╗ + ║ ║ + ║B] Sending connection establishment info ║ + ║ ║ + ╚════════════════════════════════════════════════╝ + + + + ┌───────────────┐ + │ Matrix │ + ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + │ + │ + │ + ┌─────────────────┐ │ ┌───────────────────────────┐ ┌─────────────────┐ + │ Caller │ │ Signaling Room │ │ │ Callee │ + └─────────────────┘ │ ├───────────────────────────┤ └─────────────────┘ + │ ┌────────────────────┐ │ │ + │ │ │ m.call.invite │ │ + ┌─────────────────┐ │ │ mx event │ │ │ ┌─────────────────┐ + │ │ ┌────┐ │ │ └────────────────────┘ │ │ │ + │ │ │ 3 │ │ ┌────────────────────┐ │ │ │ │ + │ │──────┴────┴───────┼──────────────────┼─▶│ m.call.candidates │ │ │ │ + │ Riot.im │ │ │ mx event │ │ │ │ Riot.im │ + │ App │ │ │ └────────────────────┘ │ │ App │ + │ │ │ │ │ │ │ + │ │ │ │ │ │ │ + │ │ │ │ │ │ │ + └─────────────────┘ │ │ │ └─────────────────┘ + ▲ │ │ │ + ├────┐ │ └───────────────────────────┘ + │ 2 │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + ┌───────┴────┴────┐ ┌─────────────────┐ + │ │ │ │ + │ │ │ │ + │ WebRtc │ ┌───────────────┐ │ WebRtc │ + │ │ │ Stun / Turn │ │ │ + │ │ ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ │ + │ │ │ │ │ + └─────────────────┘ │ └─────────────────┘ + ▲ │ + │ │ + └──────────┬────┬───────────▶ │ + ┌───────────────┐ │ 1 │ │ + │ │ └────┘ │ + │ Network Stack │ │ + │ │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + │ │ + └───────────────┘ + + + + + ┌────┐ + │ 1 │ The WebRtc layer gathers information on how it can be reach by the other peer directly (Ice candidates) + └────┘ + ┌──────────────────────────────────────────────────────────────────┐ + │candidate:1 1 tcp 1518149375 127.0.0.1 35990 typ host │ + └──────────────────────────────────────────────────────────────────┘ + ┌──────────────────────────────────────────────────────────────────┐ + │candidate:2 1 UDP 2130706431 192.168.1.102 1816 typ host │ + └──────────────────────────────────────────────────────────────────┘ + + ┌────┐ + │ 2 │ The WebRTC layer notifies the App layer when it finds new Ice Candidates + └────┘ + + + + ┌────┐ The app layer, takes the ice candidates, encapsulate them in one or several matrix event adds the callId and + │ 3 │ send it to the other user via the room + └────┘ + ┌──────────────┐ + │ mx event │ + ├──────────────┴────────────────────────┐ + │ type: m.call.candidates │ + │ │ + │ +CallId │ + │ │ + │ ┌──────────────────┐ │ + │ │ice candidate sdp │ │ + │ └──────────────────┘ │ + │ ┌──────────────────┐ │ + │ │ice candidate sdp │ │ + │ └──────────────────┘ │ + │ ┌──────────────────┐ │ + │ │ice candidate sdp │ │ + │ └──────────────────┘ │ + └───────────────────────────────────────┘ + + + + + + ╔════════════════════════════════════════════════╗ + ║ ║ + ║C] Receiving a call offer ║ + ║ ║ + ╚════════════════════════════════════════════════╝ + + + + ┌───────────────┐ + │ Matrix │ + ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + │ + │ ┌─────────────────┐ + │ │ Callee │ + ┌─────────────────┐ │ ┌───────────────────────────┐ └─────────────────┘ + │ Caller │ │ Signaling Room │ │ + └─────────────────┘ │ ├───────────────────────────┤ + │ ┌────────────────────┐ │ │ ┌─────────────────┐ + │ │ │ m.call.invite │───┼────────────────────────────┬────┬───▶│ │ + ┌─────────────────┐ │ │ mx event │ │ │ │ 1 │ │ │ + │ │ │ │ └────────────────────┘ │ └────┘ │ │ + │ │ │ ┌────────────────────┐ │ │ │ Riot.im │ + │ │ │ │ │ m.call.candidates │ │ │ App │ + │ Riot.im │ │ │ mx event │ │ │ │ │ + │ App │ │ │ └────────────────────┘ │ │ │ + │ │ │ ┌────────────────────┐◀──┼─────────────────┼───┬────┬───────────┤ │ + │ │◀──────────────────┼──────────────────┼──│ m.call.answer │ │ │ 4 │ └──┬──────────────┘ + │ │ │ │ mx event │ │ │ └────┘ ├────┐ ▲ + └────┬────────────┘ │ │ └────────────────────┘ │ │ 2 │ ├────┐ + │ │ │ │ ├────┘ │ 3 │ + │ │ └───────────────────────────┘ ┌──▼─────────┴────┤ + ┌────▼────────────┐ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ │ │ + │ │ │ │ + │ │ │ WebRtc │ + │ WebRtc │ │ ┌──┴─────────────────┐ + │ │ │ │ caller offer │ + ┌──────────┴─────────┐ │ │ └──┬─────────────────┘ + │ callee answer │ │ └─────────────────┘ + └────────────────────┴───────┘ + + + + + + + + ┌────┐ + │ 1 │ Bob receives a call.invite event in a room, then creates a WebRTC peer connection object + └────┘ + + ┌────┐ + │ 2 │ The encapsulated call offer sdp from the mx event is transmitted to WebRTC + └────┘ + + ┌────┐ + │ 3 │ WebRTC then creates a call answer for the offer and send it back to app layer + └────┘ + + + ┌────┐ The app layer, takes the webrtc answer, encapsulate it in a matrix event adds a callId and send it to the + │ 3 │ other user via the room + └────┘ + ┌──────────────┐ + │ mx event │ + ├──────────────┴────────┐ + │ type: m.call.answer │ + │ + callId │ + │ │ + │ ┌──────────────────┐ │ + │ │ webrtc sdp │ │ + │ └──────────────────┘ │ + └───────────────────────┘ + + + + + + + ╔════════════════════════════════════════════════╗ + ║ ║ + ║D] Callee sends connection establishment info ║ + ║ ║ + ╚════════════════════════════════════════════════╝ + + + + ┌───────────────┐ + │ Matrix │ + ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + │ + │ + │ + ┌─────────────────┐ │ ┌───────────────────────────┐ ┌─────────────────┐ + │ Caller │ │ Signaling Room │ │ │ Callee │ + └─────────────────┘ │ ├───────────────────────────┤ └─────────────────┘ + │ ┌────────────────────┐ │ │ + │ │ │ m.call.invite │ │ + ┌─────────────────┐ │ │ mx event │ │ │ ┌─────────────────┐ + │ │ │ │ └────────────────────┘ │ │ │ + │ │ │ ┌────────────────────┐ │ │ │ │ + │ │ │ │ │ m.call.candidates │ │ │ │ + │ Riot.im │ │ │ mx event │ │ │ │ Riot.im │ + │ App │ │ │ └────────────────────┘ │ ┌────┐ │ App │ + │ │ │ ┌────────────────────┐ │ │ │ 3 │ │ │ + │ │◀──────────────────┼┐ │ │ m.call.answer │ │ ┌───────┴────┴────────│ │ + │ │ │ │ │ mx event │ │ ││ │ │ + └─────────────────┘ ││ │ └────────────────────┘ │ │ └─────────────────┘ + │ │ │ ┌────────────────────┐ │ ││ ▲ + │ │└─────────────────┼──│ m.call.candidates │ │ │ ├────┐ + ▼ │ │ mx event │◀──┼────────────────┘│ │ 2 │ + ┌─────────────────┐ │ │ └────────────────────┘ │ ┌────┴────┴───────┐ + │ │ └───────────────────────────┘ │ │ │ + │ │ │ │ │ + │ WebRtc │ │ │ WebRtc │ + │ │ └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ ┌───┴────────────────┐ + │ │ │ │ caller offer │ + ┌────────┴───────────┐ │ │ └───┬────────────────┘ + │ callee answer ├─────┘ ┌───────────────┐ └─────────────────┘ + ├────────────────────┤ │ Stun / Turn │ ▲ + │ callee ice │ ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ┌────┐ │ + │ candidates │ │ │ 1 │ │ + └────────────────────┘ │ ├────┴──┴───────┐ + │ │ │ + │ │ Network Stack │ + │◀─────────────────────┤ │ + │ │ │ + │ └───────────────┘ + │ + ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + + + ┌────┐ + │ 1 │ The WebRtc layer gathers information on how it can be reach by the other peer directly (Ice candidates) + └────┘ + ┌──────────────────────────────────────────────────────────────────┐ + │candidate:1 1 tcp 1518149375 127.0.0.1 35990 typ host │ + └──────────────────────────────────────────────────────────────────┘ + ┌──────────────────────────────────────────────────────────────────┐ + │candidate:2 1 UDP 2130706431 192.168.1.102 1816 typ host │ + └──────────────────────────────────────────────────────────────────┘ + + ┌────┐ + │ 2 │ The WebRTC layer notifies the App layer when it finds new Ice Candidates + └────┘ + + + + ┌────┐ The app layer, takes the ice candidates, encapsulate them in one or several matrix event adds the callId and + │ 3 │ send it to the other user via the room + └────┘ + ┌──────────────┐ + │ mx event │ + ├──────────────┴────────────────────────┐ + │ type: m.call.candidates │ + │ │ + │ +CallId │ + │ │ + │ ┌──────────────────┐ │ + │ │ice candidate sdp │ │ + │ └──────────────────┘ │ + │ ┌──────────────────┐ │ + │ │ice candidate sdp │ │ + │ └──────────────────┘ │ + │ ┌──────────────────┐ │ + │ │ice candidate sdp │ │ + │ └──────────────────┘ │ + └───────────────────────────────────────┘ + + + + + + + ╔════════════════════════════════════════════════╗ + ║ ║ + ║D] Caller Callee connection ║ + ║ ║ + ╚════════════════════════════════════════════════╝ + + + + + + + + + ┌───────────────┐ + ┌─────────────────┐ │ Matrix │ ┌─────────────────┐ + │ Caller │ ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ Callee │ + └─────────────────┘ │ └─────────────────┘ + │ + │ + ┌─────────────────┐ │ ┌─────────────────┐ + │ │ │ │ │ + │ │ │ │ │ + │ │ │ │ │ + │ Riot.im │ └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ Riot.im │ + │ App │ │ App │ + │ │ │ │ + │ │ │ │ + │ │ │ │ + └─────────────────┘ └─────────────────┘ + ┌───────────────┐ + │ Internet │ + ├───────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + ┌─────────────────┐ │ ┌─────────────────┐ + │ │ │ │ │ + │ ├───────────────────────────────────────────────────────────────────────────────────┴─────────────────────┤ │ + │ WebRtc │█████████████████████████████████████████████████████████████████████████████████████████████████████████│ WebRtc │ + ┌─────────────┴──────┐ ├────────────────────────────────────────┬──────────────────────────┬───────────────┬─────────────────────┤ ┌─────┴──────────────┐ + │ callee answer │ │ │ │ Video / Audio Stream │ │ │ caller offer │ + ├────────────────────┤ │ └──────────────────────────┘ │ │ ├────────────────────┤ + │ callee ice ├──────────┘ └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ └───────────┤ caller ice │ + │ candidates │ │ candidates │ + └────────────────────┘ └────────────────────┘ + + + + ┌─────────────────────────────────────────────────────┐ + │ │░ + │ If connection is impossible (firewall), and a turn │░ + │server is available, connection could happen through │░ + │ a relay │░ + │ │░ + └─────────────────────────────────────────────────────┘░ + ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ + + + ┌───────────────┐ + │ Internet │ + └─┬─────────────┴ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + ┌─────────────────┐ │ ┌─────────────────┐ + │ │ │ ┌─────────────────────────┐ │ │ + │ ├───────────────────────────────────────┐│ │ │ │ │ + │ WebRtc │███████████████████████████████████████││ │ │ WebRtc │ + │ ├───────────────────────────────────────┘│ │ │ │ │ + │ │ ┌────────┴─────────────────┐ │ Relay │┌─────────────────────────────────────┤ │ +┌───────────────┴────┐ │ │ Video / Audio Stream │ │ ││█████████████████████████████████████│ ┌───────┴────────────┐ +│ callee answer ├────────────┘ └────────┬─────────────────┘ │ │└─────────────────────────────────────┴─────────┤ caller offer │ +├────────────────────┤ │ │ │ ├────────────────────┤ +│ callee ice │ │ │ │ │ caller ice │ +│ candidates │ └─────────────────────────┘ │ │ candidates │ +└────────────────────┘ │ └────────────────────┘ + │ + │ + │ + │ + │ + └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ \ No newline at end of file From eabb0bb41de3c148deb772a4e1287e8cc1f32949 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 15 Jun 2020 18:17:59 +0200 Subject: [PATCH 47/83] Restart capture when camera is back to available --- .../call/CameraEventsHandlerAdapter.kt | 46 +++++++++++++ .../call/SharedActiveCallViewModel.kt | 4 ++ .../riotx/features/call/VectorCallActivity.kt | 1 + .../features/call/VectorCallViewModel.kt | 15 +++++ .../call/WebRtcPeerConnectionManager.kt | 64 ++++++++++++++++--- 5 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt diff --git a/vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt b/vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt new file mode 100644 index 0000000000..48f4b9b27b --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import org.webrtc.CameraVideoCapturer +import timber.log.Timber + +open class CameraEventsHandlerAdapter : CameraVideoCapturer.CameraEventsHandler { + override fun onCameraError(p0: String?) { + Timber.v("## VOIP onCameraError $p0") + } + + override fun onCameraOpening(p0: String?) { + Timber.v("## VOIP onCameraOpening $p0") + } + + override fun onCameraDisconnected() { + Timber.v("## VOIP onCameraOpening") + } + + override fun onCameraFreezed(p0: String?) { + Timber.v("## VOIP onCameraFreezed $p0") + } + + override fun onFirstFrameAvailable() { + Timber.v("## VOIP onFirstFrameAvailable") + } + + override fun onCameraClosed() { + Timber.v("## VOIP onCameraClosed") + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt index efd8541e1c..e76b690a23 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt @@ -37,6 +37,10 @@ class SharedActiveCallViewModel @Inject constructor( override fun onCurrentCallChange(call: MxCall?) { activeCall.postValue(call) } + + override fun onCaptureStateChanged(captureInError: Boolean) { + // nop + } } init { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 5bb1d8d93c..256592656b 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -264,6 +264,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis if (callArgs.isVideoCall) { callVideoGroup.isVisible = true callInfoGroup.isVisible = false + pip_video_view.isVisible = !state.isVideoCaptureInError } else { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 809df48517..47888dfe76 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -41,6 +41,7 @@ data class VectorCallViewState( val isVideoCall: Boolean, val isAudioMuted: Boolean = false, val isVideoEnabled: Boolean = true, + val isVideoCaptureInError: Boolean = false, val soundDevice: CallAudioManager.SoundDevice = CallAudioManager.SoundDevice.PHONE, val otherUserMatrixItem: Async = Uninitialized, val callState: Async = Uninitialized @@ -83,12 +84,25 @@ class VectorCallViewModel @AssistedInject constructor( } } + private val currentCallListener = object : WebRtcPeerConnectionManager.CurrentCallListener { + override fun onCurrentCallChange(call: MxCall?) { + } + + override fun onCaptureStateChanged(captureInError: Boolean) { + setState { + copy(isVideoCaptureInError = captureInError) + } + } + } + init { autoReplyIfNeeded = args.autoAccept initialState.callId?.let { + webRtcPeerConnectionManager.addCurrentCallListener(currentCallListener) + session.callSignalingService().getCallWithId(it)?.let { mxCall -> this.call = mxCall mxCall.otherUserId @@ -109,6 +123,7 @@ class VectorCallViewModel @AssistedInject constructor( override fun onCleared() { // session.callService().removeCallListener(callServiceListener) + webRtcPeerConnectionManager.removeCurrentCallListener(currentCallListener) this.call?.removeListener(callStateListener) super.onCleared() } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index f40d02d2f8..9cbd0dfddd 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -17,6 +17,9 @@ package im.vector.riotx.features.call import android.content.Context +import android.hardware.camera2.CameraManager +import android.os.Build +import androidx.annotation.RequiresApi import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallState @@ -71,6 +74,7 @@ class WebRtcPeerConnectionManager @Inject constructor( interface CurrentCallListener { fun onCurrentCallChange(call: MxCall?) + fun onCaptureStateChanged(captureInError: Boolean) } private val currentCallsListeners = emptyList().toMutableList() @@ -118,6 +122,9 @@ class WebRtcPeerConnectionManager @Inject constructor( var remoteCandidateSource: ReplaySubject? = null var remoteIceCandidateDisposable: Disposable? = null + // We register an availability callback if we loose access to camera + var cameraAvailabilityCallback: CameraRestarter? = null + fun release() { remoteIceCandidateDisposable?.dispose() iceCandidateDisposable?.dispose() @@ -149,6 +156,14 @@ class WebRtcPeerConnectionManager @Inject constructor( private var videoCapturer: VideoCapturer? = null + var capturerIsInError = false + set(value) { + field = value + currentCallsListeners.forEach { + tryThis { it.onCaptureStateChanged(value) } + } + } + var localSurfaceRenderer: WeakReference? = null var remoteSurfaceRenderer: WeakReference? = null @@ -355,7 +370,27 @@ class WebRtcPeerConnectionManager @Inject constructor( ?.firstOrNull { cameraIterator.isFrontFacing(it) } ?: cameraIterator.deviceNames?.first() - val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + // TODO detect when no camera or no front camera + + val videoCapturer = cameraIterator.createCapturer(frontCamera, object : CameraEventsHandlerAdapter() { + override fun onFirstFrameAvailable() { + super.onFirstFrameAvailable() + capturerIsInError = false + } + + override fun onCameraClosed() { + // This could happen if you open the camera app in chat + // We then register in order to restart capture as soon as the camera is available again + Timber.v("## VOIP onCameraClosed") + this@WebRtcPeerConnectionManager.capturerIsInError = true + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val restarter = CameraRestarter(frontCamera ?: "", callContext.mxCall.callId) + callContext.cameraAvailabilityCallback = restarter + val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + cameraManager.registerAvailabilityCallback(restarter, null) + } + } + }) val videoSource = peerConnectionFactory!!.createVideoSource(videoCapturer.isScreencast) val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) @@ -374,14 +409,7 @@ class WebRtcPeerConnectionManager @Inject constructor( callContext.localVideoSource = videoSource callContext.localVideoTrack = localVideoTrack -// localViewRenderer?.let { localVideoTrack?.addSink(it) } localMediaStream?.addTrack(localVideoTrack) -// callContext.localMediaStream = localMediaStream -// remoteVideoTrack?.setEnabled(true) -// remoteVideoTrack?.let { -// it.setEnabled(true) -// it.addSink(remoteViewRenderer) -// } } } @@ -542,6 +570,12 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun endCall() { + currentCall?.cameraAvailabilityCallback?.let { cameraAvailabilityCallback -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + cameraManager.unregisterAvailabilityCallback(cameraAvailabilityCallback) + } + } currentCall?.mxCall?.hangUp() currentCall = null audioManager.stop() @@ -746,4 +780,18 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP StreamObserver onAddTrack") } } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + inner class CameraRestarter(val cameraId: String, val callId: String) : CameraManager.AvailabilityCallback() { + + override fun onCameraAvailable(cameraId: String) { + if (this.cameraId == cameraId && currentCall?.mxCall?.callId == callId) { + // re-start the capture + // TODO notify that video is enabled + videoCapturer?.startCapture(1280, 720, 30) + (context.getSystemService(Context.CAMERA_SERVICE) as? CameraManager) + ?.unregisterAvailabilityCallback(this) + } + } + } } From 8662797cf8aa2120517be09958c4f181efe09f69 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 15 Jun 2020 18:20:13 +0200 Subject: [PATCH 48/83] Restart capture after close for older android --- .../riotx/features/call/WebRtcPeerConnectionManager.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 9cbd0dfddd..1ebaf11a91 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -301,6 +301,11 @@ class WebRtcPeerConnectionManager @Inject constructor( } } else -> { + // Fallback for old android, try to restart capture when attached + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP && capturerIsInError && call.mxCall.isVideoCall) { + // try to restart capture? + videoCapturer?.startCapture(1280, 720, 30) + } // sink existing tracks (configuration change, e.g screen rotation) attachViewRenderersInternal() } From d8cf44fdc9019a27ab1a6f697f6d60926294fac7 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 16 Jun 2020 00:51:43 +0200 Subject: [PATCH 49/83] Simple cache of turn server response in memory cache in service + show active call banner only if connected --- .../api/session/call/CallSignalingService.kt | 2 +- .../{TurnServer.kt => TurnServerResponse.kt} | 2 +- .../matrix/android/api/session/call/VoipApi.kt | 2 +- .../session/call/DefaultCallSignalingService.kt | 16 ++++++++++++---- .../internal/session/call/GetTurnServerTask.kt | 6 +++--- .../features/call/SharedActiveCallViewModel.kt | 12 ++++++++++++ .../features/call/WebRtcPeerConnectionManager.kt | 16 ++++++++-------- .../home/room/detail/RoomDetailFragment.kt | 5 +++-- 8 files changed, 41 insertions(+), 20 deletions(-) rename matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/{TurnServer.kt => TurnServerResponse.kt} (96%) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallSignalingService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallSignalingService.kt index faf85b0ce5..ec0cb0e4e1 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallSignalingService.kt @@ -21,7 +21,7 @@ import im.vector.matrix.android.api.util.Cancelable interface CallSignalingService { - fun getTurnServer(callback: MatrixCallback): Cancelable + fun getTurnServer(callback: MatrixCallback): Cancelable /** * Create an outgoing call diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServerResponse.kt similarity index 96% rename from matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt rename to matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServerResponse.kt index fddd1c0c6a..bee854de33 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServerResponse.kt @@ -20,7 +20,7 @@ import com.squareup.moshi.Json import com.squareup.moshi.JsonClass @JsonClass(generateAdapter = true) -data class TurnServer( +data class TurnServerResponse( @Json(name = "username") val username: String?, @Json(name = "password") val password: String?, @Json(name = "uris") val uris: List?, diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt index 4a2b9cabef..489d946368 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt @@ -24,5 +24,5 @@ import retrofit2.http.GET internal interface VoipApi { @GET(NetworkConstants.URI_API_PREFIX_PATH_R0 + "voip/turnServer") - fun getTurnServer(): Call + fun getTurnServer(): Call } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt index fa486b7310..37ce1e62d6 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt @@ -21,7 +21,7 @@ import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallSignalingService import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.MxCall -import im.vector.matrix.android.api.session.call.TurnServer +import im.vector.matrix.android.api.session.call.TurnServerResponse import im.vector.matrix.android.api.session.events.model.Event import im.vector.matrix.android.api.session.events.model.EventType import im.vector.matrix.android.api.session.events.model.toModel @@ -30,6 +30,7 @@ import im.vector.matrix.android.api.session.room.model.call.CallCandidatesConten import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.matrix.android.api.util.Cancelable +import im.vector.matrix.android.api.util.NoOpCancellable import im.vector.matrix.android.internal.di.UserId import im.vector.matrix.android.internal.session.SessionScope import im.vector.matrix.android.internal.session.call.model.MxCallImpl @@ -54,11 +55,18 @@ internal class DefaultCallSignalingService @Inject constructor( private val activeCalls = mutableListOf() - override fun getTurnServer(callback: MatrixCallback): Cancelable { + private var cachedTurnServerResponse: TurnServerResponse? = null + + override fun getTurnServer(callback: MatrixCallback): Cancelable { + if (cachedTurnServerResponse != null) { + cachedTurnServerResponse?.let { callback.onSuccess(it) } + return NoOpCancellable + } return turnServerTask .configureWith(GetTurnServerTask.Params) { - this.callback = object : MatrixCallback { - override fun onSuccess(data: TurnServer) { + this.callback = object : MatrixCallback { + override fun onSuccess(data: TurnServerResponse) { + cachedTurnServerResponse = data callback.onSuccess(data) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt index d9e17d90eb..bac7a36c05 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt @@ -16,21 +16,21 @@ package im.vector.matrix.android.internal.session.call -import im.vector.matrix.android.api.session.call.TurnServer +import im.vector.matrix.android.api.session.call.TurnServerResponse import im.vector.matrix.android.api.session.call.VoipApi import im.vector.matrix.android.internal.network.executeRequest import im.vector.matrix.android.internal.task.Task import org.greenrobot.eventbus.EventBus import javax.inject.Inject -internal abstract class GetTurnServerTask : Task { +internal abstract class GetTurnServerTask : Task { object Params } internal class DefaultGetTurnServerTask @Inject constructor(private val voipAPI: VoipApi, private val eventBus: EventBus) : GetTurnServerTask() { - override suspend fun execute(params: Params): TurnServer { + override suspend fun execute(params: Params): TurnServerResponse { return executeRequest(eventBus) { apiCall = voipAPI.getTurnServer() } diff --git a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt index e76b690a23..22becc82f5 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt @@ -33,9 +33,20 @@ class SharedActiveCallViewModel @Inject constructor( val activeCall: MutableLiveData = MutableLiveData() + val callStateListener = object: MxCall.StateListener { + + override fun onStateUpdate(call: MxCall) { + if (activeCall.value?.callId == call.callId) { + activeCall.postValue(call) + } + } + } + private val listener = object : WebRtcPeerConnectionManager.CurrentCallListener { override fun onCurrentCallChange(call: MxCall?) { + activeCall.value?.removeListener(callStateListener) activeCall.postValue(call) + call?.addListener(callStateListener) } override fun onCaptureStateChanged(captureInError: Boolean) { @@ -49,6 +60,7 @@ class SharedActiveCallViewModel @Inject constructor( } override fun onCleared() { + activeCall.value?.removeListener(callStateListener) webRtcPeerConnectionManager.removeCurrentCallListener(listener) super.onCleared() } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 1ebaf11a91..ecbad9e269 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -26,7 +26,7 @@ import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCall -import im.vector.matrix.android.api.session.call.TurnServer +import im.vector.matrix.android.api.session.call.TurnServerResponse import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent @@ -213,9 +213,9 @@ class WebRtcPeerConnectionManager @Inject constructor( // attachViewRenderersInternal() } - private fun createPeerConnection(callContext: CallContext, turnServer: TurnServer?) { + private fun createPeerConnection(callContext: CallContext, turnServerResponse: TurnServerResponse?) { val iceServers = mutableListOf().apply { - turnServer?.let { server -> + turnServerResponse?.let { server -> server.uris?.forEach { uri -> add( PeerConnection @@ -250,9 +250,9 @@ class WebRtcPeerConnectionManager @Inject constructor( }, constraints) } - private fun getTurnServer(callback: ((TurnServer?) -> Unit)) { - sessionHolder.getActiveSession().callSignalingService().getTurnServer(object : MatrixCallback { - override fun onSuccess(data: TurnServer?) { + private fun getTurnServer(callback: ((TurnServerResponse?) -> Unit)) { + sessionHolder.getActiveSession().callSignalingService().getTurnServer(object : MatrixCallback { + override fun onSuccess(data: TurnServerResponse?) { callback(data) } @@ -313,10 +313,10 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - private fun internalAcceptIncomingCall(callContext: CallContext, turnServer: TurnServer?) { + private fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) { executor.execute { // 1) create peer connection - createPeerConnection(callContext, turnServer) + createPeerConnection(callContext, turnServerResponse) // create sdp using offer, and set remote description // the offer has beed stored when invite was received diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index 2bb3fc5d1d..c4c87ac2f5 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -63,6 +63,7 @@ import com.jakewharton.rxbinding3.widget.textChanges import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.permalinks.PermalinkFactory import im.vector.matrix.android.api.session.Session +import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.content.ContentAttachmentData import im.vector.matrix.android.api.session.events.model.Event import im.vector.matrix.android.api.session.events.model.toModel @@ -295,8 +296,8 @@ class RoomDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - // TODO delay a bit if it's a new call to let call activity launch before .. - activeCallView.isVisible = it != null + val hasActiveCall = it?.state == CallState.CONNECTED + activeCallView.isVisible = hasActiveCall }) roomDetailViewModel.selectSubscribe(this, RoomDetailViewState::tombstoneEventHandling, uniqueOnly("tombstoneEventHandling")) { From 843da1d48d3750f96d2a029bba00d636497ff5ea Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 17 Jun 2020 15:03:51 +0200 Subject: [PATCH 50/83] Incoming notification + ringing --- .../call/DefaultCallSignalingService.kt | 3 +- .../internal/session/call/model/MxCallImpl.kt | 3 + .../src/main/res/values/strings.xml | 1 + vector/src/main/AndroidManifest.xml | 1 + .../riotx/core/extensions/FragmentManager.kt | 7 +- .../riotx/core/platform/VectorBaseActivity.kt | 3 + .../riotx/core/services/CallRingPlayer.kt | 93 ++++++++ .../vector/riotx/core/services/CallService.kt | 200 +++++++++++++----- .../riotx/features/call/VectorCallActivity.kt | 194 ++++++++++++----- .../features/call/VectorCallViewModel.kt | 8 + .../call/WebRtcPeerConnectionManager.kt | 111 ++++++++-- .../call/service/CallHeadsUpActionReceiver.kt | 21 +- .../notifications/NotificationUtils.kt | 141 ++++++++---- 13 files changed, 607 insertions(+), 179 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/core/services/CallRingPlayer.kt diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt index 37ce1e62d6..a3bbcd6444 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt @@ -38,6 +38,7 @@ import im.vector.matrix.android.internal.session.room.send.LocalEchoEventFactory import im.vector.matrix.android.internal.session.room.send.RoomEventSender import im.vector.matrix.android.internal.task.TaskExecutor import im.vector.matrix.android.internal.task.configureWith +import timber.log.Timber import java.util.UUID import javax.inject.Inject @@ -100,8 +101,8 @@ internal class DefaultCallSignalingService @Inject constructor( override fun removeCallListener(listener: CallsListener) { callListeners.remove(listener) } - override fun getCallWithId(callId: String): MxCall? { + Timber.v("## VOIP getCallWithId $callId all calls ${activeCalls.map { it.callId }}") return activeCalls.find { it.callId == callId } } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt index 53c075579a..90b475d5b8 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt @@ -83,6 +83,7 @@ internal class MxCallImpl( override fun offerSdp(sdp: SessionDescription) { if (!isOutgoing) return + Timber.v("## VOIP offerSdp $callId") state = CallState.DIALING CallInviteContent( callId = callId, @@ -113,6 +114,7 @@ internal class MxCallImpl( } override fun hangUp() { + Timber.v("## VOIP hangup $callId") CallHangupContent( callId = callId ) @@ -122,6 +124,7 @@ internal class MxCallImpl( } override fun accept(sdp: SessionDescription) { + Timber.v("## VOIP accept $callId") if (isOutgoing) return state = CallState.ANSWERING CallAnswerContent( diff --git a/matrix-sdk-android/src/main/res/values/strings.xml b/matrix-sdk-android/src/main/res/values/strings.xml index bab4f6c622..a992b11c06 100644 --- a/matrix-sdk-android/src/main/res/values/strings.xml +++ b/matrix-sdk-android/src/main/res/values/strings.xml @@ -365,5 +365,6 @@ Accept Decline + Hang Up diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 6b0253c5fc..64299ff1ae 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -19,6 +19,7 @@ + diff --git a/vector/src/main/java/im/vector/riotx/core/extensions/FragmentManager.kt b/vector/src/main/java/im/vector/riotx/core/extensions/FragmentManager.kt index caf1bf90f8..83ac540830 100644 --- a/vector/src/main/java/im/vector/riotx/core/extensions/FragmentManager.kt +++ b/vector/src/main/java/im/vector/riotx/core/extensions/FragmentManager.kt @@ -17,9 +17,14 @@ package im.vector.riotx.core.extensions import androidx.fragment.app.FragmentTransaction +import im.vector.matrix.android.api.extensions.tryThis inline fun androidx.fragment.app.FragmentManager.commitTransactionNow(func: FragmentTransaction.() -> FragmentTransaction) { - beginTransaction().func().commitNow() + // Could throw and make the app crash + // e.g sharedActionViewModel.observe() + tryThis("Failed to commitTransactionNow") { + beginTransaction().func().commitNow() + } } inline fun androidx.fragment.app.FragmentManager.commitTransaction(func: FragmentTransaction.() -> FragmentTransaction) { diff --git a/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseActivity.kt b/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseActivity.kt index 270e67cf34..ba9e7320d2 100644 --- a/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseActivity.kt +++ b/vector/src/main/java/im/vector/riotx/core/platform/VectorBaseActivity.kt @@ -165,6 +165,7 @@ abstract class VectorBaseActivity : AppCompatActivity(), HasScreenInjector { } override fun onCreate(savedInstanceState: Bundle?) { + Timber.i("onCreate Activity ${this.javaClass.simpleName}") val vectorComponent = getVectorComponent() screenComponent = DaggerScreenComponent.factory().create(vectorComponent, this) val timeForInjection = measureTimeMillis { @@ -252,6 +253,7 @@ abstract class VectorBaseActivity : AppCompatActivity(), HasScreenInjector { override fun onDestroy() { super.onDestroy() + Timber.i("onDestroy Activity ${this.javaClass.simpleName}") unBinder?.unbind() unBinder = null @@ -279,6 +281,7 @@ abstract class VectorBaseActivity : AppCompatActivity(), HasScreenInjector { override fun onPause() { super.onPause() + Timber.i("onPause Activity ${this.javaClass.simpleName}") rageShake.stop() diff --git a/vector/src/main/java/im/vector/riotx/core/services/CallRingPlayer.kt b/vector/src/main/java/im/vector/riotx/core/services/CallRingPlayer.kt new file mode 100644 index 0000000000..f7f64d65f5 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/core/services/CallRingPlayer.kt @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.core.services + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioManager +import android.media.MediaPlayer +import android.os.Build +import im.vector.riotx.R +import timber.log.Timber + +class CallRingPlayer( + context: Context +) { + + private val applicationContext = context.applicationContext + + private var player: MediaPlayer? = null + + fun start() { + val audioManager: AudioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + player?.release() + player = createPlayer() + + // Check if sound is enabled + val ringerMode = audioManager.ringerMode + if (player != null && ringerMode == AudioManager.RINGER_MODE_NORMAL) { + try { + if (player?.isPlaying == false) { + player?.start() + Timber.v("## VOIP Starting ringing") + } else { + Timber.v("## VOIP already playing") + } + } catch (failure: Throwable) { + Timber.e(failure, "## VOIP Failed to start ringing") + player = null + } + } else { + Timber.v("## VOIP Can't play $player ode $ringerMode") + } + } + + fun stop() { + player?.release() + player = null + } + + private fun createPlayer(): MediaPlayer? { + try { + val mediaPlayer = MediaPlayer.create(applicationContext, R.raw.ring) + + mediaPlayer.setOnErrorListener(MediaPlayerErrorListener()) + mediaPlayer.isLooping = true + if (Build.VERSION.SDK_INT <= 21) { + @Suppress("DEPRECATION") + mediaPlayer.setAudioStreamType(AudioManager.STREAM_RING) + } else { + mediaPlayer.setAudioAttributes(AudioAttributes.Builder() + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION_SIGNALLING) + .build()) + } + return mediaPlayer + } catch (failure: Throwable) { + Timber.e(failure, "Failed to create Call ring player") + return null + } + } + + inner class MediaPlayerErrorListener : MediaPlayer.OnErrorListener { + override fun onError(mp: MediaPlayer, what: Int, extra: Int): Boolean { + Timber.w("onError($mp, $what, $extra") + player = null + return false + } + } +} diff --git a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt index 8337e56403..f10bbd908d 100644 --- a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt @@ -1,6 +1,6 @@ - /* * Copyright 2019 New Vector Ltd + * Copyright 2020 New Vector Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ package im.vector.riotx.core.services import android.content.Context import android.content.Intent import android.os.Binder -import android.text.TextUtils import androidx.core.content.ContextCompat import im.vector.riotx.core.extensions.vectorComponent import im.vector.riotx.features.call.WebRtcPeerConnectionManager @@ -38,7 +37,7 @@ class CallService : VectorService() { /** * call in progress (foreground notification) */ - private var mCallIdInProgress: String? = null +// private var mCallIdInProgress: String? = null private lateinit var notificationUtils: NotificationUtils private lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager @@ -46,15 +45,19 @@ class CallService : VectorService() { /** * incoming (foreground notification) */ - private var mIncomingCallId: String? = null +// private var mIncomingCallId: String? = null + + private var callRingPlayer: CallRingPlayer? = null override fun onCreate() { super.onCreate() notificationUtils = vectorComponent().notificationUtils() webRtcPeerConnectionManager = vectorComponent().webRtcPeerConnectionManager() + callRingPlayer = CallRingPlayer(applicationContext) } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + Timber.v("## VOIP onStartCommand $intent") if (intent == null) { // Service started again by the system. // TODO What do we do here? @@ -62,12 +65,34 @@ class CallService : VectorService() { } when (intent.action) { - ACTION_INCOMING_CALL -> displayIncomingCallNotification(intent) - ACTION_PENDING_CALL -> displayCallInProgressNotification(intent) - ACTION_NO_ACTIVE_CALL -> hideCallNotifications() - else -> + ACTION_INCOMING_RINGING_CALL -> { + callRingPlayer?.start() + displayIncomingCallNotification(intent) + } + ACTION_OUTGOING_RINGING_CALL -> { + callRingPlayer?.start() + displayOutgoingRingingCallNotification(intent) + } + ACTION_ONGOING_CALL -> { + callRingPlayer?.stop() + displayCallInProgressNotification(intent) + } + ACTION_NO_ACTIVE_CALL -> hideCallNotifications() + ACTION_CALL_CONNECTING -> { + // lower notification priority + displayCallInProgressNotification(intent) + // stop ringing + callRingPlayer?.stop() + } + ACTION_ONGOING_CALL_BG -> { + // there is an ongoing call but call activity is in background + displayCallOnGoingInBackground(intent) + } + else -> { // Should not happen + callRingPlayer?.stop() myStopSelf() + } } // We want the system to restore the service if killed @@ -87,29 +112,29 @@ class CallService : VectorService() { * @param callId the callId */ private fun displayIncomingCallNotification(intent: Intent) { - Timber.v("displayIncomingCallNotification") + Timber.v("## VOIP displayIncomingCallNotification $intent") // the incoming call in progress is already displayed - if (!TextUtils.isEmpty(mIncomingCallId)) { - Timber.v("displayIncomingCallNotification : the incoming call in progress is already displayed") - } else if (!TextUtils.isEmpty(mCallIdInProgress)) { - Timber.v("displayIncomingCallNotification : a 'call in progress' notification is displayed") - } else -// if (null == webRtcPeerConnectionManager.currentCall) - { - val callId = intent.getStringExtra(EXTRA_CALL_ID) +// if (!TextUtils.isEmpty(mIncomingCallId)) { +// Timber.v("displayIncomingCallNotification : the incoming call in progress is already displayed") +// } else if (!TextUtils.isEmpty(mCallIdInProgress)) { +// Timber.v("displayIncomingCallNotification : a 'call in progress' notification is displayed") +// } else +// // if (null == webRtcPeerConnectionManager.currentCall) +// { + val callId = intent.getStringExtra(EXTRA_CALL_ID) - Timber.v("displayIncomingCallNotification : display the dedicated notification") - val notification = notificationUtils.buildIncomingCallNotification( - intent.getBooleanExtra(EXTRA_IS_VIDEO, false), - intent.getStringExtra(EXTRA_ROOM_NAME) ?: "", - intent.getStringExtra(EXTRA_MATRIX_ID) ?: "", - callId ?: "") - startForeground(NOTIFICATION_ID, notification) + Timber.v("displayIncomingCallNotification : display the dedicated notification") + val notification = notificationUtils.buildIncomingCallNotification( + intent.getBooleanExtra(EXTRA_IS_VIDEO, false), + intent.getStringExtra(EXTRA_ROOM_NAME) ?: "", + intent.getStringExtra(EXTRA_ROOM_ID) ?: "", + callId ?: "") + startForeground(NOTIFICATION_ID, notification) - mIncomingCallId = callId +// mIncomingCallId = callId - // turn the screen on for 3 seconds + // turn the screen on for 3 seconds // if (Matrix.getInstance(VectorApp.getInstance())!!.pushManager.isScreenTurnedOn) { // try { // val pm = getSystemService(Context.POWER_SERVICE) as PowerManager @@ -123,16 +148,29 @@ class CallService : VectorService() { // } // // } - } +// } // else { // Timber.i("displayIncomingCallNotification : do not display the incoming call notification because there is a pending call") // } } + private fun displayOutgoingRingingCallNotification(intent: Intent) { + val callId = intent.getStringExtra(EXTRA_CALL_ID) + + Timber.v("displayOutgoingCallNotification : display the dedicated notification") + val notification = notificationUtils.buildOutgoingRingingCallNotification( + intent.getBooleanExtra(EXTRA_IS_VIDEO, false), + intent.getStringExtra(EXTRA_ROOM_NAME) ?: "", + intent.getStringExtra(EXTRA_ROOM_ID) ?: "", + callId ?: "") + startForeground(NOTIFICATION_ID, notification) + } + /** * Display a call in progress notification. */ private fun displayCallInProgressNotification(intent: Intent) { + Timber.v("## VOIP displayCallInProgressNotification") val callId = intent.getStringExtra(EXTRA_CALL_ID) ?: "" val notification = notificationUtils.buildPendingCallNotification( @@ -144,7 +182,27 @@ class CallService : VectorService() { startForeground(NOTIFICATION_ID, notification) - mCallIdInProgress = callId + // mCallIdInProgress = callId + } + + /** + * Display a call in progress notification. + */ + private fun displayCallOnGoingInBackground(intent: Intent) { + Timber.v("## VOIP displayCallInProgressNotification") + val callId = intent.getStringExtra(EXTRA_CALL_ID) ?: "" + + val notification = notificationUtils.buildPendingCallNotification( + isVideo = intent.getBooleanExtra(EXTRA_IS_VIDEO, false), + roomName = intent.getStringExtra(EXTRA_ROOM_NAME) ?: "", + roomId = intent.getStringExtra(EXTRA_ROOM_ID) ?: "", + matrixId = intent.getStringExtra(EXTRA_MATRIX_ID) ?: "", + callId = callId, + fromBg = true) + + startForeground(NOTIFICATION_ID, notification) + + // mCallIdInProgress = callId } /** @@ -159,6 +217,11 @@ class CallService : VectorService() { myStopSelf() } + override fun onDestroy() { + super.onDestroy() + callRingPlayer?.stop() + } + fun addConnection(callConnection: CallConnection) { connections[callConnection.callId] = callConnection } @@ -166,10 +229,14 @@ class CallService : VectorService() { companion object { private const val NOTIFICATION_ID = 6480 - private const val ACTION_INCOMING_CALL = "im.vector.riotx.core.services.CallService.INCOMING_CALL" - private const val ACTION_PENDING_CALL = "im.vector.riotx.core.services.CallService.PENDING_CALL" + private const val ACTION_INCOMING_RINGING_CALL = "im.vector.riotx.core.services.CallService.ACTION_INCOMING_RINGING_CALL" + private const val ACTION_OUTGOING_RINGING_CALL = "im.vector.riotx.core.services.CallService.ACTION_OUTGOING_RINGING_CALL" + private const val ACTION_CALL_CONNECTING = "im.vector.riotx.core.services.CallService.ACTION_CALL_CONNECTING" + private const val ACTION_ONGOING_CALL = "im.vector.riotx.core.services.CallService.ACTION_ONGOING_CALL" + private const val ACTION_ONGOING_CALL_BG = "im.vector.riotx.core.services.CallService.ACTION_ONGOING_CALL_BG" private const val ACTION_NO_ACTIVE_CALL = "im.vector.riotx.core.services.CallService.NO_ACTIVE_CALL" -// private const val ACTION_ON_ACTIVE_CALL = "im.vector.riotx.core.services.CallService.ACTIVE_CALL" +// private const val ACTION_ACTIVITY_VISIBLE = "im.vector.riotx.core.services.CallService.ACTION_ACTIVITY_VISIBLE" +// private const val ACTION_STOP_RINGING = "im.vector.riotx.core.services.CallService.ACTION_STOP_RINGING" private const val EXTRA_IS_VIDEO = "EXTRA_IS_VIDEO" private const val EXTRA_ROOM_NAME = "EXTRA_ROOM_NAME" @@ -177,15 +244,15 @@ class CallService : VectorService() { private const val EXTRA_MATRIX_ID = "EXTRA_MATRIX_ID" private const val EXTRA_CALL_ID = "EXTRA_CALL_ID" - fun onIncomingCall(context: Context, - isVideo: Boolean, - roomName: String, - roomId: String, - matrixId: String, - callId: String) { + fun onIncomingCallRinging(context: Context, + isVideo: Boolean, + roomName: String, + roomId: String, + matrixId: String, + callId: String) { val intent = Intent(context, CallService::class.java) .apply { - action = ACTION_INCOMING_CALL + action = ACTION_INCOMING_RINGING_CALL putExtra(EXTRA_IS_VIDEO, isVideo) putExtra(EXTRA_ROOM_NAME, roomName) putExtra(EXTRA_ROOM_ID, roomId) @@ -196,24 +263,43 @@ class CallService : VectorService() { ContextCompat.startForegroundService(context, intent) } -// fun onActiveCall(context: Context, -// isVideo: Boolean, -// roomName: String, -// roomId: String, -// matrixId: String, -// callId: String) { -// val intent = Intent(context, CallService::class.java) -// .apply { -// action = ACTION_ON_ACTIVE_CALL -// putExtra(EXTRA_IS_VIDEO, isVideo) -// putExtra(EXTRA_ROOM_NAME, roomName) -// putExtra(EXTRA_ROOM_ID, roomId) -// putExtra(EXTRA_MATRIX_ID, matrixId) -// putExtra(EXTRA_CALL_ID, callId) -// } -// -// ContextCompat.startForegroundService(context, intent) -// } + fun onOnGoingCallBackground(context: Context, + isVideo: Boolean, + roomName: String, + roomId: String, + matrixId: String, + callId: String) { + val intent = Intent(context, CallService::class.java) + .apply { + action = ACTION_ONGOING_CALL_BG + putExtra(EXTRA_IS_VIDEO, isVideo) + putExtra(EXTRA_ROOM_NAME, roomName) + putExtra(EXTRA_ROOM_ID, roomId) + putExtra(EXTRA_MATRIX_ID, matrixId) + putExtra(EXTRA_CALL_ID, callId) + } + + ContextCompat.startForegroundService(context, intent) + } + + fun onOutgoingCallRinging(context: Context, + isVideo: Boolean, + roomName: String, + roomId: String, + matrixId: String, + callId: String) { + val intent = Intent(context, CallService::class.java) + .apply { + action = ACTION_OUTGOING_RINGING_CALL + putExtra(EXTRA_IS_VIDEO, isVideo) + putExtra(EXTRA_ROOM_NAME, roomName) + putExtra(EXTRA_ROOM_ID, roomId) + putExtra(EXTRA_MATRIX_ID, matrixId) + putExtra(EXTRA_CALL_ID, callId) + } + + ContextCompat.startForegroundService(context, intent) + } fun onPendingCall(context: Context, isVideo: Boolean, @@ -223,7 +309,7 @@ class CallService : VectorService() { callId: String) { val intent = Intent(context, CallService::class.java) .apply { - action = ACTION_PENDING_CALL + action = ACTION_ONGOING_CALL putExtra(EXTRA_IS_VIDEO, isVideo) putExtra(EXTRA_ROOM_NAME, roomName) putExtra(EXTRA_ROOM_ID, roomId) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 256592656b..a6aede7f48 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -20,6 +20,7 @@ package im.vector.riotx.features.call import android.app.KeyguardManager import android.content.Context import android.content.Intent +import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP import android.os.Build import android.os.Bundle import android.os.Parcelable @@ -31,6 +32,7 @@ import androidx.core.view.isInvisible import androidx.core.view.isVisible import androidx.core.view.updatePadding import butterknife.BindView +import com.airbnb.mvrx.Fail import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel import com.jakewharton.rxbinding3.view.clicks @@ -46,10 +48,11 @@ import im.vector.riotx.core.utils.PERMISSIONS_FOR_VIDEO_IP_CALL import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions import im.vector.riotx.features.home.AvatarRenderer +import im.vector.riotx.features.home.room.detail.RoomDetailActivity +import im.vector.riotx.features.home.room.detail.RoomDetailArgs import io.reactivex.android.schedulers.AndroidSchedulers import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* -import kotlinx.android.synthetic.main.fragment_attachments_preview.* import org.webrtc.EglBase import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer @@ -101,19 +104,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis override fun doBeforeSetContentView() { // Set window styles for fullscreen-window size. Needs to be done before adding content. requestWindowFeature(Window.FEATURE_NO_TITLE) - window.addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN or WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { - setTurnScreenOn(true) - setShowWhenLocked(true) - getSystemService(KeyguardManager::class.java)?.requestDismissKeyguard(this, null) - } else { - @Suppress("DEPRECATION") - window.addFlags( - WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON - or WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED - or WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD - ) - } hideSystemUI() setContentView(R.layout.activity_call) @@ -179,31 +169,22 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis if (intent.hasExtra(MvRx.KEY_ARG)) { callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! } else { + Timber.e("## VOIP missing callArgs for VectorCall Activity") + CallService.onNoActiveCall(this) finish() } + Timber.v("## VOIP EXTRA_MODE is ${intent.getStringExtra(EXTRA_MODE)}") + if (intent.getStringExtra(EXTRA_MODE) == INCOMING_RINGING) { + turnScreenOnAndKeyguardOff() + } + constraintLayout.clicks() .throttleFirst(300, TimeUnit.MILLISECONDS) .observeOn(AndroidSchedulers.mainThread()) .subscribe { toggleUiSystemVisibility() } .disposeOnDestroy() - if (isFirstCreation()) { - // Reduce priority of notification as the activity is on screen - CallService.onPendingCall( - this, - callArgs.isVideoCall, - callArgs.participantUserId, - callArgs.roomId, - "", - callArgs.callId ?: "" - ) - } - - rootEglBase = EglUtils.rootEglBase ?: return Unit.also { - finish() - } - configureCallViews() callViewModel.subscribe(this) { @@ -229,8 +210,68 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } } + override fun onDestroy() { + super.onDestroy() + peerConnectionManager.detachRenderers() + turnScreenOffAndKeyguardOn() + } + +// override fun onResume() { +// super.onResume() +// } +// +// override fun onStop() { +// super.onStop() +// when(callViewModel.call?.state) { +// CallState.DIALING -> { +// CallService.onIncomingCall( +// this, +// callArgs.isVideoCall, +// callArgs.participantUserId, +// callArgs.roomId, +// "", +// callArgs.callId ?: "" +// ) +// } +// CallState.LOCAL_RINGING -> { +// CallService.onIncomingCall( +// this, +// callArgs.isVideoCall, +// callArgs.participantUserId, +// callArgs.roomId, +// "", +// callArgs.callId ?: "" +// ) +// } +// CallState.ANSWERING, +// CallState.CONNECTING, +// CallState.CONNECTED -> { +// CallService.onPendingCall( +// this, +// callArgs.isVideoCall, +// callArgs.participantUserId, +// callArgs.roomId, +// "", +// callArgs.callId ?: "" +// ) +// } +// CallState.TERMINATED , +// CallState.IDLE , +// null -> { +// +// } +// } +// } + private fun renderState(state: VectorCallViewState) { Timber.v("## VOIP renderState call $state") + if (state.callState is Fail) { + // be sure to clear notification + CallService.onNoActiveCall(this) + finish() + return + } + callControlsView.updateForState(state) when (state.callState.invoke()) { CallState.IDLE, @@ -271,6 +312,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis configureCallInfo(state) callStatusText.text = null } + // ensure all attached? + peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null) } CallState.TERMINATED -> { finish() @@ -290,20 +333,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private fun configureCallViews() { callControlsView.interactionListener = this -// if (callArgs.isVideoCall) { -// iv_call_speaker.isVisible = false -// iv_call_flip_camera.isVisible = true -// iv_call_videocam_off.isVisible = true -// } else { -// iv_call_speaker.isVisible = true -// iv_call_flip_camera.isVisible = false -// iv_call_videocam_off.isVisible = false -// } -// -// iv_end_call.setOnClickListener { -// callViewModel.handle(VectorCallViewActions.EndCall) -// finish() -// } } override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { @@ -315,7 +344,12 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } } - private fun start(): Boolean { + private fun start() { + rootEglBase = EglUtils.rootEglBase ?: return Unit.also { + Timber.v("## VOIP rootEglBase is null") + finish() + } + // Init Picture in Picture renderer pipRenderer.init(rootEglBase!!.eglBaseContext, null) pipRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) @@ -330,16 +364,31 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() }) - return false } - override fun onPause() { - peerConnectionManager.detachRenderers() - super.onPause() - } +// override fun onResume() { +// super.onResume() +// withState(callViewModel) { +// if(it.callState.invoke() == CallState.CONNECTED) { +// peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) +// } +// } +// } +// override fun onPause() { +// peerConnectionManager.detachRenderers() +// super.onPause() +// } private fun handleViewEvents(event: VectorCallViewEvents?) { - Timber.v("handleViewEvents $event") + Timber.v("## VOIP handleViewEvents $event") + when (event) { + VectorCallViewEvents.DismissNoCall -> { + CallService.onNoActiveCall(this) + finish() + } + null -> { + } + } // when (event) { // is VectorCallViewEvents.CallAnswered -> { // } @@ -360,6 +409,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis fun newIntent(context: Context, mxCall: MxCallDetail): Intent { return Intent(context, VectorCallActivity::class.java).apply { + // what could be the best flags? flags = Intent.FLAG_ACTIVITY_NEW_TASK putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.callId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall, false)) putExtra(EXTRA_MODE, OUTGOING_CREATED) @@ -375,7 +425,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis accept: Boolean, mode: String?): Intent { return Intent(context, VectorCallActivity::class.java).apply { - flags = Intent.FLAG_ACTIVITY_NEW_TASK + // what could be the best flags? + flags = Intent.FLAG_ACTIVITY_CLEAR_TOP putExtra(MvRx.KEY_ARG, CallArgs(roomId, callId, otherUserId, isIncomingCall, isVideoCall, accept)) putExtra(EXTRA_MODE, mode) } @@ -403,11 +454,48 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } override fun returnToChat() { - // TODO, what if the room is not in backstack?? + val args = RoomDetailArgs(callArgs.roomId) + val intent = RoomDetailActivity.newIntent(this, args).apply { + flags = FLAG_ACTIVITY_CLEAR_TOP + } + startActivity(intent) + // is it needed? finish() } override fun didTapMore() { CallControlsBottomSheet().show(supportFragmentManager, "Controls") } + + // Needed to let you answer call when phone is locked + private fun turnScreenOnAndKeyguardOff() { + Timber.v("## VOIP turnScreenOnAndKeyguardOff") + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + setShowWhenLocked(true) + setTurnScreenOn(true) + } else { + window.addFlags( + WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON + or WindowManager.LayoutParams.FLAG_ALLOW_LOCK_WHILE_SCREEN_ON + ) + } + + with(getSystemService(Context.KEYGUARD_SERVICE) as KeyguardManager) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + requestDismissKeyguard(this@VectorCallActivity, null) + } + } + } + + private fun turnScreenOffAndKeyguardOn() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + setShowWhenLocked(false) + setTurnScreenOn(false) + } else { + window.clearFlags( + WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON + or WindowManager.LayoutParams.FLAG_ALLOW_LOCK_WHILE_SCREEN_ON + ) + } + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 47888dfe76..b8af552dc4 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -17,6 +17,7 @@ package im.vector.riotx.features.call import com.airbnb.mvrx.Async +import com.airbnb.mvrx.Fail import com.airbnb.mvrx.Loading import com.airbnb.mvrx.MvRxState import com.airbnb.mvrx.MvRxViewModelFactory @@ -58,6 +59,7 @@ sealed class VectorCallViewActions : VectorViewModelAction { sealed class VectorCallViewEvents : VectorViewEvents { + object DismissNoCall : VectorCallViewEvents() // data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() // data class CallHangup(val content: CallHangupContent) : VectorCallViewEvents() // object CallAccepted : VectorCallViewEvents() @@ -117,6 +119,12 @@ class VectorCallViewModel @AssistedInject constructor( soundDevice = webRtcPeerConnectionManager.audioManager.getCurrentSoundDevice() ) } + } ?: run { + setState { + copy( + callState = Fail(IllegalArgumentException("No call")) + ) + } } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index ecbad9e269..0c7f2be691 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -266,6 +266,23 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer") this.localSurfaceRenderer = WeakReference(localViewRenderer) this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + + // The call is going to resume from background, we can reduce notif + currentCall?.mxCall + ?.takeIf { it.state == CallState.CONNECTING || it.state == CallState.CONNECTED } + ?.let { mxCall -> + val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() + ?: mxCall.roomId + // Start background service with notification + CallService.onPendingCall( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", + callId = mxCall.callId) + } + getTurnServer { turnServer -> val call = currentCall ?: return@getTurnServer when (mode) { @@ -314,6 +331,19 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) { + val mxCall = callContext.mxCall + // Update service state + + val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() + ?: mxCall.roomId + CallService.onPendingCall( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", + callId = mxCall.callId + ) executor.execute { // 1) create peer connection createPeerConnection(callContext, turnServerResponse) @@ -435,7 +465,8 @@ class WebRtcPeerConnectionManager @Inject constructor( fun acceptIncomingCall() { Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") - if (currentCall?.mxCall?.state == CallState.LOCAL_RINGING) { + val mxCall = currentCall?.mxCall + if (mxCall?.state == CallState.LOCAL_RINGING) { getTurnServer { turnServer -> internalAcceptIncomingCall(currentCall!!, turnServer) } @@ -443,6 +474,24 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun detachRenderers() { + // The call is going to continue in background, so ensure notification is visible + currentCall?.mxCall + ?.takeIf { it.state == CallState.CONNECTING || it.state == CallState.CONNECTED } + ?.let { mxCall -> + // Start background service with notification + + val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() + ?: mxCall.otherUserId + CallService.onOnGoingCallBackground( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", + callId = mxCall.callId + ) + } + Timber.v("## VOIP detachRenderers") // currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } localSurfaceRenderer?.get()?.let { @@ -496,6 +545,16 @@ class WebRtcPeerConnectionManager @Inject constructor( audioManager.startForCall(createdCall) currentCall = callContext + val name = sessionHolder.getSafeActiveSession()?.getUser(createdCall.otherUserId)?.getBestName() + ?: createdCall.otherUserId + CallService.onOutgoingCallRinging( + context = context, + isVideo = createdCall.isVideoCall, + roomName = name, + roomId = createdCall.roomId, + matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", + callId = createdCall.callId) + executor.execute { callContext.remoteCandidateSource = ReplaySubject.create() } @@ -524,9 +583,8 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP onCallInviteReceived callId ${mxCall.callId}") // TODO What if a call is currently active? if (currentCall != null) { - Timber.w("## VOIP TODO: Automatically reject incoming call?") - mxCall.hangUp() - audioManager.stop() + Timber.w("## VOIP receiving incoming call while already in call?") + // Just ignore, maybe we could answer from other session? return } @@ -537,12 +595,17 @@ class WebRtcPeerConnectionManager @Inject constructor( callContext.remoteCandidateSource = ReplaySubject.create() } - CallService.onIncomingCall(context, - mxCall.isVideoCall, - mxCall.otherUserId, - mxCall.roomId, - sessionHolder.getSafeActiveSession()?.myUserId ?: "", - mxCall.callId) + // Start background service with notification + val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() + ?: mxCall.otherUserId + CallService.onIncomingCallRinging( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", + callId = mxCall.callId + ) callContext.offerSdp = callInviteContent.offer } @@ -575,12 +638,19 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun endCall() { + // Update service state + CallService.onNoActiveCall(context) + // close tracks ASAP + currentCall?.localVideoTrack?.setEnabled(false) + currentCall?.localVideoTrack?.setEnabled(false) + currentCall?.cameraAvailabilityCallback?.let { cameraAvailabilityCallback -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager cameraManager.unregisterAvailabilityCallback(cameraAvailabilityCallback) } } + currentCall?.mxCall?.hangUp() currentCall = null audioManager.stop() @@ -592,6 +662,18 @@ class WebRtcPeerConnectionManager @Inject constructor( if (call.mxCall.callId != callAnswerContent.callId) return Unit.also { Timber.w("onCallAnswerReceived for non active call? ${callAnswerContent.callId}") } + val mxCall = call.mxCall + // Update service state + val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() + ?: mxCall.otherUserId + CallService.onPendingCall( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", + callId = mxCall.callId + ) executor.execute { Timber.v("## VOIP onCallAnswerReceived ${callAnswerContent.callId}") val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) @@ -626,7 +708,8 @@ class WebRtcPeerConnectionManager @Inject constructor( * One or more of the ICE transports on the connection is in the "failed" state. */ PeerConnection.PeerConnectionState.FAILED -> { - endCall() + // This can be temporary, e.g when other ice not yet received... + // callContext.mxCall.state = CallState.ERROR } /** * At least one of the connection's ICE transports (RTCIceTransports or RTCDtlsTransports) are in the "new" state, @@ -711,7 +794,9 @@ class WebRtcPeerConnectionManager @Inject constructor( * It is, however, possible that the ICE agent did find compatible connections for some components. */ PeerConnection.IceConnectionState.FAILED -> { - callContext.mxCall.hangUp() + // I should not hangup here.. + // because new candidates could arrive + // callContext.mxCall.hangUp() } /** * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. @@ -742,7 +827,7 @@ class WebRtcPeerConnectionManager @Inject constructor( remoteVideoTrack.setEnabled(true) callContext.remoteVideoTrack = remoteVideoTrack // sink to renderer if attached - remoteSurfaceRenderer?.get().let { remoteVideoTrack.addSink(it) } + remoteSurfaceRenderer?.get()?.let { remoteVideoTrack.addSink(it) } } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt index 6b38a272a2..199dcd3b14 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpActionReceiver.kt @@ -23,9 +23,15 @@ import im.vector.riotx.core.di.HasVectorInjector import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.notifications.NotificationUtils import im.vector.riotx.features.settings.VectorLocale.context +import timber.log.Timber class CallHeadsUpActionReceiver : BroadcastReceiver() { + companion object { + const val EXTRA_CALL_ACTION_KEY = "EXTRA_CALL_ACTION_KEY" + const val CALL_ACTION_REJECT = 0 + } + private lateinit var peerConnectionManager: WebRtcPeerConnectionManager private lateinit var notificationUtils: NotificationUtils @@ -38,10 +44,9 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { } override fun onReceive(context: Context, intent: Intent?) { -// when (intent?.getIntExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, 0)) { -// CallHeadsUpService.CALL_ACTION_ANSWER -> onCallAnswerClicked(context) -// CallHeadsUpService.CALL_ACTION_REJECT -> onCallRejectClicked() -// } + when (intent?.getIntExtra(EXTRA_CALL_ACTION_KEY, 0)) { + CALL_ACTION_REJECT -> onCallRejectClicked() + } // Not sure why this should be needed // val it = Intent(Intent.ACTION_CLOSE_SYSTEM_DIALOGS) @@ -51,10 +56,10 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { // context.stopService(Intent(context, CallHeadsUpService::class.java)) } -// private fun onCallRejectClicked() { -// Timber.d("onCallRejectClicked") -// peerConnectionManager.endCall() -// } + private fun onCallRejectClicked() { + Timber.d("onCallRejectClicked") + peerConnectionManager.endCall() + } // private fun onCallAnswerClicked(context: Context) { // Timber.d("onCallAnswerClicked") diff --git a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt index 72471ccf1d..88a19fff89 100755 --- a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt +++ b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt @@ -283,31 +283,29 @@ class NotificationUtils @Inject constructor(private val context: Context, .setSmallIcon(R.drawable.incoming_call_notification_transparent) .setCategory(NotificationCompat.CATEGORY_CALL) .setLights(accentColor, 500, 500) + .setOngoing(true) // Compat: Display the incoming call notification on the lock screen builder.priority = NotificationCompat.PRIORITY_HIGH - // clear the activity stack to home activity - // val intent = Intent(context, HomeActivity::class.java) - // .setFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_NEW_TASK) - // TODO .putExtra(VectorHomeActivity.EXTRA_CALL_SESSION_ID, matrixId) - // TODO .putExtra(VectorHomeActivity.EXTRA_CALL_ID, callId) - - // Recreate the back stack -// val stackBuilder = TaskStackBuilder.create(context) -// .addParentStack(HomeActivity::class.java) -// .addNextIntent(intent) - - // android 4.3 issue - // use a generator for the private requestCode. - // When using 0, the intent is not created/launched when the user taps on the notification. // val requestId = Random.nextInt(1000) // val pendingIntent = stackBuilder.getPendingIntent(requestId, PendingIntent.FLAG_UPDATE_CURRENT) - val contentPendingIntent = TaskStackBuilder.create(context) - .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) - .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, otherUserId, true, isVideo, false, VectorCallActivity.INCOMING_RINGING)) - .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) + + val contentIntent = VectorCallActivity.newIntent( + context, callId, roomId, otherUserId, true, isVideo, + false, VectorCallActivity.INCOMING_RINGING + ).apply { + flags = Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP + data = Uri.parse("foobar://$callId") + } + val contentPendingIntent = PendingIntent.getActivity(context, System.currentTimeMillis().toInt(), contentIntent, 0) + +// val contentPendingIntent = TaskStackBuilder.create(context) +// .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) +// .addNextIntent(RoomDetailActivity.newIntent(context, RoomDetailArgs(roomId = roomId)) +// .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, otherUserId, true, isVideo, false, VectorCallActivity.INCOMING_RINGING)) +// .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) val answerCallPendingIntent = TaskStackBuilder.create(context) .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) @@ -321,7 +319,12 @@ class NotificationUtils @Inject constructor(private val context: Context, // putExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, CallHeadsUpService.CALL_ACTION_REJECT) } // val answerCallPendingIntent = PendingIntent.getBroadcast(context, requestId, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) - val rejectCallPendingIntent = PendingIntent.getBroadcast(context, requestId + 1, rejectCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) + val rejectCallPendingIntent = PendingIntent.getBroadcast( + context, + requestId + 1, + rejectCallActionReceiver, + PendingIntent.FLAG_UPDATE_CURRENT + ) builder.addAction( NotificationCompat.Action( @@ -340,6 +343,54 @@ class NotificationUtils @Inject constructor(private val context: Context, rejectCallPendingIntent) ) + builder.setFullScreenIntent(contentPendingIntent, true) + + return builder.build() + } + + fun buildOutgoingRingingCallNotification(isVideo: Boolean, + otherUserId: String, + roomId: String, + callId: String): Notification { + val accentColor = ContextCompat.getColor(context, R.color.notification_accent_color) + + val builder = NotificationCompat.Builder(context, SILENT_NOTIFICATION_CHANNEL_ID) + .setContentTitle(ensureTitleNotEmpty(otherUserId)) + .apply { + setContentText(stringProvider.getString(R.string.call_ring)) + } + .setSmallIcon(R.drawable.incoming_call_notification_transparent) + .setCategory(NotificationCompat.CATEGORY_CALL) + .setLights(accentColor, 500, 500) + .setOngoing(true) + + val requestId = Random.nextInt(1000) + + val contentIntent = VectorCallActivity.newIntent( + context, callId, roomId, otherUserId, true, isVideo, + false, null).apply { + flags = Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP + data = Uri.parse("foobar://$callId") + } + val contentPendingIntent = PendingIntent.getActivity(context, System.currentTimeMillis().toInt(), contentIntent, 0) + + val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { + putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ACTION_KEY, CallHeadsUpActionReceiver.CALL_ACTION_REJECT) + } + + val rejectCallPendingIntent = PendingIntent.getBroadcast( + context, + requestId + 1, + rejectCallActionReceiver, + PendingIntent.FLAG_UPDATE_CURRENT + ) + + builder.addAction( + NotificationCompat.Action( + IconCompat.createWithResource(context, R.drawable.ic_call_end).setTint(ContextCompat.getColor(context, R.color.riotx_notice)), + context.getString(R.string.call_notification_hangup), + rejectCallPendingIntent) + ) builder.setContentIntent(contentPendingIntent) return builder.build() @@ -360,8 +411,8 @@ class NotificationUtils @Inject constructor(private val context: Context, roomName: String, roomId: String, matrixId: String, - callId: String): Notification { - val builder = NotificationCompat.Builder(context, CALL_NOTIFICATION_CHANNEL_ID) + callId: String, fromBg: Boolean = false): Notification { + val builder = NotificationCompat.Builder(context, if (fromBg) CALL_NOTIFICATION_CHANNEL_ID else SILENT_NOTIFICATION_CHANNEL_ID) .setContentTitle(ensureTitleNotEmpty(roomName)) .apply { if (isVideo) { @@ -373,7 +424,29 @@ class NotificationUtils @Inject constructor(private val context: Context, .setSmallIcon(R.drawable.incoming_call_notification_transparent) .setCategory(NotificationCompat.CATEGORY_CALL) - builder.priority = NotificationCompat.PRIORITY_DEFAULT + if (fromBg) { + builder.priority = NotificationCompat.PRIORITY_LOW + builder.setOngoing(true) + } + + val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { + data = Uri.parse("mxcall://end?$callId") + putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ACTION_KEY, CallHeadsUpActionReceiver.CALL_ACTION_REJECT) + } + + val rejectCallPendingIntent = PendingIntent.getBroadcast( + context, + System.currentTimeMillis().toInt(), + rejectCallActionReceiver, + PendingIntent.FLAG_UPDATE_CURRENT + ) + + builder.addAction( + NotificationCompat.Action( + IconCompat.createWithResource(context, R.drawable.ic_call_end).setTint(ContextCompat.getColor(context, R.color.riotx_notice)), + context.getString(R.string.call_notification_hangup), + rejectCallPendingIntent) + ) val contentPendingIntent = TaskStackBuilder.create(context) .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) @@ -381,32 +454,8 @@ class NotificationUtils @Inject constructor(private val context: Context, .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, "otherUserId", true, isVideo, false, null)) .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) - // android 4.3 issue - // use a generator for the private requestCode. - // When using 0, the intent is not created/launched when the user taps on the notification. builder.setContentIntent(contentPendingIntent) - /* TODO - // Build the pending intent for when the notification is clicked - val roomIntent = Intent(context, VectorRoomActivity::class.java) - .putExtra(VectorRoomActivity.EXTRA_ROOM_ID, roomId) - .putExtra(VectorRoomActivity.EXTRA_MATRIX_ID, matrixId) - .putExtra(VectorRoomActivity.EXTRA_START_CALL_ID, callId) - - // Recreate the back stack - val stackBuilder = TaskStackBuilder.create(context) - .addParentStack(VectorRoomActivity::class.java) - .addNextIntent(roomIntent) - - // android 4.3 issue - // use a generator for the private requestCode. - // When using 0, the intent is not created/launched when the user taps on the notification. - // - val pendingIntent = stackBuilder.getPendingIntent(Random().nextInt(1000), PendingIntent.FLAG_UPDATE_CURRENT) - - builder.setContentIntent(pendingIntent) - */ - return builder.build() } From c85ba51274251fb423082e9dbad719805e3a7957 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 17 Jun 2020 15:23:03 +0200 Subject: [PATCH 51/83] Basic discard of old call events --- .../internal/session/call/CallEventsObserverTask.kt | 7 +++++++ .../riotx/features/call/WebRtcPeerConnectionManager.kt | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt index b8879739f4..950381ef6b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt @@ -52,10 +52,17 @@ internal class DefaultCallEventsObserverTask @Inject constructor( } private fun update(realm: Realm, events: List, userId: String) { + val now = System.currentTimeMillis() events.forEach { event -> event.roomId ?: return@forEach Unit.also { Timber.w("Event with no room id ${event.eventId}") } + val age = now - (event.ageLocalTs ?: now) + if (age > 40_000) { + // To old to ring? + return@forEach + } + event.ageLocalTs decryptIfNeeded(event) if (EventType.isCallEvent(event.getClearType())) { callService.onCallEvent(event) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 0c7f2be691..d75184ecca 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -581,7 +581,7 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { Timber.v("## VOIP onCallInviteReceived callId ${mxCall.callId}") - // TODO What if a call is currently active? + // to simplify we only treat one call at a time, and ignore others if (currentCall != null) { Timber.w("## VOIP receiving incoming call while already in call?") // Just ignore, maybe we could answer from other session? From fd3f591541fe6c91dd394d149b70b3bee34a8be3 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 17 Jun 2020 16:37:17 +0200 Subject: [PATCH 52/83] Show error on connecting timeout + refactoring --- .../android/api/session/call/CallState.kt | 25 ++-- .../call/DefaultCallSignalingService.kt | 22 +-- .../internal/session/call/model/MxCallImpl.kt | 12 +- .../riotx/features/call/CallControlsView.kt | 29 ++-- .../riotx/features/call/VectorCallActivity.kt | 126 +++++++----------- .../features/call/VectorCallViewModel.kt | 37 ++++- .../call/WebRtcPeerConnectionManager.kt | 25 ++-- .../home/room/detail/RoomDetailFragment.kt | 17 ++- .../notifications/NotificationUtils.kt | 41 +++--- vector/src/main/res/values/strings.xml | 4 + 10 files changed, 167 insertions(+), 171 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt index 1ab53854aa..3b3a393026 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallState.kt @@ -16,26 +16,29 @@ package im.vector.matrix.android.api.session.call -enum class CallState { +import org.webrtc.PeerConnection + +sealed class CallState { /** Idle, setting up objects */ - IDLE, + object Idle : CallState() /** Dialing. Outgoing call is signaling the remote peer */ - DIALING, + object Dialing : CallState() /** Local ringing. Incoming call offer received */ - LOCAL_RINGING, + object LocalRinging : CallState() /** Answering. Incoming call is responding to remote peer */ - ANSWERING, + object Answering : CallState() - /** Connecting. Incoming/Outgoing Offer and answer are known, Currently checking and testing pairs of ice candidates */ - CONNECTING, - - /** Connected. Incoming/Outgoing call, the call is connected */ - CONNECTED, + /** + * Connected. Incoming/Outgoing call, ice layer connecting or connected + * Notice that the PeerState failed is not always final, if you switch network, new ice candidtates + * could be exchanged, and the connection could go back to connected + * */ + data class Connected(val iceConnectionState: PeerConnection.PeerConnectionState) : CallState() /** Terminated. Incoming/Outgoing call, the call is terminated */ - TERMINATED, + object Terminated : CallState() } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt index a3bbcd6444..ed9361b4c5 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt @@ -101,6 +101,7 @@ internal class DefaultCallSignalingService @Inject constructor( override fun removeCallListener(listener: CallsListener) { callListeners.remove(listener) } + override fun getCallWithId(callId: String): MxCall? { Timber.v("## VOIP getCallWithId $callId all calls ${activeCalls.map { it.callId }}") return activeCalls.find { it.callId == callId } @@ -189,25 +190,4 @@ internal class DefaultCallSignalingService @Inject constructor( companion object { const val CALL_TIMEOUT_MS = 120_000 } - -// internal class PeerSignalingClientFactory @Inject constructor( -// @UserId private val userId: String, -// private val localEchoEventFactory: LocalEchoEventFactory, -// private val sendEventTask: SendEventTask, -// private val taskExecutor: TaskExecutor, -// private val cryptoService: CryptoService -// ) { -// -// fun create(roomId: String, callId: String): PeerSignalingClient { -// return RoomPeerSignalingClient( -// callID = callId, -// roomId = roomId, -// userId = userId, -// localEchoEventFactory = localEchoEventFactory, -// sendEventTask = sendEventTask, -// taskExecutor = taskExecutor, -// cryptoService = cryptoService -// ) -// } -// } } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt index 90b475d5b8..fe9b9e447c 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/model/MxCallImpl.kt @@ -46,7 +46,7 @@ internal class MxCallImpl( private val roomEventSender: RoomEventSender ) : MxCall { - override var state: CallState = CallState.IDLE + override var state: CallState = CallState.Idle set(value) { field = value dispatchStateChange() @@ -74,17 +74,17 @@ internal class MxCallImpl( init { if (isOutgoing) { - state = CallState.IDLE + state = CallState.Idle } else { // because it's created on reception of an offer - state = CallState.LOCAL_RINGING + state = CallState.LocalRinging } } override fun offerSdp(sdp: SessionDescription) { if (!isOutgoing) return Timber.v("## VOIP offerSdp $callId") - state = CallState.DIALING + state = CallState.Dialing CallInviteContent( callId = callId, lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, @@ -120,13 +120,13 @@ internal class MxCallImpl( ) .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) } .also { roomEventSender.sendEvent(it) } - state = CallState.TERMINATED + state = CallState.Terminated } override fun accept(sdp: SessionDescription) { Timber.v("## VOIP accept $callId") if (isOutgoing) return - state = CallState.ANSWERING + state = CallState.Answering CallAnswerContent( callId = callId, answer = CallAnswerContent.Answer(sdp = sdp.description) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index 9c4bf5f6e1..adf05184b2 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -29,6 +29,7 @@ import butterknife.OnClick import im.vector.matrix.android.api.session.call.CallState import im.vector.riotx.R import kotlinx.android.synthetic.main.fragment_call_controls.view.* +import org.webrtc.PeerConnection class CallControlsView @JvmOverloads constructor( context: Context, attrs: AttributeSet? = null, defStyleAttr: Int = 0 @@ -99,28 +100,34 @@ class CallControlsView @JvmOverloads constructor( videoToggleIcon.setImageResource(if (state.isVideoEnabled) R.drawable.ic_video else R.drawable.ic_video_off) when (callState) { - CallState.IDLE, - CallState.DIALING, - CallState.CONNECTING, - CallState.ANSWERING -> { + is CallState.Idle, + is CallState.Dialing, + is CallState.Answering -> { ringingControls.isVisible = true ringingControlAccept.isVisible = false ringingControlDecline.isVisible = true connectedControls.isVisible = false } - CallState.LOCAL_RINGING -> { + is CallState.LocalRinging -> { ringingControls.isVisible = true ringingControlAccept.isVisible = true ringingControlDecline.isVisible = true connectedControls.isVisible = false } - CallState.CONNECTED -> { - ringingControls.isVisible = false - connectedControls.isVisible = true - iv_video_toggle.isVisible = state.isVideoCall + is CallState.Connected -> { + if (callState.iceConnectionState == PeerConnection.PeerConnectionState.CONNECTED) { + ringingControls.isVisible = false + connectedControls.isVisible = true + iv_video_toggle.isVisible = state.isVideoCall + } else { + ringingControls.isVisible = true + ringingControlAccept.isVisible = false + ringingControlDecline.isVisible = true + connectedControls.isVisible = false + } } - CallState.TERMINATED, - null -> { + is CallState.Terminated, + null -> { ringingControls.isVisible = false connectedControls.isVisible = false } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index a6aede7f48..b0fe9865fd 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -27,6 +27,7 @@ import android.os.Parcelable import android.view.View import android.view.Window import android.view.WindowManager +import androidx.appcompat.app.AlertDialog import androidx.core.view.ViewCompat import androidx.core.view.isInvisible import androidx.core.view.isVisible @@ -39,6 +40,7 @@ import com.jakewharton.rxbinding3.view.clicks import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCallDetail +import im.vector.matrix.android.api.session.call.TurnServerResponse import im.vector.riotx.R import im.vector.riotx.core.di.ScreenComponent import im.vector.riotx.core.platform.VectorBaseActivity @@ -54,6 +56,7 @@ import io.reactivex.android.schedulers.AndroidSchedulers import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.webrtc.EglBase +import org.webrtc.PeerConnection import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer import timber.log.Timber @@ -66,8 +69,7 @@ data class CallArgs( val callId: String?, val participantUserId: String, val isIncomingCall: Boolean, - val isVideoCall: Boolean, - val autoAccept: Boolean + val isVideoCall: Boolean ) : Parcelable class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionListener { @@ -216,53 +218,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis turnScreenOffAndKeyguardOn() } -// override fun onResume() { -// super.onResume() -// } -// -// override fun onStop() { -// super.onStop() -// when(callViewModel.call?.state) { -// CallState.DIALING -> { -// CallService.onIncomingCall( -// this, -// callArgs.isVideoCall, -// callArgs.participantUserId, -// callArgs.roomId, -// "", -// callArgs.callId ?: "" -// ) -// } -// CallState.LOCAL_RINGING -> { -// CallService.onIncomingCall( -// this, -// callArgs.isVideoCall, -// callArgs.participantUserId, -// callArgs.roomId, -// "", -// callArgs.callId ?: "" -// ) -// } -// CallState.ANSWERING, -// CallState.CONNECTING, -// CallState.CONNECTED -> { -// CallService.onPendingCall( -// this, -// callArgs.isVideoCall, -// callArgs.participantUserId, -// callArgs.roomId, -// "", -// callArgs.callId ?: "" -// ) -// } -// CallState.TERMINATED , -// CallState.IDLE , -// null -> { -// -// } -// } -// } - private fun renderState(state: VectorCallViewState) { Timber.v("## VOIP renderState call $state") if (state.callState is Fail) { @@ -273,52 +228,55 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } callControlsView.updateForState(state) - when (state.callState.invoke()) { - CallState.IDLE, - CallState.DIALING -> { + val callState = state.callState.invoke() + when (callState) { + is CallState.Idle, + is CallState.Dialing -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.setText(R.string.call_ring) configureCallInfo(state) } - CallState.LOCAL_RINGING -> { + is CallState.LocalRinging -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.text = null configureCallInfo(state) } - CallState.ANSWERING -> { + is CallState.Answering -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.setText(R.string.call_connecting) configureCallInfo(state) } - CallState.CONNECTING -> { - callVideoGroup.isInvisible = true - callInfoGroup.isVisible = true - configureCallInfo(state) - callStatusText.setText(R.string.call_connecting) - } - CallState.CONNECTED -> { - if (callArgs.isVideoCall) { - callVideoGroup.isVisible = true - callInfoGroup.isVisible = false - pip_video_view.isVisible = !state.isVideoCaptureInError + is CallState.Connected -> { + if (callState.iceConnectionState == PeerConnection.PeerConnectionState.CONNECTED) { + if (callArgs.isVideoCall) { + callVideoGroup.isVisible = true + callInfoGroup.isVisible = false + pip_video_view.isVisible = !state.isVideoCaptureInError + } else { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + configureCallInfo(state) + callStatusText.text = null + } } else { + // This state is not final, if you change network, new candidates will be sent callVideoGroup.isInvisible = true callInfoGroup.isVisible = true configureCallInfo(state) - callStatusText.text = null + callStatusText.setText(R.string.call_connecting) } // ensure all attached? peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null) } - CallState.TERMINATED -> { + is CallState.Terminated -> { finish() } - null -> { + null -> { } } } @@ -382,20 +340,29 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private fun handleViewEvents(event: VectorCallViewEvents?) { Timber.v("## VOIP handleViewEvents $event") when (event) { - VectorCallViewEvents.DismissNoCall -> { + VectorCallViewEvents.DismissNoCall -> { CallService.onNoActiveCall(this) finish() } - null -> { + is VectorCallViewEvents.ConnectionTimout -> { + onErrorTimoutConnect(event.turn) + } + null -> { } } -// when (event) { -// is VectorCallViewEvents.CallAnswered -> { -// } -// is VectorCallViewEvents.CallHangup -> { -// finish() -// } -// } + } + + private fun onErrorTimoutConnect(turn: TurnServerResponse?) { + Timber.d("## VOIP onErrorTimoutConnect $turn") + // TODO ask to use default stun, etc... + AlertDialog + .Builder(this) + .setTitle(R.string.call_failed_no_connection) + .setMessage(getString(R.string.call_failed_no_connection_description)) + .setNegativeButton(R.string.ok) { _, _ -> + callViewModel.handle(VectorCallViewActions.EndCall) + } + .show() } companion object { @@ -411,7 +378,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis return Intent(context, VectorCallActivity::class.java).apply { // what could be the best flags? flags = Intent.FLAG_ACTIVITY_NEW_TASK - putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.callId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall, false)) + putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.callId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall)) putExtra(EXTRA_MODE, OUTGOING_CREATED) } } @@ -422,12 +389,11 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis otherUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean, - accept: Boolean, mode: String?): Intent { return Intent(context, VectorCallActivity::class.java).apply { // what could be the best flags? flags = Intent.FLAG_ACTIVITY_CLEAR_TOP - putExtra(MvRx.KEY_ARG, CallArgs(roomId, callId, otherUserId, isIncomingCall, isVideoCall, accept)) + putExtra(MvRx.KEY_ARG, CallArgs(roomId, callId, otherUserId, isIncomingCall, isVideoCall)) putExtra(EXTRA_MODE, mode) } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index b8af552dc4..ea15c07841 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -26,15 +26,20 @@ import com.airbnb.mvrx.Uninitialized import com.airbnb.mvrx.ViewModelContext import com.squareup.inject.assisted.Assisted import com.squareup.inject.assisted.AssistedInject +import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.MxCall +import im.vector.matrix.android.api.session.call.TurnServerResponse import im.vector.matrix.android.api.util.MatrixItem import im.vector.matrix.android.api.util.toMatrixItem import im.vector.riotx.core.extensions.exhaustive import im.vector.riotx.core.platform.VectorViewEvents import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.platform.VectorViewModelAction +import org.webrtc.PeerConnection +import java.util.Timer +import java.util.TimerTask data class VectorCallViewState( val callId: String? = null, @@ -60,6 +65,7 @@ sealed class VectorCallViewActions : VectorViewModelAction { sealed class VectorCallViewEvents : VectorViewEvents { object DismissNoCall : VectorCallViewEvents() + data class ConnectionTimout(val turn: TurnServerResponse?) : VectorCallViewEvents() // data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() // data class CallHangup(val content: CallHangupContent) : VectorCallViewEvents() // object CallAccepted : VectorCallViewEvents() @@ -72,15 +78,38 @@ class VectorCallViewModel @AssistedInject constructor( val webRtcPeerConnectionManager: WebRtcPeerConnectionManager ) : VectorViewModel(initialState) { - var autoReplyIfNeeded: Boolean = false - var call: MxCall? = null + var connectionTimoutTimer: Timer? = null + private val callStateListener = object : MxCall.StateListener { override fun onStateUpdate(call: MxCall) { + val callState = call.state + if (callState is CallState.Connected && callState.iceConnectionState == PeerConnection.PeerConnectionState.CONNECTED) { + connectionTimoutTimer?.cancel() + connectionTimoutTimer = null + } else { + // do we reset as long as it's moving? + connectionTimoutTimer?.cancel() + connectionTimoutTimer = Timer().apply { + schedule(object : TimerTask() { + override fun run() { + session.callSignalingService().getTurnServer(object : MatrixCallback { + override fun onFailure(failure: Throwable) { + _viewEvents.post(VectorCallViewEvents.ConnectionTimout(null)) + } + + override fun onSuccess(data: TurnServerResponse) { + _viewEvents.post(VectorCallViewEvents.ConnectionTimout(data)) + } + }) + } + }, 30_000) + } + } setState { copy( - callState = Success(call.state) + callState = Success(callState) ) } } @@ -99,8 +128,6 @@ class VectorCallViewModel @AssistedInject constructor( init { - autoReplyIfNeeded = args.autoAccept - initialState.callId?.let { webRtcPeerConnectionManager.addCurrentCallListener(currentCallListener) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index d75184ecca..be8380a3d8 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -269,7 +269,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // The call is going to resume from background, we can reduce notif currentCall?.mxCall - ?.takeIf { it.state == CallState.CONNECTING || it.state == CallState.CONNECTED } + ?.takeIf { it.state is CallState.Connected } ?.let { mxCall -> val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() ?: mxCall.roomId @@ -466,7 +466,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun acceptIncomingCall() { Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") val mxCall = currentCall?.mxCall - if (mxCall?.state == CallState.LOCAL_RINGING) { + if (mxCall?.state == CallState.LocalRinging) { getTurnServer { turnServer -> internalAcceptIncomingCall(currentCall!!, turnServer) } @@ -476,7 +476,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun detachRenderers() { // The call is going to continue in background, so ensure notification is visible currentCall?.mxCall - ?.takeIf { it.state == CallState.CONNECTING || it.state == CallState.CONNECTED } + ?.takeIf { it.state is CallState.Connected } ?.let { mxCall -> // Start background service with notification @@ -687,7 +687,7 @@ class WebRtcPeerConnectionManager @Inject constructor( if (call.mxCall.callId != callHangupContent.callId) return Unit.also { Timber.w("onCallHangupReceived for non active call? ${callHangupContent.callId}") } - call.mxCall.state = CallState.TERMINATED + call.mxCall.state = CallState.Terminated currentCall = null close() } @@ -702,7 +702,7 @@ class WebRtcPeerConnectionManager @Inject constructor( * or is closed (state "closed"); in addition, at least one transport is either "connected" or "completed" */ PeerConnection.PeerConnectionState.CONNECTED -> { - callContext.mxCall.state = CallState.CONNECTED + callContext.mxCall.state = CallState.Connected(newState) } /** * One or more of the ICE transports on the connection is in the "failed" state. @@ -710,6 +710,7 @@ class WebRtcPeerConnectionManager @Inject constructor( PeerConnection.PeerConnectionState.FAILED -> { // This can be temporary, e.g when other ice not yet received... // callContext.mxCall.state = CallState.ERROR + callContext.mxCall.state = CallState.Connected(newState) } /** * At least one of the connection's ICE transports (RTCIceTransports or RTCDtlsTransports) are in the "new" state, @@ -723,20 +724,20 @@ class WebRtcPeerConnectionManager @Inject constructor( * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state */ PeerConnection.PeerConnectionState.CONNECTING -> { - callContext.mxCall.state = CallState.CONNECTING + callContext.mxCall.state = CallState.Connected(PeerConnection.PeerConnectionState.CONNECTING) } /** * The RTCPeerConnection is closed. * This value was in the RTCSignalingState enum (and therefore found by reading the value of the signalingState) * property until the May 13, 2016 draft of the specification. */ - PeerConnection.PeerConnectionState.CLOSED -> { - } - /** - * At least one of the ICE transports for the connection is in the "disconnected" state and none of - * the other transports are in the state "failed", "connecting", or "checking". - */ + PeerConnection.PeerConnectionState.CLOSED, + /** + * At least one of the ICE transports for the connection is in the "disconnected" state and none of + * the other transports are in the state "failed", "connecting", or "checking". + */ PeerConnection.PeerConnectionState.DISCONNECTED -> { + callContext.mxCall.state = CallState.Connected(newState) } null -> { } diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index c4c87ac2f5..0f8c98128f 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -296,7 +296,7 @@ class RoomDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - val hasActiveCall = it?.state == CallState.CONNECTED + val hasActiveCall = it?.state is CallState.Connected activeCallView.isVisible = hasActiveCall }) @@ -1517,14 +1517,13 @@ class RoomDetailFragment @Inject constructor( override fun onTapToReturnToCall() { sharedCallActionViewModel.activeCall.value?.let { call -> VectorCallActivity.newIntent( - requireContext(), - call.callId, - call.roomId, - call.otherUserId, - !call.isOutgoing, - call.isVideoCall, - false, - null + context = requireContext(), + callId = call.callId, + roomId = call.roomId, + otherUserId = call.otherUserId, + isIncomingCall = !call.isOutgoing, + isVideoCall = call.isVideoCall, + mode = null ).let { startActivity(it) } diff --git a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt index 88a19fff89..d936935330 100755 --- a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt +++ b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt @@ -293,30 +293,34 @@ class NotificationUtils @Inject constructor(private val context: Context, // val pendingIntent = stackBuilder.getPendingIntent(requestId, PendingIntent.FLAG_UPDATE_CURRENT) val contentIntent = VectorCallActivity.newIntent( - context, callId, roomId, otherUserId, true, isVideo, - false, VectorCallActivity.INCOMING_RINGING + context = context, + callId = callId, + roomId = roomId, + otherUserId = otherUserId, + isIncomingCall = true, + isVideoCall = isVideo, + mode = VectorCallActivity.INCOMING_RINGING ).apply { flags = Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP data = Uri.parse("foobar://$callId") } val contentPendingIntent = PendingIntent.getActivity(context, System.currentTimeMillis().toInt(), contentIntent, 0) -// val contentPendingIntent = TaskStackBuilder.create(context) -// .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) -// .addNextIntent(RoomDetailActivity.newIntent(context, RoomDetailArgs(roomId = roomId)) -// .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, otherUserId, true, isVideo, false, VectorCallActivity.INCOMING_RINGING)) -// .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) - val answerCallPendingIntent = TaskStackBuilder.create(context) .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) - .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, otherUserId, true, isVideo, true, VectorCallActivity.INCOMING_ACCEPT)) + .addNextIntent(VectorCallActivity.newIntent( + context = context, + callId = callId, + roomId = roomId, + otherUserId = otherUserId, + isIncomingCall = true, + isVideoCall = isVideo, + mode = VectorCallActivity.INCOMING_ACCEPT) + ) .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) -// val answerCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { -// putExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, CallHeadsUpService.CALL_ACTION_ANSWER) -// } val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { - // putExtra(CallHeadsUpService.EXTRA_CALL_ACTION_KEY, CallHeadsUpService.CALL_ACTION_REJECT) + putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ACTION_KEY, CallHeadsUpActionReceiver.CALL_ACTION_REJECT) } // val answerCallPendingIntent = PendingIntent.getBroadcast(context, requestId, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) val rejectCallPendingIntent = PendingIntent.getBroadcast( @@ -367,8 +371,13 @@ class NotificationUtils @Inject constructor(private val context: Context, val requestId = Random.nextInt(1000) val contentIntent = VectorCallActivity.newIntent( - context, callId, roomId, otherUserId, true, isVideo, - false, null).apply { + context = context, + callId = callId, + roomId = roomId, + otherUserId = otherUserId, + isIncomingCall = true, + isVideoCall = isVideo, + mode = null).apply { flags = Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP data = Uri.parse("foobar://$callId") } @@ -451,7 +460,7 @@ class NotificationUtils @Inject constructor(private val context: Context, val contentPendingIntent = TaskStackBuilder.create(context) .addNextIntentWithParentStack(Intent(context, HomeActivity::class.java)) // TODO other userId - .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, "otherUserId", true, isVideo, false, null)) + .addNextIntent(VectorCallActivity.newIntent(context, callId, roomId, "otherUserId", true, isVideo, null)) .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) builder.setContentIntent(contentPendingIntent) diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index ebbd78f11a..0a6d46e937 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -210,6 +210,10 @@ Please ask the administrator of your homeserver (%1$s) to configure a TURN server in order for calls to work reliably.\n\nAlternatively, you can try to use the public server at %2$s, but this will not be as reliable, and it will share your IP address with that server. You can also manage this in Settings." Try using %s Do not ask me again + + RiotX Call Failed + Failed to establish real time connection.\nPlease ask the administrator of your homeserver to configure a TURN server in order for calls to work reliably. + Select Sound Device Phone Speaker From 30d47b4fa61f408a0465e601d5b3c39be3d77dda Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 17 Jun 2020 17:52:32 +0200 Subject: [PATCH 53/83] Clear incoming calls managed by other session --- .../android/api/session/call/CallsListener.kt | 2 + .../session/call/CallEventsObserverTask.kt | 1 + .../call/DefaultCallSignalingService.kt | 55 +++++++++++++++++-- .../call/WebRtcPeerConnectionManager.kt | 5 ++ 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index 49207f9ed0..7ec46e7c10 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -49,4 +49,6 @@ interface CallsListener { fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) fun onCallHangupReceived(callHangupContent: CallHangupContent) + + fun onCallManagedByOtherSession(callId: String) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt index 950381ef6b..4810853881 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt @@ -53,6 +53,7 @@ internal class DefaultCallEventsObserverTask @Inject constructor( private fun update(realm: Realm, events: List, userId: String) { val now = System.currentTimeMillis() + // TODO might check if an invite is not closed (hangup/answsered) in the same event batch? events.forEach { event -> event.roomId ?: return@forEach Unit.also { Timber.w("Event with no room id ${event.eventId}") diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt index ed9361b4c5..b8ecd5abe4 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallSignalingService.kt @@ -19,6 +19,7 @@ package im.vector.matrix.android.internal.session.call import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallSignalingService +import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.MxCall import im.vector.matrix.android.api.session.call.TurnServerResponse @@ -108,19 +109,33 @@ internal class DefaultCallSignalingService @Inject constructor( } internal fun onCallEvent(event: Event) { - // TODO if handled by other of my sessions - // this test is too simple, should notify upstream - if (event.senderId == userId) { - // ignore local echos! - return - } when (event.getClearType()) { EventType.CALL_ANSWER -> { event.getClearContent().toModel()?.let { + if (event.senderId == userId) { + // ok it's an answer from me.. is it remote echo or other session + val knownCall = getCallWithId(it.callId) + if (knownCall == null) { + Timber.d("## VOIP onCallEvent ${event.getClearType()} id ${it.callId} send by me") + } else if (!knownCall.isOutgoing) { + // incoming call + // if it was anwsered by this session, the call state would be in Answering(or connected) state + if (knownCall.state == CallState.LocalRinging) { + // discard current call, it's answered by another of my session + onCallManageByOtherSession(it.callId) + } + } + return + } + onCallAnswer(it) } } EventType.CALL_INVITE -> { + if (event.senderId == userId) { + // Always ignore local echos of invite + return + } event.getClearContent().toModel()?.let { content -> val incomingCall = MxCallImpl( callId = content.callId ?: return@let, @@ -138,11 +153,31 @@ internal class DefaultCallSignalingService @Inject constructor( } EventType.CALL_HANGUP -> { event.getClearContent().toModel()?.let { content -> + + if (event.senderId == userId) { + // ok it's an answer from me.. is it remote echo or other session + val knownCall = getCallWithId(content.callId) + if (knownCall == null) { + Timber.d("## VOIP onCallEvent ${event.getClearType()} id ${content.callId} send by me") + } else if (!knownCall.isOutgoing) { + // incoming call + if (knownCall.state == CallState.LocalRinging) { + // discard current call, it's answered by another of my session + onCallManageByOtherSession(content.callId) + } + } + return + } + onCallHangup(content) activeCalls.removeAll { it.callId == content.callId } } } EventType.CALL_CANDIDATES -> { + if (event.senderId == userId) { + // Always ignore local echos of invite + return + } event.getClearContent().toModel()?.let { content -> activeCalls.firstOrNull { it.callId == content.callId }?.let { onCallIceCandidate(it, content) @@ -168,6 +203,14 @@ internal class DefaultCallSignalingService @Inject constructor( } } + private fun onCallManageByOtherSession(callId: String) { + callListeners.toList().forEach { + tryThis { + it.onCallManagedByOtherSession(callId) + } + } + } + private fun onCallInvite(incomingCall: MxCall, invite: CallInviteContent) { // Ignore the invitation from current user if (incomingCall.otherUserId == userId) return diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index be8380a3d8..d91bf270fe 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -692,6 +692,11 @@ class WebRtcPeerConnectionManager @Inject constructor( close() } + override fun onCallManagedByOtherSession(callId: String) { + currentCall = null + CallService.onNoActiveCall(context) + } + private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer { override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { From 3e2d892fb563f43397de425fc843ece509d316c9 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 11:56:55 +0200 Subject: [PATCH 54/83] Headset support + detect plug/unplugg --- .../vector/riotx/core/services/CallService.kt | 22 +++-- .../services/WiredHeadsetStateReceiver.kt | 85 +++++++++++++++++++ .../riotx/features/call/CallAudioManager.kt | 29 +++++-- .../features/call/CallControlsBottomSheet.kt | 65 ++++++++++---- .../riotx/features/call/VectorCallActivity.kt | 6 +- .../features/call/VectorCallViewModel.kt | 54 ++++++++---- .../call/WebRtcPeerConnectionManager.kt | 17 ++++ vector/src/main/res/values/strings.xml | 1 + 8 files changed, 229 insertions(+), 50 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/core/services/WiredHeadsetStateReceiver.kt diff --git a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt index f10bbd908d..d8addd46a7 100644 --- a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt @@ -30,7 +30,7 @@ import timber.log.Timber /** * Foreground service to manage calls */ -class CallService : VectorService() { +class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListener { private val connections = mutableMapOf() @@ -49,11 +49,21 @@ class CallService : VectorService() { private var callRingPlayer: CallRingPlayer? = null + private var wiredHeadsetStateReceiver: WiredHeadsetStateReceiver? = null + override fun onCreate() { super.onCreate() notificationUtils = vectorComponent().notificationUtils() webRtcPeerConnectionManager = vectorComponent().webRtcPeerConnectionManager() callRingPlayer = CallRingPlayer(applicationContext) + wiredHeadsetStateReceiver = WiredHeadsetStateReceiver.createAndRegister(this, this) + } + + override fun onDestroy() { + super.onDestroy() + callRingPlayer?.stop() + wiredHeadsetStateReceiver?.let { WiredHeadsetStateReceiver.unRegister(this, it) } + wiredHeadsetStateReceiver = null } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { @@ -217,11 +227,6 @@ class CallService : VectorService() { myStopSelf() } - override fun onDestroy() { - super.onDestroy() - callRingPlayer?.stop() - } - fun addConnection(callConnection: CallConnection) { connections[callConnection.callId] = callConnection } @@ -335,4 +340,9 @@ class CallService : VectorService() { return this@CallService } } + + override fun onHeadsetEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { + Timber.v("## VOIP: onHeadsetEvent $event") + webRtcPeerConnectionManager.onWireDeviceEvent(event) + } } diff --git a/vector/src/main/java/im/vector/riotx/core/services/WiredHeadsetStateReceiver.kt b/vector/src/main/java/im/vector/riotx/core/services/WiredHeadsetStateReceiver.kt new file mode 100644 index 0000000000..e63c7f5049 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/core/services/WiredHeadsetStateReceiver.kt @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.core.services + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.media.AudioManager +import android.os.Build +import timber.log.Timber +import java.lang.ref.WeakReference + +/** + * Dynamic broadcast receiver to detect headset plug/unplug + */ +class WiredHeadsetStateReceiver : BroadcastReceiver() { + + interface HeadsetEventListener { + fun onHeadsetEvent(event: HeadsetPlugEvent) + } + + var delegate: WeakReference? = null + + data class HeadsetPlugEvent( + val plugged: Boolean, + val headsetName: String?, + val hasMicrophone: Boolean + ) + + override fun onReceive(context: Context?, intent: Intent?) { + // The intent will have the following extra values: + // state 0 for unplugged, 1 for plugged + // name Headset type, human readable string + // microphone 1 if headset has a microphone, 0 otherwise + + val isPlugged = when (intent?.getIntExtra("state", -1)) { + 0 -> false + 1 -> true + else -> return Unit.also { + Timber.v("## VOIP WiredHeadsetStateReceiver invalid state") + } + } + val hasMicrophone = when (intent.getIntExtra("microphone", -1)) { + 1 -> true + else -> false + } + + delegate?.get()?.onHeadsetEvent( + HeadsetPlugEvent(plugged = isPlugged, headsetName = intent.getStringExtra("name"), hasMicrophone = hasMicrophone) + ) + } + + companion object { + fun createAndRegister(context: Context, listener: HeadsetEventListener): WiredHeadsetStateReceiver { + val receiver = WiredHeadsetStateReceiver() + receiver.delegate = WeakReference(listener) + val action = if (Build.VERSION.SDK_INT >= 21) { + AudioManager.ACTION_HEADSET_PLUG + } else { + Intent.ACTION_HEADSET_PLUG + } + context.registerReceiver(receiver, IntentFilter(action)) + return receiver + } + + fun unRegister(context: Context, receiver: WiredHeadsetStateReceiver) { + context.unregisterReceiver(receiver) + } + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt index b3bdc23707..ad2eae510f 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -28,7 +28,8 @@ class CallAudioManager( enum class SoundDevice { PHONE, - SPEAKER + SPEAKER, + HEADSET } private val audioManager: AudioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager @@ -71,14 +72,24 @@ class CallAudioManager( // Always disable microphone mute during a WebRTC call. setMicrophoneMute(false) - // TODO check if there are headsets? - if (mxCall.isVideoCall) { + // If there are no headset, start video output in speaker + // (you can't watch the video and have the phone close to your ear) + if (mxCall.isVideoCall && !isHeadsetOn()) { setSpeakerphoneOn(true) } else { + // if a headset is plugged, sound will be directed to it + // (can't really force earpiece when headset is plugged) setSpeakerphoneOn(false) } } + fun getAvailableSoundDevices(): List { + return listOf( + if (isHeadsetOn()) SoundDevice.HEADSET else SoundDevice.PHONE, + SoundDevice.SPEAKER + ) + } + fun stop() { Timber.v("## VOIP: AudioManager stopCall") @@ -91,21 +102,27 @@ class CallAudioManager( audioManager.abandonAudioFocus(audioFocusChangeListener) } - fun getCurrentSoundDevice() : SoundDevice { + fun getCurrentSoundDevice(): SoundDevice { if (audioManager.isSpeakerphoneOn) { return SoundDevice.SPEAKER } else { - return SoundDevice.PHONE + return if (isHeadsetOn()) SoundDevice.HEADSET else SoundDevice.PHONE } } - fun setCurrentSoundDevice(device: SoundDevice) { + fun setCurrentSoundDevice(device: SoundDevice) { when (device) { + SoundDevice.HEADSET, SoundDevice.PHONE -> setSpeakerphoneOn(false) SoundDevice.SPEAKER -> setSpeakerphoneOn(true) } } + private fun isHeadsetOn(): Boolean { + @Suppress("DEPRECATION") + return audioManager.isWiredHeadsetOn || audioManager.isBluetoothScoOn + } + /** Sets the speaker phone mode. */ private fun setSpeakerphoneOn(on: Boolean) { Timber.v("## VOIP: AudioManager setSpeakerphoneOn $on") diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt index 6e4a304932..70c6113160 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -23,6 +23,7 @@ import com.airbnb.mvrx.activityViewModel import im.vector.riotx.R import im.vector.riotx.core.platform.VectorBaseBottomSheetDialogFragment import kotlinx.android.synthetic.main.bottom_sheet_call_controls.* +import me.gujun.android.span.span class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { override fun getLayoutResId() = R.layout.bottom_sheet_call_controls @@ -37,25 +38,54 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { } callControlsSoundDevice.clickableView.debouncedClicks { - val soundDevices = listOf( - getString(R.string.sound_device_phone), - getString(R.string.sound_device_speaker) - ) - AlertDialog.Builder(requireContext()) - .setItems(soundDevices.toTypedArray()) { d, n -> - d.cancel() - when (soundDevices[n]) { - getString(R.string.sound_device_phone) -> { - callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.PHONE)) - } - getString(R.string.sound_device_speaker) -> { - callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.SPEAKER)) - } + callViewModel.handle(VectorCallViewActions.SwitchSoundDevice) + } + + callViewModel.observeViewEvents { + when (it) { + is VectorCallViewEvents.ShowSoundDeviceChooser -> { + showSoundDeviceChooser(it.available, it.current) + } + else -> { + } + } + } + } + + private fun showSoundDeviceChooser(available: List, current: CallAudioManager.SoundDevice) { + val soundDevices = available.map { + when (it) { + CallAudioManager.SoundDevice.PHONE -> span { + text = getString(R.string.sound_device_phone) + textStyle = if (current == it) "bold" else "normal" + } + CallAudioManager.SoundDevice.SPEAKER -> span { + text = getString(R.string.sound_device_speaker) + textStyle = if (current == it) "bold" else "normal" + } + CallAudioManager.SoundDevice.HEADSET -> span { + text = getString(R.string.sound_device_headset) + textStyle = if (current == it) "bold" else "normal" + } + } + } + AlertDialog.Builder(requireContext()) + .setItems(soundDevices.toTypedArray()) { d, n -> + d.cancel() + when (soundDevices[n].toString()) { + getString(R.string.sound_device_phone) -> { + callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.PHONE)) + } + getString(R.string.sound_device_speaker) -> { + callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.SPEAKER)) + } + getString(R.string.sound_device_headset) -> { + callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.HEADSET)) } } - .setNegativeButton(R.string.cancel, null) - .show() - } + } + .setNegativeButton(R.string.cancel, null) + .show() } // override fun onCreateDialog(savedInstanceState: Bundle?): Dialog { @@ -73,6 +103,7 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { callControlsSoundDevice.subTitle = when (state.soundDevice) { CallAudioManager.SoundDevice.PHONE -> getString(R.string.sound_device_phone) CallAudioManager.SoundDevice.SPEAKER -> getString(R.string.sound_device_speaker) + CallAudioManager.SoundDevice.HEADSET -> getString(R.string.sound_device_headset) } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index b0fe9865fd..67ecd42a70 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -340,14 +340,14 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private fun handleViewEvents(event: VectorCallViewEvents?) { Timber.v("## VOIP handleViewEvents $event") when (event) { - VectorCallViewEvents.DismissNoCall -> { + VectorCallViewEvents.DismissNoCall -> { CallService.onNoActiveCall(this) finish() } - is VectorCallViewEvents.ConnectionTimout -> { + is VectorCallViewEvents.ConnectionTimeout -> { onErrorTimoutConnect(event.turn) } - null -> { + null -> { } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index ea15c07841..b721d31cd5 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -49,6 +49,7 @@ data class VectorCallViewState( val isVideoEnabled: Boolean = true, val isVideoCaptureInError: Boolean = false, val soundDevice: CallAudioManager.SoundDevice = CallAudioManager.SoundDevice.PHONE, + val availableSoundDevices: List = emptyList(), val otherUserMatrixItem: Async = Uninitialized, val callState: Async = Uninitialized ) : MvRxState @@ -60,12 +61,17 @@ sealed class VectorCallViewActions : VectorViewModelAction { object ToggleMute : VectorCallViewActions() object ToggleVideo : VectorCallViewActions() data class ChangeAudioDevice(val device: CallAudioManager.SoundDevice) : VectorCallViewActions() + object SwitchSoundDevice : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { object DismissNoCall : VectorCallViewEvents() - data class ConnectionTimout(val turn: TurnServerResponse?) : VectorCallViewEvents() + data class ConnectionTimeout(val turn: TurnServerResponse?) : VectorCallViewEvents() + data class ShowSoundDeviceChooser( + val available: List, + val current: CallAudioManager.SoundDevice + ) : VectorCallViewEvents() // data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() // data class CallHangup(val content: CallHangupContent) : VectorCallViewEvents() // object CallAccepted : VectorCallViewEvents() @@ -96,11 +102,11 @@ class VectorCallViewModel @AssistedInject constructor( override fun run() { session.callSignalingService().getTurnServer(object : MatrixCallback { override fun onFailure(failure: Throwable) { - _viewEvents.post(VectorCallViewEvents.ConnectionTimout(null)) + _viewEvents.post(VectorCallViewEvents.ConnectionTimeout(null)) } override fun onSuccess(data: TurnServerResponse) { - _viewEvents.post(VectorCallViewEvents.ConnectionTimout(data)) + _viewEvents.post(VectorCallViewEvents.ConnectionTimeout(data)) } }) } @@ -124,6 +130,15 @@ class VectorCallViewModel @AssistedInject constructor( copy(isVideoCaptureInError = captureInError) } } + + override fun onAudioDevicesChange(mgr: WebRtcPeerConnectionManager) { + setState { + copy( + availableSoundDevices = mgr.audioManager.getAvailableSoundDevices(), + soundDevice = mgr.audioManager.getCurrentSoundDevice() + ) + } + } } init { @@ -143,7 +158,8 @@ class VectorCallViewModel @AssistedInject constructor( isVideoCall = mxCall.isVideoCall, callState = Success(mxCall.state), otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized, - soundDevice = webRtcPeerConnectionManager.audioManager.getCurrentSoundDevice() + soundDevice = webRtcPeerConnectionManager.audioManager.getCurrentSoundDevice(), + availableSoundDevices = webRtcPeerConnectionManager.audioManager.getAvailableSoundDevices() ) } } ?: run { @@ -163,7 +179,7 @@ class VectorCallViewModel @AssistedInject constructor( super.onCleared() } - override fun handle(action: VectorCallViewActions) = withState { + override fun handle(action: VectorCallViewActions) = withState { state -> when (action) { VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() VectorCallViewActions.AcceptCall -> { @@ -179,24 +195,21 @@ class VectorCallViewModel @AssistedInject constructor( webRtcPeerConnectionManager.endCall() } VectorCallViewActions.ToggleMute -> { - withState { - val muted = it.isAudioMuted - webRtcPeerConnectionManager.muteCall(!muted) - setState { - copy(isAudioMuted = !muted) - } + val muted = state.isAudioMuted + webRtcPeerConnectionManager.muteCall(!muted) + setState { + copy(isAudioMuted = !muted) } } VectorCallViewActions.ToggleVideo -> { - withState { - if (it.isVideoCall) { - val videoEnabled = it.isVideoEnabled - webRtcPeerConnectionManager.enableVideo(!videoEnabled) - setState { - copy(isVideoEnabled = !videoEnabled) - } + if (state.isVideoCall) { + val videoEnabled = state.isVideoEnabled + webRtcPeerConnectionManager.enableVideo(!videoEnabled) + setState { + copy(isVideoEnabled = !videoEnabled) } } + Unit } is VectorCallViewActions.ChangeAudioDevice -> { webRtcPeerConnectionManager.audioManager.setCurrentSoundDevice(action.device) @@ -206,6 +219,11 @@ class VectorCallViewModel @AssistedInject constructor( ) } } + VectorCallViewActions.SwitchSoundDevice -> { + _viewEvents.post( + VectorCallViewEvents.ShowSoundDeviceChooser(state.availableSoundDevices, state.soundDevice) + ) + } }.exhaustive } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index d91bf270fe..8b2e1ef74c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -33,6 +33,7 @@ import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.riotx.core.di.ActiveSessionHolder import im.vector.riotx.core.services.CallService +import im.vector.riotx.core.services.WiredHeadsetStateReceiver import io.reactivex.disposables.Disposable import io.reactivex.subjects.PublishSubject import io.reactivex.subjects.ReplaySubject @@ -75,6 +76,7 @@ class WebRtcPeerConnectionManager @Inject constructor( interface CurrentCallListener { fun onCurrentCallChange(call: MxCall?) fun onCaptureStateChanged(captureInError: Boolean) + fun onAudioDevicesChange(mgr: WebRtcPeerConnectionManager) {} } private val currentCallsListeners = emptyList().toMutableList() @@ -657,6 +659,21 @@ class WebRtcPeerConnectionManager @Inject constructor( close() } + fun onWireDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { + currentCall ?: return + // if it's plugged and speaker is on we should route to headset + if (event.plugged && audioManager.getCurrentSoundDevice() == CallAudioManager.SoundDevice.SPEAKER) { + audioManager.setCurrentSoundDevice(CallAudioManager.SoundDevice.HEADSET) + } else if (!event.plugged) { + // if it's unplugged ? always route to speaker? + // this is questionable? + audioManager.setCurrentSoundDevice(CallAudioManager.SoundDevice.SPEAKER) + } + currentCallsListeners.forEach { + it.onAudioDevicesChange(this) + } + } + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { val call = currentCall ?: return if (call.mxCall.callId != callAnswerContent.callId) return Unit.also { diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index 0a6d46e937..7d67e30a70 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -217,6 +217,7 @@ Select Sound Device Phone Speaker + Headset Send files From 9653f082a3b86211f7bea1bb06611a38189de545 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 12:19:20 +0200 Subject: [PATCH 55/83] accept/hangup on press on headset button --- .../riotx/features/call/VectorCallActivity.kt | 22 +++++++++---------- .../features/call/VectorCallViewModel.kt | 12 ++++++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 67ecd42a70..de0d3151a9 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -24,6 +24,7 @@ import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP import android.os.Build import android.os.Bundle import android.os.Parcelable +import android.view.KeyEvent import android.view.View import android.view.Window import android.view.WindowManager @@ -324,18 +325,15 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() }) } -// override fun onResume() { -// super.onResume() -// withState(callViewModel) { -// if(it.callState.invoke() == CallState.CONNECTED) { -// peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) -// } -// } -// } -// override fun onPause() { -// peerConnectionManager.detachRenderers() -// super.onPause() -// } + override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { + when (keyCode) { + KeyEvent.KEYCODE_HEADSETHOOK -> { + callViewModel.handle(VectorCallViewActions.HeadSetButtonPressed) + return true + } + } + return super.onKeyDown(keyCode, event) + } private fun handleViewEvents(event: VectorCallViewEvents?) { Timber.v("## VOIP handleViewEvents $event") diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index b721d31cd5..e7bc2662dd 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -62,6 +62,7 @@ sealed class VectorCallViewActions : VectorViewModelAction { object ToggleVideo : VectorCallViewActions() data class ChangeAudioDevice(val device: CallAudioManager.SoundDevice) : VectorCallViewActions() object SwitchSoundDevice : VectorCallViewActions() + object HeadSetButtonPressed: VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { @@ -224,6 +225,17 @@ class VectorCallViewModel @AssistedInject constructor( VectorCallViewEvents.ShowSoundDeviceChooser(state.availableSoundDevices, state.soundDevice) ) } + VectorCallViewActions.HeadSetButtonPressed -> { + if (state.callState.invoke() is CallState.LocalRinging) { + // accept call + webRtcPeerConnectionManager.acceptIncomingCall() + } + if (state.callState.invoke() is CallState.Connected) { + // end call? + webRtcPeerConnectionManager.endCall() + } + Unit + } }.exhaustive } From 4c61dfef62bf147903d0a888959c4224f9d8e30e Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 14:30:08 +0200 Subject: [PATCH 56/83] Support headset buttons in background --- vector/src/main/AndroidManifest.xml | 17 +++++++- .../vector/riotx/core/services/CallService.kt | 42 ++++++++++++++----- .../riotx/features/call/VectorCallActivity.kt | 12 ++++-- .../call/WebRtcPeerConnectionManager.kt | 13 ++++++ 4 files changed, 69 insertions(+), 15 deletions(-) diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 64299ff1ae..eafe99f86b 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -196,7 +196,12 @@ + android:exported="false" > + + + + + + + + + + + + () - /** - * call in progress (foreground notification) - */ -// private var mCallIdInProgress: String? = null - private lateinit var notificationUtils: NotificationUtils private lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager - /** - * incoming (foreground notification) - */ -// private var mIncomingCallId: String? = null - private var callRingPlayer: CallRingPlayer? = null private var wiredHeadsetStateReceiver: WiredHeadsetStateReceiver? = null + // A media button receiver receives and helps translate hardware media playback buttons, + // such as those found on wired and wireless headsets, into the appropriate callbacks in your app + private var mediaSession : MediaSessionCompat? = null + private val mediaSessionButtonCallback = object : MediaSessionCompat.Callback() { + override fun onMediaButtonEvent(mediaButtonEvent: Intent?): Boolean { + val keyEvent = mediaButtonEvent?.getParcelableExtra(Intent.EXTRA_KEY_EVENT) ?: return false + if (keyEvent.keyCode == KeyEvent.KEYCODE_HEADSETHOOK) { + webRtcPeerConnectionManager.headSetButtonTapped() + return true + } + return false + } + } + override fun onCreate() { super.onCreate() notificationUtils = vectorComponent().notificationUtils() @@ -64,22 +71,36 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe callRingPlayer?.stop() wiredHeadsetStateReceiver?.let { WiredHeadsetStateReceiver.unRegister(this, it) } wiredHeadsetStateReceiver = null + mediaSession?.release() + mediaSession = null } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { Timber.v("## VOIP onStartCommand $intent") + if (mediaSession == null) { + mediaSession = MediaSessionCompat(applicationContext, CallService::class.java.name).apply { + setCallback(mediaSessionButtonCallback) + } + } if (intent == null) { // Service started again by the system. // TODO What do we do here? return START_STICKY } + mediaSession?.let { + // This ensures that the correct callbacks to MediaSessionCompat.Callback + // will be triggered based on the incoming KeyEvent. + MediaButtonReceiver.handleIntent(it, intent) + } when (intent.action) { ACTION_INCOMING_RINGING_CALL -> { + mediaSession?.isActive = true callRingPlayer?.start() displayIncomingCallNotification(intent) } ACTION_OUTGOING_RINGING_CALL -> { + mediaSession?.isActive = true callRingPlayer?.start() displayOutgoingRingingCallNotification(intent) } @@ -221,6 +242,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe private fun hideCallNotifications() { val notification = notificationUtils.buildCallEndedNotification() + mediaSession?.isActive = false // It's mandatory to startForeground to avoid crash startForeground(NOTIFICATION_ID, notification) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index de0d3151a9..2c1c909a97 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -326,10 +326,14 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { - when (keyCode) { - KeyEvent.KEYCODE_HEADSETHOOK -> { - callViewModel.handle(VectorCallViewActions.HeadSetButtonPressed) - return true + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + // for newer version, it will be passed automatically to active media session + // in call service + when (keyCode) { + KeyEvent.KEYCODE_HEADSETHOOK -> { + callViewModel.handle(VectorCallViewActions.HeadSetButtonPressed) + return true + } } } return super.onKeyDown(keyCode, event) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 8b2e1ef74c..9fbc38d816 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -184,6 +184,19 @@ class WebRtcPeerConnectionManager @Inject constructor( } } + fun headSetButtonTapped() { + Timber.v("## VOIP headSetButtonTapped") + val call = currentCall?.mxCall ?: return + if (call.state is CallState.LocalRinging) { + // accept call + acceptIncomingCall() + } + if (call.state is CallState.Connected) { + // end call? + endCall() + } + } + private fun createPeerConnectionFactory() { if (peerConnectionFactory != null) return Timber.v("## VOIP createPeerConnectionFactory") From 5dfa08ace61549000bddbd1fc62ff35d12d7c61d Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 17:46:50 +0200 Subject: [PATCH 57/83] Bluetooth headset support --- vector/src/main/AndroidManifest.xml | 2 + .../core/services/BluetoothHeadsetReceiver.kt | 92 ++++++++++ .../vector/riotx/core/services/CallService.kt | 15 +- .../riotx/features/call/CallAudioManager.kt | 171 ++++++++++++++---- .../features/call/CallControlsBottomSheet.kt | 8 + .../call/WebRtcPeerConnectionManager.kt | 28 +-- vector/src/main/res/values/strings.xml | 1 + 7 files changed, 268 insertions(+), 49 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/core/services/BluetoothHeadsetReceiver.kt diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index eafe99f86b..652534a7ff 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -3,6 +3,8 @@ xmlns:tools="http://schemas.android.com/tools" package="im.vector.riotx"> + + diff --git a/vector/src/main/java/im/vector/riotx/core/services/BluetoothHeadsetReceiver.kt b/vector/src/main/java/im/vector/riotx/core/services/BluetoothHeadsetReceiver.kt new file mode 100644 index 0000000000..a56c4c73c6 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/core/services/BluetoothHeadsetReceiver.kt @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.core.services + +import android.bluetooth.BluetoothAdapter +import android.bluetooth.BluetoothClass +import android.bluetooth.BluetoothDevice +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import java.lang.ref.WeakReference + +class BluetoothHeadsetReceiver : BroadcastReceiver() { + + interface EventListener { + fun onBTHeadsetEvent(event: BTHeadsetPlugEvent) + } + + var delegate: WeakReference? = null + + data class BTHeadsetPlugEvent( + val plugged: Boolean, + val headsetName: String?, + /** + * BluetoothClass.Device.AUDIO_VIDEO_HANDSFREE + * BluetoothClass.Device.AUDIO_VIDEO_CAR_AUDIO + * AUDIO_VIDEO_WEARABLE_HEADSET + */ + val deviceClass: Int + ) + + override fun onReceive(context: Context?, intent: Intent?) { + // This intent will have 3 extras: + // EXTRA_CONNECTION_STATE - The current connection state + // EXTRA_PREVIOUS_CONNECTION_STATE}- The previous connection state. + // BluetoothDevice#EXTRA_DEVICE - The remote device. + // EXTRA_CONNECTION_STATE or EXTRA_PREVIOUS_CONNECTION_STATE can be any of + // STATE_DISCONNECTED}, STATE_CONNECTING, STATE_CONNECTED, STATE_DISCONNECTING + + val headsetConnected = when (intent?.getIntExtra(BluetoothAdapter.EXTRA_CONNECTION_STATE, -1)) { + BluetoothAdapter.STATE_CONNECTED -> true + BluetoothAdapter.STATE_DISCONNECTED -> false + else -> return // ignore intermediate states + } + + val device = intent.getParcelableExtra(BluetoothDevice.EXTRA_DEVICE) + val deviceName = device?.name + when (device?.bluetoothClass?.deviceClass) { + BluetoothClass.Device.AUDIO_VIDEO_HANDSFREE, + BluetoothClass.Device.AUDIO_VIDEO_CAR_AUDIO, + BluetoothClass.Device.AUDIO_VIDEO_WEARABLE_HEADSET -> { + // filter only device that we care about for + delegate?.get()?.onBTHeadsetEvent( + BTHeadsetPlugEvent( + plugged = headsetConnected, + headsetName = deviceName, + deviceClass = device.bluetoothClass.deviceClass + ) + ) + } + else -> return + } + } + + companion object { + fun createAndRegister(context: Context, listener: EventListener): BluetoothHeadsetReceiver { + val receiver = BluetoothHeadsetReceiver() + receiver.delegate = WeakReference(listener) + context.registerReceiver(receiver, IntentFilter(BluetoothAdapter.ACTION_CONNECTION_STATE_CHANGED)) + return receiver + } + + fun unRegister(context: Context, receiver: BluetoothHeadsetReceiver) { + context.unregisterReceiver(receiver) + } + } +} diff --git a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt index 832f484542..723cfe3add 100644 --- a/vector/src/main/java/im/vector/riotx/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/riotx/core/services/CallService.kt @@ -33,7 +33,7 @@ import timber.log.Timber /** * Foreground service to manage calls */ -class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListener { +class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListener, BluetoothHeadsetReceiver.EventListener { private val connections = mutableMapOf() @@ -43,10 +43,11 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe private var callRingPlayer: CallRingPlayer? = null private var wiredHeadsetStateReceiver: WiredHeadsetStateReceiver? = null + private var bluetoothHeadsetStateReceiver: BluetoothHeadsetReceiver? = null // A media button receiver receives and helps translate hardware media playback buttons, // such as those found on wired and wireless headsets, into the appropriate callbacks in your app - private var mediaSession : MediaSessionCompat? = null + private var mediaSession: MediaSessionCompat? = null private val mediaSessionButtonCallback = object : MediaSessionCompat.Callback() { override fun onMediaButtonEvent(mediaButtonEvent: Intent?): Boolean { val keyEvent = mediaButtonEvent?.getParcelableExtra(Intent.EXTRA_KEY_EVENT) ?: return false @@ -64,6 +65,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe webRtcPeerConnectionManager = vectorComponent().webRtcPeerConnectionManager() callRingPlayer = CallRingPlayer(applicationContext) wiredHeadsetStateReceiver = WiredHeadsetStateReceiver.createAndRegister(this, this) + bluetoothHeadsetStateReceiver = BluetoothHeadsetReceiver.createAndRegister(this, this) } override fun onDestroy() { @@ -71,6 +73,8 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe callRingPlayer?.stop() wiredHeadsetStateReceiver?.let { WiredHeadsetStateReceiver.unRegister(this, it) } wiredHeadsetStateReceiver = null + bluetoothHeadsetStateReceiver?.let { BluetoothHeadsetReceiver.unRegister(this, it) } + bluetoothHeadsetStateReceiver = null mediaSession?.release() mediaSession = null } @@ -365,6 +369,11 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe override fun onHeadsetEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { Timber.v("## VOIP: onHeadsetEvent $event") - webRtcPeerConnectionManager.onWireDeviceEvent(event) + webRtcPeerConnectionManager.onWiredDeviceEvent(event) + } + + override fun onBTHeadsetEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) { + Timber.v("## VOIP: onBTHeadsetEvent $event") + webRtcPeerConnectionManager.onWirelessDeviceEvent(event) } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt index ad2eae510f..9290d9a3e8 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -16,28 +16,59 @@ package im.vector.riotx.features.call +import android.bluetooth.BluetoothManager +import android.bluetooth.BluetoothProfile import android.content.Context import android.content.pm.PackageManager import android.media.AudioManager import im.vector.matrix.android.api.session.call.MxCall +import im.vector.riotx.core.services.WiredHeadsetStateReceiver import timber.log.Timber +import java.util.concurrent.Executors class CallAudioManager( - val applicationContext: Context + val applicationContext: Context, + val configChange: (() -> Unit)? ) { enum class SoundDevice { PHONE, SPEAKER, - HEADSET + HEADSET, + WIRELESS_HEADSET } - private val audioManager: AudioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + /* + * if all calls to audio manager not in the same thread it's not working well... + */ + private val executor = Executors.newSingleThreadExecutor() + + private var audioManager: AudioManager? = null private var savedIsSpeakerPhoneOn = false private var savedIsMicrophoneMute = false private var savedAudioMode = AudioManager.MODE_INVALID + private var connectedBlueToothHeadset: BluetoothProfile? = null + private var wantsBluetoothConnection = false + + init { + executor.execute { + audioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + } + val bm = applicationContext.getSystemService(Context.BLUETOOTH_SERVICE) as? BluetoothManager + bm?.adapter?.getProfileProxy(applicationContext, object : BluetoothProfile.ServiceListener { + override fun onServiceDisconnected(profile: Int) { + connectedBlueToothHeadset = null + } + + override fun onServiceConnected(profile: Int, proxy: BluetoothProfile?) { + connectedBlueToothHeadset = proxy + configChange?.invoke() + } + }, BluetoothProfile.HEADSET) + } + private val audioFocusChangeListener = AudioManager.OnAudioFocusChangeListener { focusChange -> // Called on the listener to notify if the audio focus for this listener has been changed. @@ -49,6 +80,7 @@ class CallAudioManager( fun startForCall(mxCall: MxCall) { Timber.v("## VOIP: AudioManager startForCall ${mxCall.callId}") + val audioManager = audioManager ?: return savedIsSpeakerPhoneOn = audioManager.isSpeakerphoneOn savedIsMicrophoneMute = audioManager.isMicrophoneMute savedAudioMode = audioManager.mode @@ -72,77 +104,150 @@ class CallAudioManager( // Always disable microphone mute during a WebRTC call. setMicrophoneMute(false) - // If there are no headset, start video output in speaker - // (you can't watch the video and have the phone close to your ear) - if (mxCall.isVideoCall && !isHeadsetOn()) { - setSpeakerphoneOn(true) - } else { - // if a headset is plugged, sound will be directed to it - // (can't really force earpiece when headset is plugged) - setSpeakerphoneOn(false) + executor.execute { + // If there are no headset, start video output in speaker + // (you can't watch the video and have the phone close to your ear) + if (mxCall.isVideoCall && !isHeadsetOn()) { + Timber.v("##VOIP: AudioManager default to speaker ") + setCurrentSoundDevice(SoundDevice.SPEAKER) + } else { + // if a wired headset is plugged, sound will be directed to it + // (can't really force earpiece when headset is plugged) + if (isBluetoothHeadsetOn()) { + Timber.v("##VOIP: AudioManager default to WIRELESS_HEADSET ") + setCurrentSoundDevice(SoundDevice.WIRELESS_HEADSET) + // try now in case already connected? + audioManager.isBluetoothScoOn = true + } else { + Timber.v("##VOIP: AudioManager default to PHONE/HEADSET ") + setCurrentSoundDevice(if (isWiredHeadsetOn()) SoundDevice.HEADSET else SoundDevice.PHONE) + } + } } } fun getAvailableSoundDevices(): List { - return listOf( - if (isHeadsetOn()) SoundDevice.HEADSET else SoundDevice.PHONE, - SoundDevice.SPEAKER - ) + return ArrayList().apply { + if (isBluetoothHeadsetOn()) add(SoundDevice.WIRELESS_HEADSET) + add(if (isWiredHeadsetOn()) SoundDevice.HEADSET else SoundDevice.PHONE) + add(SoundDevice.SPEAKER) + } } fun stop() { Timber.v("## VOIP: AudioManager stopCall") + executor.execute { + // Restore previously stored audio states. + setSpeakerphoneOn(savedIsSpeakerPhoneOn) + setMicrophoneMute(savedIsMicrophoneMute) + audioManager?.mode = savedAudioMode - // Restore previously stored audio states. - setSpeakerphoneOn(savedIsSpeakerPhoneOn) - setMicrophoneMute(savedIsMicrophoneMute) - audioManager.mode = savedAudioMode - - @Suppress("DEPRECATION") - audioManager.abandonAudioFocus(audioFocusChangeListener) + @Suppress("DEPRECATION") + audioManager?.abandonAudioFocus(audioFocusChangeListener) + } } fun getCurrentSoundDevice(): SoundDevice { + val audioManager = audioManager ?: return SoundDevice.PHONE if (audioManager.isSpeakerphoneOn) { return SoundDevice.SPEAKER } else { + if (isBluetoothHeadsetOn() && (wantsBluetoothConnection || audioManager.isBluetoothScoOn)) return SoundDevice.WIRELESS_HEADSET return if (isHeadsetOn()) SoundDevice.HEADSET else SoundDevice.PHONE } } fun setCurrentSoundDevice(device: SoundDevice) { - when (device) { - SoundDevice.HEADSET, - SoundDevice.PHONE -> setSpeakerphoneOn(false) - SoundDevice.SPEAKER -> setSpeakerphoneOn(true) + executor.execute { + Timber.v("## VOIP setCurrentSoundDevice $device") + when (device) { + SoundDevice.HEADSET, + SoundDevice.PHONE -> { + wantsBluetoothConnection = false + if (isBluetoothHeadsetOn()) { + audioManager?.stopBluetoothSco() + audioManager?.isBluetoothScoOn = false + } + setSpeakerphoneOn(false) + } + SoundDevice.SPEAKER -> { + setSpeakerphoneOn(true) + wantsBluetoothConnection = false + audioManager?.stopBluetoothSco() + audioManager?.isBluetoothScoOn = false + } + SoundDevice.WIRELESS_HEADSET -> { + setSpeakerphoneOn(false) + // I cannot directly do it, i have to start then wait that it's connected + // to route to bt + audioManager?.startBluetoothSco() + wantsBluetoothConnection = true + } + } + + configChange?.invoke() + } + } + + fun bluetoothStateChange(plugged: Boolean) { + executor.execute { + if (plugged && wantsBluetoothConnection) { + audioManager?.isBluetoothScoOn = true + } else if (!plugged && !wantsBluetoothConnection) { + audioManager?.stopBluetoothSco() + } + + configChange?.invoke() + } + } + + fun wiredStateChange(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { + executor.execute { + // if it's plugged and speaker is on we should route to headset + if (event.plugged && getCurrentSoundDevice() == SoundDevice.SPEAKER) { + setCurrentSoundDevice(CallAudioManager.SoundDevice.HEADSET) + } else if (!event.plugged) { + // if it's unplugged ? always route to speaker? + // this is questionable? + if (!wantsBluetoothConnection) { + setCurrentSoundDevice(SoundDevice.SPEAKER) + } + } + configChange?.invoke() } } private fun isHeadsetOn(): Boolean { + return isWiredHeadsetOn() || isBluetoothHeadsetOn() + } + + private fun isWiredHeadsetOn(): Boolean { @Suppress("DEPRECATION") - return audioManager.isWiredHeadsetOn || audioManager.isBluetoothScoOn + return audioManager?.isWiredHeadsetOn ?: false + } + + private fun isBluetoothHeadsetOn(): Boolean { + return connectedBlueToothHeadset != null } /** Sets the speaker phone mode. */ private fun setSpeakerphoneOn(on: Boolean) { Timber.v("## VOIP: AudioManager setSpeakerphoneOn $on") - val wasOn = audioManager.isSpeakerphoneOn + val wasOn = audioManager?.isSpeakerphoneOn ?: false if (wasOn == on) { return } - audioManager.isSpeakerphoneOn = on + audioManager?.isSpeakerphoneOn = on } /** Sets the microphone mute state. */ private fun setMicrophoneMute(on: Boolean) { Timber.v("## VOIP: AudioManager setMicrophoneMute $on") - val wasMuted = audioManager.isMicrophoneMute + val wasMuted = audioManager?.isMicrophoneMute ?: false if (wasMuted == on) { return } - audioManager.isMicrophoneMute = on - - audioManager.isMusicActive + audioManager?.isMicrophoneMute = on } /** true if the device has a telephony radio with data diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt index 70c6113160..010a89cf2a 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -55,6 +55,10 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { private fun showSoundDeviceChooser(available: List, current: CallAudioManager.SoundDevice) { val soundDevices = available.map { when (it) { + CallAudioManager.SoundDevice.WIRELESS_HEADSET -> span { + text = getString(R.string.sound_device_wireless_headset) + textStyle = if (current == it) "bold" else "normal" + } CallAudioManager.SoundDevice.PHONE -> span { text = getString(R.string.sound_device_phone) textStyle = if (current == it) "bold" else "normal" @@ -82,6 +86,9 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { getString(R.string.sound_device_headset) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.HEADSET)) } + getString(R.string.sound_device_wireless_headset) -> { + callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.WIRELESS_HEADSET)) + } } } .setNegativeButton(R.string.cancel, null) @@ -104,6 +111,7 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { CallAudioManager.SoundDevice.PHONE -> getString(R.string.sound_device_phone) CallAudioManager.SoundDevice.SPEAKER -> getString(R.string.sound_device_speaker) CallAudioManager.SoundDevice.HEADSET -> getString(R.string.sound_device_headset) + CallAudioManager.SoundDevice.WIRELESS_HEADSET -> getString(R.string.sound_device_wireless_headset) } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 9fbc38d816..57c3f75f55 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -32,6 +32,7 @@ import im.vector.matrix.android.api.session.room.model.call.CallCandidatesConten import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.riotx.core.di.ActiveSessionHolder +import im.vector.riotx.core.services.BluetoothHeadsetReceiver import im.vector.riotx.core.services.CallService import im.vector.riotx.core.services.WiredHeadsetStateReceiver import io.reactivex.disposables.Disposable @@ -88,7 +89,11 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCallsListeners.remove(listener) } - val audioManager = CallAudioManager(context.applicationContext) + val audioManager = CallAudioManager(context.applicationContext) { + currentCallsListeners.forEach { + tryThis { it.onAudioDevicesChange(this) } + } + } data class CallContext( val mxCall: MxCall, @@ -672,19 +677,16 @@ class WebRtcPeerConnectionManager @Inject constructor( close() } - fun onWireDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { + fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { + Timber.v("## VOIP onWiredDeviceEvent $event") currentCall ?: return - // if it's plugged and speaker is on we should route to headset - if (event.plugged && audioManager.getCurrentSoundDevice() == CallAudioManager.SoundDevice.SPEAKER) { - audioManager.setCurrentSoundDevice(CallAudioManager.SoundDevice.HEADSET) - } else if (!event.plugged) { - // if it's unplugged ? always route to speaker? - // this is questionable? - audioManager.setCurrentSoundDevice(CallAudioManager.SoundDevice.SPEAKER) - } - currentCallsListeners.forEach { - it.onAudioDevicesChange(this) - } + // sometimes we received un-wanted unplugged... + audioManager.wiredStateChange(event) + } + + fun onWirelessDeviceEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) { + Timber.v("## VOIP onWirelessDeviceEvent $event") + audioManager.bluetoothStateChange(event.plugged) } override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index 7d67e30a70..a94f6f80d3 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -218,6 +218,7 @@ Phone Speaker Headset + Wireless Headset Send files From 77a01f0cd466208097a8a99a3c02d62e1fb1b3fc Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 18:14:58 +0200 Subject: [PATCH 58/83] lazy create and destroy peer connection factory --- .../call/WebRtcPeerConnectionManager.kt | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 57c3f75f55..f75bbede19 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -182,13 +182,6 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - init { - // TODO do this lazyly - executor.execute { - createPeerConnectionFactory() - } - } - fun headSetButtonTapped() { Timber.v("## VOIP headSetButtonTapped") val call = currentCall?.mxCall ?: return @@ -558,6 +551,12 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun startOutgoingCall(context: Context, signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { + executor.execute { + if (peerConnectionFactory == null) { + createPeerConnectionFactory() + } + } + Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall") val createdCall = sessionHolder.getSafeActiveSession()?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return val callContext = CallContext(createdCall) @@ -607,6 +606,11 @@ class WebRtcPeerConnectionManager @Inject constructor( // Just ignore, maybe we could answer from other session? return } + executor.execute { + if (peerConnectionFactory == null) { + createPeerConnectionFactory() + } + } val callContext = CallContext(mxCall) currentCall = callContext @@ -675,6 +679,11 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall = null audioManager.stop() close() + executor.execute { + if (currentCall == null) { + peerConnectionFactory?.dispose() + } + } } fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { From b27eead016ccc73428c15431ffe4aca8f808a1b8 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 23:05:18 +0200 Subject: [PATCH 59/83] Support toggle front/back camera --- .../features/call/CallControlsBottomSheet.kt | 39 +++--- .../im/vector/riotx/features/call/Cameras.kt | 27 ++++ .../riotx/features/call/VectorCallActivity.kt | 12 +- .../features/call/VectorCallViewModel.kt | 21 ++- .../call/WebRtcPeerConnectionManager.kt | 121 ++++++++++++------ .../src/main/res/drawable/ic_video_flip.xml | 42 ++++++ .../res/layout/bottom_sheet_call_controls.xml | 11 +- vector/src/main/res/values/strings.xml | 3 + 8 files changed, 208 insertions(+), 68 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/features/call/Cameras.kt create mode 100644 vector/src/main/res/drawable/ic_video_flip.xml diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt index 010a89cf2a..e502bcd7b1 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -19,6 +19,7 @@ package im.vector.riotx.features.call import android.os.Bundle import android.view.View import androidx.appcompat.app.AlertDialog +import androidx.core.view.isVisible import com.airbnb.mvrx.activityViewModel import im.vector.riotx.R import im.vector.riotx.core.platform.VectorBaseBottomSheetDialogFragment @@ -41,6 +42,11 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { callViewModel.handle(VectorCallViewActions.SwitchSoundDevice) } + callControlsSwitchCamera.clickableView.debouncedClicks { + callViewModel.handle(VectorCallViewActions.ToggleCamera) + dismiss() + } + callViewModel.observeViewEvents { when (it) { is VectorCallViewEvents.ShowSoundDeviceChooser -> { @@ -55,19 +61,19 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { private fun showSoundDeviceChooser(available: List, current: CallAudioManager.SoundDevice) { val soundDevices = available.map { when (it) { - CallAudioManager.SoundDevice.WIRELESS_HEADSET -> span { + CallAudioManager.SoundDevice.WIRELESS_HEADSET -> span { text = getString(R.string.sound_device_wireless_headset) textStyle = if (current == it) "bold" else "normal" } - CallAudioManager.SoundDevice.PHONE -> span { + CallAudioManager.SoundDevice.PHONE -> span { text = getString(R.string.sound_device_phone) textStyle = if (current == it) "bold" else "normal" } - CallAudioManager.SoundDevice.SPEAKER -> span { + CallAudioManager.SoundDevice.SPEAKER -> span { text = getString(R.string.sound_device_speaker) textStyle = if (current == it) "bold" else "normal" } - CallAudioManager.SoundDevice.HEADSET -> span { + CallAudioManager.SoundDevice.HEADSET -> span { text = getString(R.string.sound_device_headset) textStyle = if (current == it) "bold" else "normal" } @@ -77,13 +83,13 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { .setItems(soundDevices.toTypedArray()) { d, n -> d.cancel() when (soundDevices[n].toString()) { - getString(R.string.sound_device_phone) -> { + getString(R.string.sound_device_phone) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.PHONE)) } - getString(R.string.sound_device_speaker) -> { + getString(R.string.sound_device_speaker) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.SPEAKER)) } - getString(R.string.sound_device_headset) -> { + getString(R.string.sound_device_headset) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.HEADSET)) } getString(R.string.sound_device_wireless_headset) -> { @@ -95,23 +101,16 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { .show() } -// override fun onCreateDialog(savedInstanceState: Bundle?): Dialog { -// return super.onCreateDialog(savedInstanceState).apply { -// window?.addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN) -// window?.decorView?.systemUiVisibility = -// View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or -// View.SYSTEM_UI_FLAG_FULLSCREEN or -// View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY -// } -// } - private fun renderState(state: VectorCallViewState) { callControlsSoundDevice.title = getString(R.string.call_select_sound_device) callControlsSoundDevice.subTitle = when (state.soundDevice) { - CallAudioManager.SoundDevice.PHONE -> getString(R.string.sound_device_phone) - CallAudioManager.SoundDevice.SPEAKER -> getString(R.string.sound_device_speaker) - CallAudioManager.SoundDevice.HEADSET -> getString(R.string.sound_device_headset) + CallAudioManager.SoundDevice.PHONE -> getString(R.string.sound_device_phone) + CallAudioManager.SoundDevice.SPEAKER -> getString(R.string.sound_device_speaker) + CallAudioManager.SoundDevice.HEADSET -> getString(R.string.sound_device_headset) CallAudioManager.SoundDevice.WIRELESS_HEADSET -> getString(R.string.sound_device_wireless_headset) } + + callControlsSwitchCamera.isVisible = state.canSwitchCamera + callControlsSwitchCamera.subTitle = getString(if (state.isFrontCamera) R.string.call_camera_front else R.string.call_camera_back) } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/Cameras.kt b/vector/src/main/java/im/vector/riotx/features/call/Cameras.kt new file mode 100644 index 0000000000..a87dab83c4 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/Cameras.kt @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +enum class CameraType { + FRONT, + BACK +} + +data class CameraProxy( + val name: String, + val type: CameraType +) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 2c1c909a97..e6866ad0b1 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -16,7 +16,6 @@ package im.vector.riotx.features.call -// import im.vector.riotx.features.call.service.CallHeadsUpService import android.app.KeyguardManager import android.content.Context import android.content.Intent @@ -161,13 +160,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis v.updatePadding(bottom = if (systemUiVisibility) insets.systemWindowInsetBottom else 0) insets } -// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { -// // window.navigationBarColor = ContextCompat.getColor(this, R.color.riotx_background_light) -// // } - // for content intent when screen is locked -// window.addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED); -// window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) if (intent.hasExtra(MvRx.KEY_ARG)) { callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! @@ -323,6 +317,10 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() }) + + pipRenderer.setOnClickListener { + callViewModel.handle(VectorCallViewActions.ToggleCamera) + } } override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index e7bc2662dd..baa3ed41fc 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -48,6 +48,8 @@ data class VectorCallViewState( val isAudioMuted: Boolean = false, val isVideoEnabled: Boolean = true, val isVideoCaptureInError: Boolean = false, + val isFrontCamera: Boolean = true, + val canSwitchCamera: Boolean = true, val soundDevice: CallAudioManager.SoundDevice = CallAudioManager.SoundDevice.PHONE, val availableSoundDevices: List = emptyList(), val otherUserMatrixItem: Async = Uninitialized, @@ -62,7 +64,8 @@ sealed class VectorCallViewActions : VectorViewModelAction { object ToggleVideo : VectorCallViewActions() data class ChangeAudioDevice(val device: CallAudioManager.SoundDevice) : VectorCallViewActions() object SwitchSoundDevice : VectorCallViewActions() - object HeadSetButtonPressed: VectorCallViewActions() + object HeadSetButtonPressed : VectorCallViewActions() + object ToggleCamera : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { @@ -140,6 +143,15 @@ class VectorCallViewModel @AssistedInject constructor( ) } } + + override fun onCameraChange(mgr: WebRtcPeerConnectionManager) { + setState { + copy( + canSwitchCamera = mgr.canSwitchCamera(), + isFrontCamera = mgr.currentCameraType() == CameraType.FRONT + ) + } + } } init { @@ -160,7 +172,9 @@ class VectorCallViewModel @AssistedInject constructor( callState = Success(mxCall.state), otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized, soundDevice = webRtcPeerConnectionManager.audioManager.getCurrentSoundDevice(), - availableSoundDevices = webRtcPeerConnectionManager.audioManager.getAvailableSoundDevices() + availableSoundDevices = webRtcPeerConnectionManager.audioManager.getAvailableSoundDevices(), + isFrontCamera = webRtcPeerConnectionManager.currentCameraType() == CameraType.FRONT, + canSwitchCamera = webRtcPeerConnectionManager.canSwitchCamera() ) } } ?: run { @@ -236,6 +250,9 @@ class VectorCallViewModel @AssistedInject constructor( } Unit } + VectorCallViewActions.ToggleCamera -> { + webRtcPeerConnectionManager.switchCamera() + } }.exhaustive } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index f75bbede19..c035767cbb 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -42,6 +42,7 @@ import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.Camera1Enumerator import org.webrtc.Camera2Enumerator +import org.webrtc.CameraVideoCapturer import org.webrtc.DataChannel import org.webrtc.DefaultVideoDecoderFactory import org.webrtc.DefaultVideoEncoderFactory @@ -54,7 +55,6 @@ import org.webrtc.RtpReceiver import org.webrtc.SessionDescription import org.webrtc.SurfaceTextureHelper import org.webrtc.SurfaceViewRenderer -import org.webrtc.VideoCapturer import org.webrtc.VideoSource import org.webrtc.VideoTrack import timber.log.Timber @@ -78,6 +78,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun onCurrentCallChange(call: MxCall?) fun onCaptureStateChanged(captureInError: Boolean) fun onAudioDevicesChange(mgr: WebRtcPeerConnectionManager) {} + fun onCameraChange(mgr: WebRtcPeerConnectionManager) {} } private val currentCallsListeners = emptyList().toMutableList() @@ -159,9 +160,10 @@ class WebRtcPeerConnectionManager @Inject constructor( private var peerConnectionFactory: PeerConnectionFactory? = null -// private var localSdp: SessionDescription? = null + private var videoCapturer: CameraVideoCapturer? = null - private var videoCapturer: VideoCapturer? = null + private val availableCamera = ArrayList() + private var cameraInUse: CameraProxy? = null var capturerIsInError = false set(value) { @@ -413,51 +415,66 @@ class WebRtcPeerConnectionManager @Inject constructor( // add video track if needed if (callContext.mxCall.isVideoCall) { + availableCamera.clear() + val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) + + // I don't realy know how that works if there are 2 front or 2 back cameras val frontCamera = cameraIterator.deviceNames ?.firstOrNull { cameraIterator.isFrontFacing(it) } - ?: cameraIterator.deviceNames?.first() - - // TODO detect when no camera or no front camera - - val videoCapturer = cameraIterator.createCapturer(frontCamera, object : CameraEventsHandlerAdapter() { - override fun onFirstFrameAvailable() { - super.onFirstFrameAvailable() - capturerIsInError = false - } - - override fun onCameraClosed() { - // This could happen if you open the camera app in chat - // We then register in order to restart capture as soon as the camera is available again - Timber.v("## VOIP onCameraClosed") - this@WebRtcPeerConnectionManager.capturerIsInError = true - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - val restarter = CameraRestarter(frontCamera ?: "", callContext.mxCall.callId) - callContext.cameraAvailabilityCallback = restarter - val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager - cameraManager.registerAvailabilityCallback(restarter, null) + ?.let { + CameraProxy(it, CameraType.FRONT).also { availableCamera.add(it) } } - } - }) - val videoSource = peerConnectionFactory!!.createVideoSource(videoCapturer.isScreencast) - val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) - Timber.v("## VOIP Local video source created") + val backCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isBackFacing(it) } + ?.let { + CameraProxy(it, CameraType.BACK).also { availableCamera.add(it) } + } - videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) - // HD - videoCapturer.startCapture(1280, 720, 30) + val camera = frontCamera?.also { cameraInUse = frontCamera } + ?: backCamera?.also { cameraInUse = backCamera } + ?: null.also { cameraInUse = null } - this.videoCapturer = videoCapturer + if (camera != null) { + val videoCapturer = cameraIterator.createCapturer(camera.name, object : CameraEventsHandlerAdapter() { + override fun onFirstFrameAvailable() { + super.onFirstFrameAvailable() + capturerIsInError = false + } - val localVideoTrack = peerConnectionFactory!!.createVideoTrack("ARDAMSv0", videoSource) - Timber.v("## VOIP Local video track created") - localVideoTrack?.setEnabled(true) + override fun onCameraClosed() { + // This could happen if you open the camera app in chat + // We then register in order to restart capture as soon as the camera is available again + Timber.v("## VOIP onCameraClosed") + this@WebRtcPeerConnectionManager.capturerIsInError = true + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val restarter = CameraRestarter(cameraInUse?.name ?: "", callContext.mxCall.callId) + callContext.cameraAvailabilityCallback = restarter + val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + cameraManager.registerAvailabilityCallback(restarter, null) + } + } + }) - callContext.localVideoSource = videoSource - callContext.localVideoTrack = localVideoTrack + val videoSource = peerConnectionFactory!!.createVideoSource(videoCapturer.isScreencast) + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + Timber.v("## VOIP Local video source created") - localMediaStream?.addTrack(localVideoTrack) + videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) + // HD + videoCapturer.startCapture(1280, 720, 30) + this.videoCapturer = videoCapturer + + val localVideoTrack = peerConnectionFactory!!.createVideoTrack("ARDAMSv0", videoSource) + Timber.v("## VOIP Local video track created") + localVideoTrack?.setEnabled(true) + + callContext.localVideoSource = videoSource + callContext.localVideoTrack = localVideoTrack + + localMediaStream?.addTrack(localVideoTrack) + } } } @@ -661,6 +678,34 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall?.localVideoTrack?.setEnabled(enabled) } + fun switchCamera() { + Timber.v("## VOIP switchCamera") + if (currentCall != null && currentCall?.mxCall?.state is CallState.Connected && currentCall?.mxCall?.isVideoCall == true) { + videoCapturer?.switchCamera(object : CameraVideoCapturer.CameraSwitchHandler { + // Invoked on success. |isFrontCamera| is true if the new camera is front facing. + override fun onCameraSwitchDone(isFrontCamera: Boolean) { + Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera") + cameraInUse = availableCamera.first { if (isFrontCamera) it.type == CameraType.FRONT else it.type == CameraType.BACK } + currentCallsListeners.forEach { + tryThis { it.onCameraChange(this@WebRtcPeerConnectionManager) } + } + } + + override fun onCameraSwitchError(errorDescription: String?) { + Timber.v("## VOIP onCameraSwitchError isFront $errorDescription") + } + }) + } + } + + fun canSwitchCamera(): Boolean { + return availableCamera.size > 0 + } + + fun currentCameraType(): CameraType? { + return cameraInUse?.type + } + fun endCall() { // Update service state CallService.onNoActiveCall(context) diff --git a/vector/src/main/res/drawable/ic_video_flip.xml b/vector/src/main/res/drawable/ic_video_flip.xml new file mode 100644 index 0000000000..0cc540b9fb --- /dev/null +++ b/vector/src/main/res/drawable/ic_video_flip.xml @@ -0,0 +1,42 @@ + + + + + + + diff --git a/vector/src/main/res/layout/bottom_sheet_call_controls.xml b/vector/src/main/res/layout/bottom_sheet_call_controls.xml index 0c17bcb049..69d06ddb08 100644 --- a/vector/src/main/res/layout/bottom_sheet_call_controls.xml +++ b/vector/src/main/res/layout/bottom_sheet_call_controls.xml @@ -11,9 +11,18 @@ android:id="@+id/callControlsSoundDevice" android:layout_width="match_parent" android:layout_height="wrap_content" - tools:actionTitle="Select sound device" + app:actionTitle="@string/call_select_sound_device" tools:actionDescription="Speaker" app:leftIcon="@drawable/ic_call_speaker_default" app:tint="?attr/riotx_text_primary" /> + + diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index a94f6f80d3..dbacf12caf 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -219,6 +219,9 @@ Speaker Headset Wireless Headset + Switch Camera + Front + Back Send files From 25fe56116cd826ccf64eb0ca2ce0154f4e92b2ba Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 23:37:54 +0200 Subject: [PATCH 60/83] Ask for permission before starting call --- .../call/WebRtcPeerConnectionManager.kt | 6 +- .../home/room/detail/RoomDetailAction.kt | 2 +- .../home/room/detail/RoomDetailFragment.kt | 57 +++++++++++++++---- .../home/room/detail/RoomDetailViewModel.kt | 15 ++++- 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index c035767cbb..9f801d1acf 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -567,7 +567,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun startOutgoingCall(context: Context, signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { + fun startOutgoingCall(signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { executor.execute { if (peerConnectionFactory == null) { createPeerConnectionFactory() @@ -584,7 +584,7 @@ class WebRtcPeerConnectionManager @Inject constructor( val name = sessionHolder.getSafeActiveSession()?.getUser(createdCall.otherUserId)?.getBestName() ?: createdCall.otherUserId CallService.onOutgoingCallRinging( - context = context, + context = context.applicationContext, isVideo = createdCall.isVideoCall, roomName = name, roomId = createdCall.roomId, @@ -596,7 +596,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } // start the activity now - context.startActivity(VectorCallActivity.newIntent(context, createdCall)) + context.applicationContext.startActivity(VectorCallActivity.newIntent(context, createdCall)) } override fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt index 896b29009e..271917c827 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt @@ -68,7 +68,7 @@ sealed class RoomDetailAction : VectorViewModelAction { object ClearSendQueue : RoomDetailAction() object ResendAll : RoomDetailAction() - object StartCall : RoomDetailAction() + data class StartCall(val isVideo: Boolean) : RoomDetailAction() data class AcceptVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() data class DeclineVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index 0f8c98128f..8aded52717 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -107,6 +107,8 @@ import im.vector.riotx.core.ui.views.JumpToReadMarkerView import im.vector.riotx.core.ui.views.NotificationAreaView import im.vector.riotx.core.utils.Debouncer import im.vector.riotx.core.utils.KeyboardStateUtils +import im.vector.riotx.core.utils.PERMISSIONS_FOR_AUDIO_IP_CALL +import im.vector.riotx.core.utils.PERMISSIONS_FOR_VIDEO_IP_CALL import im.vector.riotx.core.utils.PERMISSIONS_FOR_WRITING_FILES import im.vector.riotx.core.utils.PERMISSION_REQUEST_CODE_DOWNLOAD_FILE import im.vector.riotx.core.utils.PERMISSION_REQUEST_CODE_INCOMING_URI @@ -120,6 +122,8 @@ import im.vector.riotx.core.utils.createJSonViewerStyleProvider import im.vector.riotx.core.utils.createUIHandler import im.vector.riotx.core.utils.getColorFromUserId import im.vector.riotx.core.utils.isValidUrl +import im.vector.riotx.core.utils.onPermissionResultAudioIpCall +import im.vector.riotx.core.utils.onPermissionResultVideoIpCall import im.vector.riotx.core.utils.openUrlInExternalBrowser import im.vector.riotx.core.utils.saveMedia import im.vector.riotx.core.utils.shareMedia @@ -132,7 +136,6 @@ import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs import im.vector.riotx.features.attachments.toGroupedContentAttachmentData import im.vector.riotx.features.call.SharedActiveCallViewModel import im.vector.riotx.features.call.VectorCallActivity -import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.command.Command import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity import im.vector.riotx.features.crypto.util.toImageRes @@ -202,8 +205,7 @@ class RoomDetailFragment @Inject constructor( val roomDetailViewModelFactory: RoomDetailViewModel.Factory, private val eventHtmlRenderer: EventHtmlRenderer, private val vectorPreferences: VectorPreferences, - private val colorProvider: ColorProvider, - private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager) : + private val colorProvider: ColorProvider) : VectorBaseFragment(), TimelineEventController.Callback, VectorInviteView.Callback, @@ -215,6 +217,9 @@ class RoomDetailFragment @Inject constructor( companion object { + private const val AUDIO_CALL_PERMISSION_REQUEST_CODE = 1 + private const val VIDEO_CALL_PERMISSION_REQUEST_CODE = 2 + /** * Sanitize the display name. * @@ -503,22 +508,22 @@ class RoomDetailFragment @Inject constructor( } R.id.voice_call, R.id.video_call -> { - roomDetailViewModel.getOtherUserIds()?.firstOrNull()?.let { // TODO CALL We should check/ask for permission here first val activeCall = sharedCallActionViewModel.activeCall.value + val isVideoCall = item.itemId == R.id.video_call if (activeCall != null) { // resume existing if same room, if not prompt to kill and then restart new call? if (activeCall.roomId == roomDetailArgs.roomId) { onTapToReturnToCall() - } else { - // TODO might not work well, and should prompt - webRtcPeerConnectionManager.endCall() - webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) } +// else { + // TODO might not work well, and should prompt +// webRtcPeerConnectionManager.endCall() +// safeStartCall(it, isVideoCall) +// } } else { - webRtcPeerConnectionManager.startOutgoingCall(requireContext(), roomDetailArgs.roomId, it, item.itemId == R.id.video_call) + safeStartCall(isVideoCall) } - } true } else -> super.onOptionsItemSelected(item) @@ -536,6 +541,22 @@ class RoomDetailFragment @Inject constructor( .show() } + private fun safeStartCall(isVideoCall: Boolean) { + val startCallAction = RoomDetailAction.StartCall(isVideoCall) + roomDetailViewModel.pendingAction = startCallAction + if (isVideoCall) { + if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, VIDEO_CALL_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { + roomDetailViewModel.pendingAction = null + roomDetailViewModel.handle(startCallAction) + } + } else { + if (checkPermissions(PERMISSIONS_FOR_AUDIO_IP_CALL, this, AUDIO_CALL_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_record_audio)) { + roomDetailViewModel.pendingAction = null + roomDetailViewModel.handle(startCallAction) + } + } + } + private fun renderRegularMode(text: String) { autoCompleter.exitSpecialMode() composerLayout.collapse() @@ -1130,6 +1151,22 @@ class RoomDetailFragment @Inject constructor( launchAttachmentProcess(pendingType) } } + AUDIO_CALL_PERMISSION_REQUEST_CODE -> { + if (onPermissionResultAudioIpCall(requireContext(), grantResults)) { + (roomDetailViewModel.pendingAction as? RoomDetailAction.StartCall)?.let { + roomDetailViewModel.pendingAction = null + roomDetailViewModel.handle(it) + } + } + } + VIDEO_CALL_PERMISSION_REQUEST_CODE -> { + if (onPermissionResultVideoIpCall(requireContext(), grantResults)) { + (roomDetailViewModel.pendingAction as? RoomDetailAction.StartCall)?.let { + roomDetailViewModel.pendingAction = null + roomDetailViewModel.handle(it) + } + } + } } } else { // Reset all pending data diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index 0894fbf004..dbbe35592f 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -67,6 +67,7 @@ import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.resources.StringProvider import im.vector.riotx.core.resources.UserPreferencesProvider import im.vector.riotx.core.utils.subscribeLogError +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.command.CommandParser import im.vector.riotx.features.command.ParsedCommand import im.vector.riotx.features.crypto.verification.SupportedVerificationMethodsProvider @@ -97,7 +98,8 @@ class RoomDetailViewModel @AssistedInject constructor( private val rainbowGenerator: RainbowGenerator, private val session: Session, private val supportedVerificationMethodsProvider: SupportedVerificationMethodsProvider, - private val stickerPickerActionHandler: StickerPickerActionHandler + private val stickerPickerActionHandler: StickerPickerActionHandler, + private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager ) : VectorViewModel(initialState), Timeline.Listener { private val room = session.getRoom(initialState.roomId)!! @@ -255,13 +257,20 @@ class RoomDetailViewModel @AssistedInject constructor( is RoomDetailAction.ResumeVerification -> handleResumeRequestVerification(action) is RoomDetailAction.ReRequestKeys -> handleReRequestKeys(action) is RoomDetailAction.SelectStickerAttachment -> handleSelectStickerAttachment() - } + is RoomDetailAction.StartCall -> handleStartCall(action) + }.exhaustive } private fun handleSendSticker(action: RoomDetailAction.SendSticker) { room.sendEvent(EventType.STICKER, action.stickerContent.toContent()) } + private fun handleStartCall(action: RoomDetailAction.StartCall) { + room.roomSummary()?.otherMemberIds?.firstOrNull()?.let { + webRtcPeerConnectionManager.startOutgoingCall(room.roomId, it, action.isVideo) + } + } + private fun handleSelectStickerAttachment() { viewModelScope.launch { val viewEvent = stickerPickerActionHandler.handle() @@ -369,7 +378,7 @@ class RoomDetailViewModel @AssistedInject constructor( } fun isMenuItemVisible(@IdRes itemId: Int) = when (itemId) { - R.id.clear_message_queue -> + R.id.clear_message_queue -> // For now always disable when not in developer mode, worker cancellation is not working properly timeline.pendingEventCount() > 0 && vectorPreferences.developerMode() R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 From 666f3ea152c71b90fcf0b8fede85d90042079cb3 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 18 Jun 2020 23:40:21 +0200 Subject: [PATCH 61/83] code quality --- .../im/vector/riotx/features/call/CallAudioManager.kt | 4 +--- .../riotx/features/home/room/detail/RoomDetailFragment.kt | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt index 9290d9a3e8..9bc680ae96 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -38,9 +38,7 @@ class CallAudioManager( WIRELESS_HEADSET } - /* - * if all calls to audio manager not in the same thread it's not working well... - */ + // if all calls to audio manager not in the same thread it's not working well. private val executor = Executors.newSingleThreadExecutor() private var audioManager: AudioManager? = null diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index 8aded52717..7b6a3072f9 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -545,12 +545,16 @@ class RoomDetailFragment @Inject constructor( val startCallAction = RoomDetailAction.StartCall(isVideoCall) roomDetailViewModel.pendingAction = startCallAction if (isVideoCall) { - if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, VIDEO_CALL_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { + if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, + this, VIDEO_CALL_PERMISSION_REQUEST_CODE, + R.string.permissions_rationale_msg_camera_and_audio)) { roomDetailViewModel.pendingAction = null roomDetailViewModel.handle(startCallAction) } } else { - if (checkPermissions(PERMISSIONS_FOR_AUDIO_IP_CALL, this, AUDIO_CALL_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_record_audio)) { + if (checkPermissions(PERMISSIONS_FOR_AUDIO_IP_CALL, + this, AUDIO_CALL_PERMISSION_REQUEST_CODE, + R.string.permissions_rationale_msg_record_audio)) { roomDetailViewModel.pendingAction = null roomDetailViewModel.handle(startCallAction) } From f3e2a55869d2d866a67c0595315d11f344cb4b71 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 10:05:51 +0200 Subject: [PATCH 62/83] Crash Fix / nullify factory after dispose --- .../im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 9f801d1acf..232f679828 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -727,6 +727,7 @@ class WebRtcPeerConnectionManager @Inject constructor( executor.execute { if (currentCall == null) { peerConnectionFactory?.dispose() + peerConnectionFactory = null } } } From 374790176f8b0389fc728a0eb9901d8a0ce5757c Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 10:11:45 +0200 Subject: [PATCH 63/83] Toggle HD/SD --- .../features/call/CallControlsBottomSheet.kt | 23 ++++++++++++++++ .../im/vector/riotx/features/call/Cameras.kt | 5 ++++ .../call/SharedActiveCallViewModel.kt | 12 +++------ .../features/call/VectorCallViewModel.kt | 16 ++++++++--- .../call/WebRtcPeerConnectionManager.kt | 27 +++++++++++++++---- vector/src/main/res/drawable/ic_hd.xml | 9 +++++++ .../src/main/res/drawable/ic_hd_disabled.xml | 9 +++++++ .../res/layout/bottom_sheet_call_controls.xml | 9 +++++++ vector/src/main/res/values/strings.xml | 3 +++ 9 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 vector/src/main/res/drawable/ic_hd.xml create mode 100644 vector/src/main/res/drawable/ic_hd_disabled.xml diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt index e502bcd7b1..e230592acf 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -19,11 +19,14 @@ package im.vector.riotx.features.call import android.os.Bundle import android.view.View import androidx.appcompat.app.AlertDialog +import androidx.core.content.ContextCompat import androidx.core.view.isVisible import com.airbnb.mvrx.activityViewModel import im.vector.riotx.R import im.vector.riotx.core.platform.VectorBaseBottomSheetDialogFragment +import kotlinx.android.synthetic.main.activity_call.* import kotlinx.android.synthetic.main.bottom_sheet_call_controls.* +import kotlinx.android.synthetic.main.vector_preference_push_rule.view.* import me.gujun.android.span.span class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { @@ -47,6 +50,11 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { dismiss() } + callControlsToggleSDHD.clickableView.debouncedClicks { + callViewModel.handle(VectorCallViewActions.ToggleHDSD) + dismiss() + } + callViewModel.observeViewEvents { when (it) { is VectorCallViewEvents.ShowSoundDeviceChooser -> { @@ -112,5 +120,20 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { callControlsSwitchCamera.isVisible = state.canSwitchCamera callControlsSwitchCamera.subTitle = getString(if (state.isFrontCamera) R.string.call_camera_front else R.string.call_camera_back) + + if (state.isVideoCall) { + callControlsToggleSDHD.isVisible = true + if (state.isHD) { + callControlsToggleSDHD.title = getString(R.string.call_format_turn_hd_off) + callControlsToggleSDHD.subTitle = null + callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd) + } else { + callControlsToggleSDHD.title = getString(R.string.call_format_turn_hd_on) + callControlsToggleSDHD.subTitle = null + callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd_disabled) + } + } else { + callControlsToggleSDHD.isVisible = false + } } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/Cameras.kt b/vector/src/main/java/im/vector/riotx/features/call/Cameras.kt index a87dab83c4..07d563ca9c 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/Cameras.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/Cameras.kt @@ -25,3 +25,8 @@ data class CameraProxy( val name: String, val type: CameraType ) + +sealed class CaptureFormat(val width: Int, val height: Int, val fps: Int) { + object HD : CaptureFormat(1280, 720, 30) + object SD : CaptureFormat(640, 480, 30) +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt index 22becc82f5..4900d39a32 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt @@ -33,12 +33,12 @@ class SharedActiveCallViewModel @Inject constructor( val activeCall: MutableLiveData = MutableLiveData() - val callStateListener = object: MxCall.StateListener { + val callStateListener = object : MxCall.StateListener { override fun onStateUpdate(call: MxCall) { - if (activeCall.value?.callId == call.callId) { - activeCall.postValue(call) - } + if (activeCall.value?.callId == call.callId) { + activeCall.postValue(call) + } } } @@ -48,10 +48,6 @@ class SharedActiveCallViewModel @Inject constructor( activeCall.postValue(call) call?.addListener(callStateListener) } - - override fun onCaptureStateChanged(captureInError: Boolean) { - // nop - } } init { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index baa3ed41fc..5c832f8f40 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -48,6 +48,7 @@ data class VectorCallViewState( val isAudioMuted: Boolean = false, val isVideoEnabled: Boolean = true, val isVideoCaptureInError: Boolean = false, + val isHD: Boolean = false, val isFrontCamera: Boolean = true, val canSwitchCamera: Boolean = true, val soundDevice: CallAudioManager.SoundDevice = CallAudioManager.SoundDevice.PHONE, @@ -66,6 +67,7 @@ sealed class VectorCallViewActions : VectorViewModelAction { object SwitchSoundDevice : VectorCallViewActions() object HeadSetButtonPressed : VectorCallViewActions() object ToggleCamera : VectorCallViewActions() + object ToggleHDSD : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { @@ -129,9 +131,12 @@ class VectorCallViewModel @AssistedInject constructor( override fun onCurrentCallChange(call: MxCall?) { } - override fun onCaptureStateChanged(captureInError: Boolean) { + override fun onCaptureStateChanged(mgr: WebRtcPeerConnectionManager) { setState { - copy(isVideoCaptureInError = captureInError) + copy( + isVideoCaptureInError = mgr.capturerIsInError, + isHD = mgr.currentCaptureFormat() is CaptureFormat.HD + ) } } @@ -174,7 +179,8 @@ class VectorCallViewModel @AssistedInject constructor( soundDevice = webRtcPeerConnectionManager.audioManager.getCurrentSoundDevice(), availableSoundDevices = webRtcPeerConnectionManager.audioManager.getAvailableSoundDevices(), isFrontCamera = webRtcPeerConnectionManager.currentCameraType() == CameraType.FRONT, - canSwitchCamera = webRtcPeerConnectionManager.canSwitchCamera() + canSwitchCamera = webRtcPeerConnectionManager.canSwitchCamera(), + isHD = mxCall.isVideoCall && webRtcPeerConnectionManager.currentCaptureFormat() is CaptureFormat.HD ) } } ?: run { @@ -253,6 +259,10 @@ class VectorCallViewModel @AssistedInject constructor( VectorCallViewActions.ToggleCamera -> { webRtcPeerConnectionManager.switchCamera() } + VectorCallViewActions.ToggleHDSD -> { + if (!state.isVideoCall) return@withState + webRtcPeerConnectionManager.setCaptureFormat(if (state.isHD) CaptureFormat.SD else CaptureFormat.HD) + } }.exhaustive } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 232f679828..215e5928f0 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -76,7 +76,7 @@ class WebRtcPeerConnectionManager @Inject constructor( interface CurrentCallListener { fun onCurrentCallChange(call: MxCall?) - fun onCaptureStateChanged(captureInError: Boolean) + fun onCaptureStateChanged(mgr: WebRtcPeerConnectionManager) {} fun onAudioDevicesChange(mgr: WebRtcPeerConnectionManager) {} fun onCameraChange(mgr: WebRtcPeerConnectionManager) {} } @@ -165,11 +165,13 @@ class WebRtcPeerConnectionManager @Inject constructor( private val availableCamera = ArrayList() private var cameraInUse: CameraProxy? = null + private var currentCaptureMode: CaptureFormat = CaptureFormat.HD + var capturerIsInError = false set(value) { field = value currentCallsListeners.forEach { - tryThis { it.onCaptureStateChanged(value) } + tryThis { it.onCaptureStateChanged(this) } } } @@ -336,7 +338,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // Fallback for old android, try to restart capture when attached if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP && capturerIsInError && call.mxCall.isVideoCall) { // try to restart capture? - videoCapturer?.startCapture(1280, 720, 30) + videoCapturer?.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps) } // sink existing tracks (configuration change, e.g screen rotation) attachViewRenderersInternal() @@ -463,7 +465,7 @@ class WebRtcPeerConnectionManager @Inject constructor( videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) // HD - videoCapturer.startCapture(1280, 720, 30) + videoCapturer.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps) this.videoCapturer = videoCapturer val localVideoTrack = peerConnectionFactory!!.createVideoTrack("ARDAMSv0", videoSource) @@ -706,6 +708,21 @@ class WebRtcPeerConnectionManager @Inject constructor( return cameraInUse?.type } + fun setCaptureFormat(format: CaptureFormat) { + Timber.v("## VOIP setCaptureFormat $format") + currentCall ?: return + executor.execute { + // videoCapturer?.stopCapture() + videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps) + currentCaptureMode = format + currentCallsListeners.forEach { tryThis { it.onCaptureStateChanged(this) } } + } + } + + fun currentCaptureFormat(): CaptureFormat { + return currentCaptureMode + } + fun endCall() { // Update service state CallService.onNoActiveCall(context) @@ -971,7 +988,7 @@ class WebRtcPeerConnectionManager @Inject constructor( if (this.cameraId == cameraId && currentCall?.mxCall?.callId == callId) { // re-start the capture // TODO notify that video is enabled - videoCapturer?.startCapture(1280, 720, 30) + videoCapturer?.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps) (context.getSystemService(Context.CAMERA_SERVICE) as? CameraManager) ?.unregisterAvailabilityCallback(this) } diff --git a/vector/src/main/res/drawable/ic_hd.xml b/vector/src/main/res/drawable/ic_hd.xml new file mode 100644 index 0000000000..3335724529 --- /dev/null +++ b/vector/src/main/res/drawable/ic_hd.xml @@ -0,0 +1,9 @@ + + + diff --git a/vector/src/main/res/drawable/ic_hd_disabled.xml b/vector/src/main/res/drawable/ic_hd_disabled.xml new file mode 100644 index 0000000000..6396b7bc7e --- /dev/null +++ b/vector/src/main/res/drawable/ic_hd_disabled.xml @@ -0,0 +1,9 @@ + + + diff --git a/vector/src/main/res/layout/bottom_sheet_call_controls.xml b/vector/src/main/res/layout/bottom_sheet_call_controls.xml index 69d06ddb08..275d50f457 100644 --- a/vector/src/main/res/layout/bottom_sheet_call_controls.xml +++ b/vector/src/main/res/layout/bottom_sheet_call_controls.xml @@ -25,4 +25,13 @@ app:leftIcon="@drawable/ic_video_flip" app:tint="?attr/riotx_text_primary" /> + + diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index dbacf12caf..caabd55651 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -222,6 +222,9 @@ Switch Camera Front Back + Turn HD off + Turn HD on + Send files From 99056a7807b201adeb8238f2830368a0b0179f14 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 11:58:28 +0200 Subject: [PATCH 64/83] Fix / inversed icons HD/SD --- .../im/vector/riotx/features/call/CallControlsBottomSheet.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt index e230592acf..2eaafef2d7 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -126,11 +126,11 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { if (state.isHD) { callControlsToggleSDHD.title = getString(R.string.call_format_turn_hd_off) callControlsToggleSDHD.subTitle = null - callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd) + callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd_disabled) } else { callControlsToggleSDHD.title = getString(R.string.call_format_turn_hd_on) callControlsToggleSDHD.subTitle = null - callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd_disabled) + callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd) } } else { callControlsToggleSDHD.isVisible = false From 96ecb1d07e6734653673a9592ea0f085692e32d2 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 11:59:03 +0200 Subject: [PATCH 65/83] Fix Crash / stop capture in wrong thread --- .../call/WebRtcPeerConnectionManager.kt | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 215e5928f0..9f60fcb568 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -537,12 +537,25 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun close() { + Timber.v("## VOIP WebRtcPeerConnectionManager close() >") CallService.onNoActiveCall(context) + audioManager.stop() + val callToEnd = currentCall + currentCall = null + // This must be done in this thread + videoCapturer?.stopCapture() + videoCapturer?.dispose() + videoCapturer = null executor.execute { - currentCall?.release() - videoCapturer?.stopCapture() - videoCapturer?.dispose() - videoCapturer = null + callToEnd?.release() + + if (currentCall == null) { + Timber.v("## VOIP Dispose peerConnectionFactory as there is no need to keep one") + peerConnectionFactory?.dispose() + peerConnectionFactory = null + } + + Timber.v("## VOIP WebRtcPeerConnectionManager close() executor done") } } @@ -712,7 +725,7 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP setCaptureFormat $format") currentCall ?: return executor.execute { - // videoCapturer?.stopCapture() + // videoCapturer?.stopCapture() videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps) currentCaptureMode = format currentCallsListeners.forEach { tryThis { it.onCaptureStateChanged(this) } } @@ -723,7 +736,7 @@ class WebRtcPeerConnectionManager @Inject constructor( return currentCaptureMode } - fun endCall() { + fun endCall(originatedByMe: Boolean = true) { // Update service state CallService.onNoActiveCall(context) // close tracks ASAP @@ -737,16 +750,11 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - currentCall?.mxCall?.hangUp() - currentCall = null - audioManager.stop() - close() - executor.execute { - if (currentCall == null) { - peerConnectionFactory?.dispose() - peerConnectionFactory = null - } + if (originatedByMe) { + // send hang up event + currentCall?.mxCall?.hangUp() } + close() } fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { @@ -788,15 +796,16 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onCallHangupReceived(callHangupContent: CallHangupContent) { val call = currentCall ?: return + // Remote echos are filtered, so it's only remote hangups that i will get here if (call.mxCall.callId != callHangupContent.callId) return Unit.also { Timber.w("onCallHangupReceived for non active call? ${callHangupContent.callId}") } call.mxCall.state = CallState.Terminated - currentCall = null - close() + endCall(false) } override fun onCallManagedByOtherSession(callId: String) { + Timber.v("## VOIP onCallManagedByOtherSession: $callId") currentCall = null CallService.onNoActiveCall(context) } From 30dee07a3ba9eddf496c64f51a4da45374ffe269 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 12:32:29 +0200 Subject: [PATCH 66/83] Hide switch camera for voice call --- .../im/vector/riotx/features/call/CallControlsBottomSheet.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt index 2eaafef2d7..3d86488720 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -118,7 +118,7 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { CallAudioManager.SoundDevice.WIRELESS_HEADSET -> getString(R.string.sound_device_wireless_headset) } - callControlsSwitchCamera.isVisible = state.canSwitchCamera + callControlsSwitchCamera.isVisible = state.isVideoCall && state.canSwitchCamera callControlsSwitchCamera.subTitle = getString(if (state.isFrontCamera) R.string.call_camera_front else R.string.call_camera_back) if (state.isVideoCall) { From 2a3d20d300b3fd803975ab7e7f858e0dec1053df Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 13:42:35 +0200 Subject: [PATCH 67/83] FIx rebase --- vector/src/main/res/layout/fragment_room_detail.xml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/vector/src/main/res/layout/fragment_room_detail.xml b/vector/src/main/res/layout/fragment_room_detail.xml index 8cd19b8a4c..70044ec6f6 100644 --- a/vector/src/main/res/layout/fragment_room_detail.xml +++ b/vector/src/main/res/layout/fragment_room_detail.xml @@ -98,12 +98,12 @@ app:layout_constraintTop_toBottomOf="@id/roomToolbar" /> + android:visibility="gone" + app:layout_constraintTop_toBottomOf="@id/syncStateView" + tools:visibility="visible" /> Date: Fri, 19 Jun 2020 15:20:10 +0200 Subject: [PATCH 68/83] hang up menu action --- .../features/home/room/detail/RoomDetailAction.kt | 1 + .../home/room/detail/RoomDetailFragment.kt | 6 ++++++ .../home/room/detail/RoomDetailViewModel.kt | 14 ++++++++++---- vector/src/main/res/menu/menu_timeline.xml | 8 ++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt index 271917c827..6911ca84f6 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt @@ -69,6 +69,7 @@ sealed class RoomDetailAction : VectorViewModelAction { object ClearSendQueue : RoomDetailAction() object ResendAll : RoomDetailAction() data class StartCall(val isVideo: Boolean) : RoomDetailAction() + object EndCall : RoomDetailAction() data class AcceptVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() data class DeclineVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index 7b6a3072f9..d46227ef7c 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -303,6 +303,7 @@ class RoomDetailFragment @Inject constructor( .observe(viewLifecycleOwner, Observer { val hasActiveCall = it?.state is CallState.Connected activeCallView.isVisible = hasActiveCall + invalidateOptionsMenu() }) roomDetailViewModel.selectSubscribe(this, RoomDetailViewState::tombstoneEventHandling, uniqueOnly("tombstoneEventHandling")) { @@ -526,6 +527,10 @@ class RoomDetailFragment @Inject constructor( } true } + R.id.hangup_call -> { + roomDetailViewModel.handle(RoomDetailAction.EndCall) + true + } else -> super.onOptionsItemSelected(item) } } @@ -804,6 +809,7 @@ class RoomDetailFragment @Inject constructor( override fun invalidate() = withState(roomDetailViewModel) { state -> renderRoomSummary(state) + invalidateOptionsMenu() val summary = state.asyncRoomSummary() val inviter = state.asyncInviter() if (summary?.membership == Membership.JOIN) { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index dbbe35592f..42b7d67f2a 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -258,6 +258,7 @@ class RoomDetailViewModel @AssistedInject constructor( is RoomDetailAction.ReRequestKeys -> handleReRequestKeys(action) is RoomDetailAction.SelectStickerAttachment -> handleSelectStickerAttachment() is RoomDetailAction.StartCall -> handleStartCall(action) + is RoomDetailAction.EndCall -> handleEndCall() }.exhaustive } @@ -271,6 +272,10 @@ class RoomDetailViewModel @AssistedInject constructor( } } + private fun handleEndCall() { + webRtcPeerConnectionManager.endCall() + } + private fun handleSelectStickerAttachment() { viewModelScope.launch { val viewEvent = stickerPickerActionHandler.handle() @@ -381,10 +386,11 @@ class RoomDetailViewModel @AssistedInject constructor( R.id.clear_message_queue -> // For now always disable when not in developer mode, worker cancellation is not working properly timeline.pendingEventCount() > 0 && vectorPreferences.developerMode() - R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 - R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 - R.id.open_matrix_apps -> true - R.id.voice_call, R.id.video_call -> room.canStartCall() + R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 + R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 + R.id.open_matrix_apps -> true + R.id.voice_call, R.id.video_call -> room.canStartCall() && webRtcPeerConnectionManager.currentCall == null + R.id.hangup_call -> webRtcPeerConnectionManager.currentCall != null else -> false } diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index 701d02c3ea..a6666d33ae 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -26,6 +26,14 @@ app:showAsAction="always" tools:visible="true" /> + Date: Fri, 19 Jun 2020 15:20:26 +0200 Subject: [PATCH 69/83] Add connection loader --- .../riotx/features/call/VectorCallActivity.kt | 3 +++ vector/src/main/res/layout/activity_call.xml | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index e6866ad0b1..1921dfe3da 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -224,6 +224,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callControlsView.updateForState(state) val callState = state.callState.invoke() + callConnectingProgress.isVisible = false when (callState) { is CallState.Idle, is CallState.Dialing -> { @@ -244,6 +245,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.setText(R.string.call_connecting) + callConnectingProgress.isVisible = true configureCallInfo(state) } is CallState.Connected -> { @@ -264,6 +266,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callInfoGroup.isVisible = true configureCallInfo(state) callStatusText.setText(R.string.call_connecting) + callConnectingProgress.isVisible = true } // ensure all attached? peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null) diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index 9b27bef6ac..d05ac9d0d2 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -3,9 +3,9 @@ @@ -73,6 +73,18 @@ app:layout_constraintTop_toBottomOf="@id/otherMemberAvatar" tools:text="Connecting..." /> + + Date: Fri, 19 Jun 2020 17:34:05 +0200 Subject: [PATCH 70/83] Missing release of webrtc surfaces --- .../im/vector/riotx/features/call/VectorCallActivity.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 1921dfe3da..a1247c4390 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -103,6 +103,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis var systemUiVisibility = false + var surfaceRenderersAreInitialized = false + override fun doBeforeSetContentView() { // Set window styles for fullscreen-window size. Needs to be done before adding content. requestWindowFeature(Window.FEATURE_NO_TITLE) @@ -208,8 +210,12 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } override fun onDestroy() { - super.onDestroy() peerConnectionManager.detachRenderers() + if (surfaceRenderersAreInitialized) { + pipRenderer.release() + fullscreenRenderer.release() + } + super.onDestroy() turnScreenOffAndKeyguardOn() } @@ -324,6 +330,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis pipRenderer.setOnClickListener { callViewModel.handle(VectorCallViewActions.ToggleCamera) } + surfaceRenderersAreInitialized = true } override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { From 17cf3fd7ad7a62c093a791a868cca0cafe5fe629 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 18:54:39 +0200 Subject: [PATCH 71/83] Active call (with PIP) , in Room and Home --- .../core/ui/views/ActiveCallViewHolder.kt | 98 ++++++++++++++ .../riotx/features/call/VectorCallActivity.kt | 2 +- .../call/WebRtcPeerConnectionManager.kt | 124 ++++++++++++------ .../riotx/features/home/HomeDetailFragment.kt | 52 +++++++- .../home/room/detail/RoomDetailFragment.kt | 18 ++- .../main/res/layout/fragment_home_detail.xml | 32 ++++- .../main/res/layout/fragment_room_detail.xml | 21 +++ 7 files changed, 298 insertions(+), 49 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt diff --git a/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt new file mode 100644 index 0000000000..70a66e7556 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.core.ui.views + +import android.view.View +import androidx.cardview.widget.CardView +import androidx.core.view.isVisible +import im.vector.matrix.android.api.session.call.CallState +import im.vector.matrix.android.api.session.call.EglUtils +import im.vector.matrix.android.api.session.call.MxCall +import im.vector.riotx.core.utils.DebouncedClickListener +import im.vector.riotx.features.call.WebRtcPeerConnectionManager +import org.webrtc.RendererCommon +import org.webrtc.SurfaceViewRenderer + +class ActiveCallViewHolder() { + + private var activeCallPiP: SurfaceViewRenderer? = null + private var activeCallView: ActiveCallView? = null + private var pipWrapper: CardView? = null + + private var activeCallPipInitialized = false + + fun updateCall(activeCall: MxCall?, webRtcPeerConnectionManager: WebRtcPeerConnectionManager) { + val hasActiveCall = activeCall?.state is CallState.Connected + if (hasActiveCall) { + val isVideoCall = activeCall?.isVideoCall == true + if (isVideoCall) initIfNeeded() + activeCallView?.isVisible = !isVideoCall + pipWrapper?.isVisible = isVideoCall + activeCallPiP?.isVisible = isVideoCall + activeCallPiP?.let { + webRtcPeerConnectionManager.attachViewRenderers(null, it, null) + } + } else { + activeCallView?.isVisible = false + activeCallPiP?.isVisible = false + pipWrapper?.isVisible = false + activeCallPiP?.let { + webRtcPeerConnectionManager.detachRenderers(listOf(it)) + } + } + } + + private fun initIfNeeded() { + if (!activeCallPipInitialized && activeCallPiP != null) { + activeCallPiP?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL) + EglUtils.rootEglBase?.let { eglBase -> + activeCallPiP?.init(eglBase.eglBaseContext, null) + activeCallPiP?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_BALANCED) + activeCallPiP?.setEnableHardwareScaler(true /* enabled */) + activeCallPiP?.setZOrderMediaOverlay(true) + activeCallPipInitialized + } + } + } + + fun bind(activeCallPiP: SurfaceViewRenderer, activeCallView: ActiveCallView, pipWrapper: CardView, interactionListener: ActiveCallView.Callback) { + this.activeCallPiP = activeCallPiP + this.activeCallView = activeCallView + this.pipWrapper = pipWrapper + + this.activeCallView?.callback = interactionListener + pipWrapper.setOnClickListener( + DebouncedClickListener(View.OnClickListener { _ -> + interactionListener.onTapToReturnToCall() + }) + ) + } + + fun unBind(webRtcPeerConnectionManager: WebRtcPeerConnectionManager) { + activeCallPiP?.let { + webRtcPeerConnectionManager.detachRenderers(listOf(it)) + } + if (activeCallPipInitialized) { + activeCallPiP?.release() + } + this.activeCallView?.callback = null + pipWrapper?.setOnClickListener(null) + activeCallPiP = null + activeCallView = null + pipWrapper = null + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index a1247c4390..7feafdd6c3 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -210,7 +210,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } override fun onDestroy() { - peerConnectionManager.detachRenderers() + peerConnectionManager.detachRenderers(listOf(pipRenderer, fullscreenRenderer)) if (surfaceRenderersAreInitialized) { pipRenderer.release() fullscreenRenderer.release() diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 9f60fcb568..455cb4611e 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -175,8 +175,28 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - var localSurfaceRenderer: WeakReference? = null - var remoteSurfaceRenderer: WeakReference? = null + var localSurfaceRenderer: MutableList> = ArrayList() + var remoteSurfaceRenderer: MutableList> = ArrayList() + + fun addIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList>) { + if (renderer == null) return + val exists = list.firstOrNull() { + it.get() == renderer + } != null + if (!exists) { + list.add(WeakReference(renderer)) + } + } + + fun removeIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList>) { + if (renderer == null) return + val exists = list.indexOfFirst { + it.get() == renderer + } + if (exists != -1) { + list.add(WeakReference(renderer)) + } + } var currentCall: CallContext? = null set(value) { @@ -279,10 +299,12 @@ class WebRtcPeerConnectionManager @Inject constructor( }) } - fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { + fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer") - this.localSurfaceRenderer = WeakReference(localViewRenderer) - this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) +// this.localSurfaceRenderer = WeakReference(localViewRenderer) +// this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + addIfNeeded(localViewRenderer, this.localSurfaceRenderer) + addIfNeeded(remoteViewRenderer, this.remoteSurfaceRenderer) // The call is going to resume from background, we can reduce notif currentCall?.mxCall @@ -482,15 +504,21 @@ class WebRtcPeerConnectionManager @Inject constructor( private fun attachViewRenderersInternal() { // render local video in pip view - localSurfaceRenderer?.get()?.let { pipSurface -> - pipSurface.setMirror(true) - currentCall?.localVideoTrack?.addSink(pipSurface) + localSurfaceRenderer.forEach { + it.get()?.let { pipSurface -> + pipSurface.setMirror(true) + // no need to check if already added, addSink is checking that + currentCall?.localVideoTrack?.addSink(pipSurface) + } } // If remote track exists, then sink it to surface - remoteSurfaceRenderer?.get()?.let { participantSurface -> - currentCall?.remoteVideoTrack?.let { - it.addSink(participantSurface) + remoteSurfaceRenderer.forEach { + it.get()?.let { participantSurface -> + currentCall?.remoteVideoTrack?.let { + // no need to check if already added, addSink is checking that + it.addSink(participantSurface) + } } } } @@ -505,35 +533,48 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun detachRenderers() { - // The call is going to continue in background, so ensure notification is visible - currentCall?.mxCall - ?.takeIf { it.state is CallState.Connected } - ?.let { mxCall -> - // Start background service with notification - - val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() - ?: mxCall.otherUserId - CallService.onOnGoingCallBackground( - context = context, - isVideo = mxCall.isVideoCall, - roomName = name, - roomId = mxCall.roomId, - matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", - callId = mxCall.callId - ) - } - + fun detachRenderers(renderes: List?) { Timber.v("## VOIP detachRenderers") // currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } - localSurfaceRenderer?.get()?.let { - currentCall?.localVideoTrack?.removeSink(it) + if (renderes.isNullOrEmpty()) { + // remove all sinks + localSurfaceRenderer.forEach { + if (it.get() != null) currentCall?.localVideoTrack?.removeSink(it.get()) + } + remoteSurfaceRenderer.forEach { + if (it.get() != null) currentCall?.remoteVideoTrack?.removeSink(it.get()) + } + localSurfaceRenderer.clear() + remoteSurfaceRenderer.clear() + } else { + renderes.forEach { + removeIfNeeded(it, localSurfaceRenderer) + removeIfNeeded(it, remoteSurfaceRenderer) + // no need to check if it's in the track, removeSink is doing it + currentCall?.localVideoTrack?.removeSink(it) + currentCall?.remoteVideoTrack?.removeSink(it) + } } - remoteSurfaceRenderer?.get()?.let { - currentCall?.remoteVideoTrack?.removeSink(it) + + if (remoteSurfaceRenderer.isEmpty()) { + // The call is going to continue in background, so ensure notification is visible + currentCall?.mxCall + ?.takeIf { it.state is CallState.Connected } + ?.let { mxCall -> + // Start background service with notification + + val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName() + ?: mxCall.otherUserId + CallService.onOnGoingCallBackground( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "", + callId = mxCall.callId + ) + } } - localSurfaceRenderer = null - remoteSurfaceRenderer = null } fun close() { @@ -946,7 +987,7 @@ class WebRtcPeerConnectionManager @Inject constructor( remoteVideoTrack.setEnabled(true) callContext.remoteVideoTrack = remoteVideoTrack // sink to renderer if attached - remoteSurfaceRenderer?.get()?.let { remoteVideoTrack.addSink(it) } + remoteSurfaceRenderer.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } } } } } @@ -954,9 +995,12 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onRemoveStream(stream: MediaStream) { Timber.v("## VOIP StreamObserver onRemoveStream") executor.execute { - remoteSurfaceRenderer?.get()?.let { - callContext.remoteVideoTrack?.removeSink(it) - } + // remoteSurfaceRenderer?.get()?.let { +// callContext.remoteVideoTrack?.removeSink(it) +// } + remoteSurfaceRenderer + .mapNotNull { it.get() } + .forEach { callContext.remoteVideoTrack?.removeSink(it) } callContext.remoteVideoTrack = null } } diff --git a/vector/src/main/java/im/vector/riotx/features/home/HomeDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/HomeDetailFragment.kt index 4e5d37af6c..c92c28079f 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/HomeDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/HomeDetailFragment.kt @@ -37,7 +37,12 @@ import im.vector.riotx.core.glide.GlideApp import im.vector.riotx.core.platform.ToolbarConfigurable import im.vector.riotx.core.platform.VectorBaseActivity import im.vector.riotx.core.platform.VectorBaseFragment +import im.vector.riotx.core.ui.views.ActiveCallView +import im.vector.riotx.core.ui.views.ActiveCallViewHolder import im.vector.riotx.core.ui.views.KeysBackupBanner +import im.vector.riotx.features.call.SharedActiveCallViewModel +import im.vector.riotx.features.call.VectorCallActivity +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.home.room.list.RoomListFragment import im.vector.riotx.features.home.room.list.RoomListParams import im.vector.riotx.features.home.room.list.UnreadCounterBadgeView @@ -46,6 +51,11 @@ import im.vector.riotx.features.popup.VerificationVectorAlert import im.vector.riotx.features.settings.VectorSettingsActivity.Companion.EXTRA_DIRECT_ACCESS_SECURITY_PRIVACY_MANAGE_SESSIONS import im.vector.riotx.features.workers.signout.SignOutViewModel import kotlinx.android.synthetic.main.fragment_home_detail.* +import kotlinx.android.synthetic.main.fragment_home_detail.activeCallPiP +import kotlinx.android.synthetic.main.fragment_home_detail.activeCallPiPWrap +import kotlinx.android.synthetic.main.fragment_home_detail.activeCallView +import kotlinx.android.synthetic.main.fragment_home_detail.syncStateView +import kotlinx.android.synthetic.main.fragment_room_detail.* import timber.log.Timber import javax.inject.Inject @@ -56,8 +66,9 @@ private const val INDEX_ROOMS = 2 class HomeDetailFragment @Inject constructor( val homeDetailViewModelFactory: HomeDetailViewModel.Factory, private val avatarRenderer: AvatarRenderer, - private val alertManager: PopupAlertManager -) : VectorBaseFragment(), KeysBackupBanner.Delegate { + private val alertManager: PopupAlertManager, + private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager +) : VectorBaseFragment(), KeysBackupBanner.Delegate, ActiveCallView.Callback { private val unreadCounterBadgeViews = arrayListOf() @@ -65,16 +76,21 @@ class HomeDetailFragment @Inject constructor( private val unknownDeviceDetectorSharedViewModel: UnknownDeviceDetectorSharedViewModel by activityViewModel() private lateinit var sharedActionViewModel: HomeSharedActionViewModel + private lateinit var sharedCallActionViewModel: SharedActiveCallViewModel override fun getLayoutResId() = R.layout.fragment_home_detail + private val activeCallViewHolder = ActiveCallViewHolder() + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) sharedActionViewModel = activityViewModelProvider.get(HomeSharedActionViewModel::class.java) + sharedCallActionViewModel = activityViewModelProvider.get(SharedActiveCallViewModel::class.java) setupBottomNavigationView() setupToolbar() setupKeysBackupBanner() + setupActiveCallView() withState(viewModel) { // Update the navigation view if needed (for when we restore the tabs) @@ -105,6 +121,13 @@ class HomeDetailFragment @Inject constructor( } } } + + sharedCallActionViewModel + .activeCall + .observe(viewLifecycleOwner, Observer { + activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager) + invalidateOptionsMenu() + }) } private fun promptForNewUnknownDevices(uid: String, state: UnknownDevicesState, newest: DeviceInfo) { @@ -203,6 +226,15 @@ class HomeDetailFragment @Inject constructor( homeKeysBackupBanner.delegate = this } + private fun setupActiveCallView() { + activeCallViewHolder.bind( + activeCallPiP, + activeCallView, + activeCallPiPWrap, + this + ) + } + private fun setupToolbar() { val parentActivity = vectorBaseActivity if (parentActivity is ToolbarConfigurable) { @@ -283,4 +315,20 @@ class HomeDetailFragment @Inject constructor( RoomListDisplayMode.ROOMS -> R.id.bottom_action_rooms else -> R.id.bottom_action_home } + + override fun onTapToReturnToCall() { + sharedCallActionViewModel.activeCall.value?.let { call -> + VectorCallActivity.newIntent( + context = requireContext(), + callId = call.callId, + roomId = call.roomId, + otherUserId = call.otherUserId, + isIncomingCall = !call.isOutgoing, + isVideoCall = call.isVideoCall, + mode = null + ).let { + startActivity(it) + } + } + } } diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index d46227ef7c..9f2fe328d0 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -63,7 +63,6 @@ import com.jakewharton.rxbinding3.widget.textChanges import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.permalinks.PermalinkFactory import im.vector.matrix.android.api.session.Session -import im.vector.matrix.android.api.session.call.CallState import im.vector.matrix.android.api.session.content.ContentAttachmentData import im.vector.matrix.android.api.session.events.model.Event import im.vector.matrix.android.api.session.events.model.toModel @@ -103,6 +102,7 @@ import im.vector.riotx.core.intent.getMimeTypeFromUri import im.vector.riotx.core.platform.VectorBaseFragment import im.vector.riotx.core.resources.ColorProvider import im.vector.riotx.core.ui.views.ActiveCallView +import im.vector.riotx.core.ui.views.ActiveCallViewHolder import im.vector.riotx.core.ui.views.JumpToReadMarkerView import im.vector.riotx.core.ui.views.NotificationAreaView import im.vector.riotx.core.utils.Debouncer @@ -136,6 +136,7 @@ import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs import im.vector.riotx.features.attachments.toGroupedContentAttachmentData import im.vector.riotx.features.call.SharedActiveCallViewModel import im.vector.riotx.features.call.VectorCallActivity +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.command.Command import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity import im.vector.riotx.features.crypto.util.toImageRes @@ -205,7 +206,8 @@ class RoomDetailFragment @Inject constructor( val roomDetailViewModelFactory: RoomDetailViewModel.Factory, private val eventHtmlRenderer: EventHtmlRenderer, private val vectorPreferences: VectorPreferences, - private val colorProvider: ColorProvider) : + private val colorProvider: ColorProvider, + private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager) : VectorBaseFragment(), TimelineEventController.Callback, VectorInviteView.Callback, @@ -270,6 +272,7 @@ class RoomDetailFragment @Inject constructor( private lateinit var attachmentTypeSelector: AttachmentTypeSelectorView private var lockSendButton = false + private val activeCallViewHolder = ActiveCallViewHolder() override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) @@ -301,8 +304,7 @@ class RoomDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - val hasActiveCall = it?.state is CallState.Connected - activeCallView.isVisible = hasActiveCall + activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager) invalidateOptionsMenu() }) @@ -407,6 +409,7 @@ class RoomDetailFragment @Inject constructor( } override fun onDestroy() { + activeCallViewHolder.unBind(webRtcPeerConnectionManager) roomDetailViewModel.handle(RoomDetailAction.ExitTrackingUnreadMessagesState) super.onDestroy() } @@ -437,7 +440,12 @@ class RoomDetailFragment @Inject constructor( } private fun setupActiveCallView() { - activeCallView.callback = this + activeCallViewHolder.bind( + activeCallPiP, + activeCallView, + activeCallPiPWrap, + this + ) } private fun navigateToEvent(action: RoomDetailViewEvents.NavigateToEvent) { diff --git a/vector/src/main/res/layout/fragment_home_detail.xml b/vector/src/main/res/layout/fragment_home_detail.xml index ae2e251d97..f8ddd40ad2 100644 --- a/vector/src/main/res/layout/fragment_home_detail.xml +++ b/vector/src/main/res/layout/fragment_home_detail.xml @@ -63,13 +63,43 @@ app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toBottomOf="@id/syncStateView" /> + + + + app:layout_constraintTop_toBottomOf="@+id/activeCallView" /> + + + + + + + + + + + + Date: Fri, 19 Jun 2020 18:57:50 +0200 Subject: [PATCH 72/83] Update change log --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 5a5b919087..ebcd244d64 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ Changes in RiotX 0.23.0 (2020-XX-XX) =================================================== Features ✨: - - + - Call with WebRTC support (##611) Improvements 🙌: - "Add Matrix app" menu is now always visible (#1495) From 9d401512d330fca00a92610a05e76ea0a0b63ec1 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 19 Jun 2020 19:06:25 +0200 Subject: [PATCH 73/83] dead code --- .../call/service/CallHeadsUpServiceArgs.kt | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt diff --git a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt b/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt deleted file mode 100644 index e0b774b3da..0000000000 --- a/vector/src/main/java/im/vector/riotx/features/call/service/CallHeadsUpServiceArgs.kt +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2020 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package im.vector.riotx.features.call.service - -import android.os.Parcelable -import kotlinx.android.parcel.Parcelize - -@Parcelize -data class CallHeadsUpServiceArgs( - val callId: String, - val roomId: String, - val otherUserId: String, - val isIncomingCall: Boolean, - val isVideoCall: Boolean -) : Parcelable From 76bcf9dcf71299d9f47b682117deb69798faf4dc Mon Sep 17 00:00:00 2001 From: Valere Date: Sat, 20 Jun 2020 09:24:05 +0200 Subject: [PATCH 74/83] Fix / activeCallPipInitialized not correctly initialized --- .../java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt index 70a66e7556..fdf230a55e 100644 --- a/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt +++ b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt @@ -64,7 +64,7 @@ class ActiveCallViewHolder() { activeCallPiP?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_BALANCED) activeCallPiP?.setEnableHardwareScaler(true /* enabled */) activeCallPiP?.setZOrderMediaOverlay(true) - activeCallPipInitialized + activeCallPipInitialized = true } } } From 64a67b57b849ef999c81f73fc178e80ca542283f Mon Sep 17 00:00:00 2001 From: Valere Date: Sat, 20 Jun 2020 09:41:09 +0200 Subject: [PATCH 75/83] Fix / android 7 unlock screen on incoming call --- .../riotx/features/call/VectorCallActivity.kt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 7feafdd6c3..bbfd8b20fc 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -215,8 +215,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis pipRenderer.release() fullscreenRenderer.release() } - super.onDestroy() turnScreenOffAndKeyguardOn() + super.onDestroy() } private fun renderState(state: VectorCallViewState) { @@ -450,9 +450,11 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis setShowWhenLocked(true) setTurnScreenOn(true) } else { + @Suppress("DEPRECATION") window.addFlags( - WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON - or WindowManager.LayoutParams.FLAG_ALLOW_LOCK_WHILE_SCREEN_ON + WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON + or WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED + or WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD ) } @@ -464,13 +466,15 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } private fun turnScreenOffAndKeyguardOn() { + Timber.v("## VOIP turnScreenOnAndKeyguardOn") if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { setShowWhenLocked(false) setTurnScreenOn(false) } else { + @Suppress("DEPRECATION") window.clearFlags( - WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON - or WindowManager.LayoutParams.FLAG_ALLOW_LOCK_WHILE_SCREEN_ON + WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED + or WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD ) } } From 6b806922ee7c2bb0ee1afc968d11804f333433b4 Mon Sep 17 00:00:00 2001 From: Valere Date: Sat, 20 Jun 2020 09:50:32 +0200 Subject: [PATCH 76/83] Fix / prevent camera switch if no second camera --- .../riotx/features/call/WebRtcPeerConnectionManager.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 455cb4611e..05f14ae4f2 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -179,7 +179,7 @@ class WebRtcPeerConnectionManager @Inject constructor( var remoteSurfaceRenderer: MutableList> = ArrayList() fun addIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList>) { - if (renderer == null) return + if (renderer == null) return val exists = list.firstOrNull() { it.get() == renderer } != null @@ -189,7 +189,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun removeIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList>) { - if (renderer == null) return + if (renderer == null) return val exists = list.indexOfFirst { it.get() == renderer } @@ -736,6 +736,7 @@ class WebRtcPeerConnectionManager @Inject constructor( fun switchCamera() { Timber.v("## VOIP switchCamera") + if (!canSwitchCamera()) return if (currentCall != null && currentCall?.mxCall?.state is CallState.Connected && currentCall?.mxCall?.isVideoCall == true) { videoCapturer?.switchCamera(object : CameraVideoCapturer.CameraSwitchHandler { // Invoked on success. |isFrontCamera| is true if the new camera is front facing. From 04a7c57d64b3dc6244bdeba6767c614a4f1c0dad Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 22 Jun 2020 09:47:34 +0200 Subject: [PATCH 77/83] Fix / false detection of bt headset + restore state after call --- .../riotx/features/call/CallAudioManager.kt | 50 ++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt index 9bc680ae96..add0bf0c8f 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -16,6 +16,7 @@ package im.vector.riotx.features.call +import android.bluetooth.BluetoothAdapter import android.bluetooth.BluetoothManager import android.bluetooth.BluetoothProfile import android.content.Context @@ -50,19 +51,30 @@ class CallAudioManager( private var connectedBlueToothHeadset: BluetoothProfile? = null private var wantsBluetoothConnection = false + private var bluetoothAdapter : BluetoothAdapter? = null init { executor.execute { audioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager } val bm = applicationContext.getSystemService(Context.BLUETOOTH_SERVICE) as? BluetoothManager - bm?.adapter?.getProfileProxy(applicationContext, object : BluetoothProfile.ServiceListener { + val adapter = bm?.adapter + Timber.d("## VOIP Bluetooth adapter $adapter") + bluetoothAdapter = adapter + adapter?.getProfileProxy(applicationContext, object : BluetoothProfile.ServiceListener { override fun onServiceDisconnected(profile: Int) { - connectedBlueToothHeadset = null + Timber.d("## VOIP onServiceDisconnected $profile") + if (profile == BluetoothProfile.HEADSET) { + connectedBlueToothHeadset = null + configChange?.invoke() + } } override fun onServiceConnected(profile: Int, proxy: BluetoothProfile?) { - connectedBlueToothHeadset = proxy - configChange?.invoke() + Timber.d("## VOIP onServiceConnected $profile , proxy:$proxy") + if (profile == BluetoothProfile.HEADSET) { + connectedBlueToothHeadset = proxy + configChange?.invoke() + } } }, BluetoothProfile.HEADSET) } @@ -140,6 +152,17 @@ class CallAudioManager( setMicrophoneMute(savedIsMicrophoneMute) audioManager?.mode = savedAudioMode + connectedBlueToothHeadset?.let { + if (audioManager != null && isBluetoothHeadsetConnected(audioManager!!)) { + audioManager?.stopBluetoothSco() + audioManager?.isBluetoothScoOn = false + audioManager?.setSpeakerphoneOn(false) + } + bluetoothAdapter?.closeProfileProxy(BluetoothProfile.HEADSET, it) + } + + audioManager?.mode = AudioManager.MODE_NORMAL + @Suppress("DEPRECATION") audioManager?.abandonAudioFocus(audioFocusChangeListener) } @@ -150,11 +173,14 @@ class CallAudioManager( if (audioManager.isSpeakerphoneOn) { return SoundDevice.SPEAKER } else { - if (isBluetoothHeadsetOn() && (wantsBluetoothConnection || audioManager.isBluetoothScoOn)) return SoundDevice.WIRELESS_HEADSET + if (isBluetoothHeadsetConnected(audioManager)) return SoundDevice.WIRELESS_HEADSET return if (isHeadsetOn()) SoundDevice.HEADSET else SoundDevice.PHONE } } + private fun isBluetoothHeadsetConnected(audioManager: AudioManager) = + isBluetoothHeadsetOn() && !connectedBlueToothHeadset?.connectedDevices.isNullOrEmpty() && (wantsBluetoothConnection || audioManager.isBluetoothScoOn) + fun setCurrentSoundDevice(device: SoundDevice) { executor.execute { Timber.v("## VOIP setCurrentSoundDevice $device") @@ -225,7 +251,19 @@ class CallAudioManager( } private fun isBluetoothHeadsetOn(): Boolean { - return connectedBlueToothHeadset != null + Timber.v("## VOIP: AudioManager isBluetoothHeadsetOn") + try { + if (connectedBlueToothHeadset == null) return false.also { + Timber.v("## VOIP: AudioManager no connected bluetooth headset") + } + if (audioManager?.isBluetoothScoAvailableOffCall == false) return false.also { + Timber.v("## VOIP: AudioManager isBluetoothScoAvailableOffCall false") + } + return true + } catch (failure: Throwable) { + Timber.e("## VOIP: AudioManager isBluetoothHeadsetOn failure ${failure.localizedMessage}") + return false + } } /** Sets the speaker phone mode. */ From 07e57b1498528cf9aefb47176995e181c23f6355 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 22 Jun 2020 09:48:01 +0200 Subject: [PATCH 78/83] clean --- .../main/java/im/vector/riotx/features/call/CallAudioManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt index add0bf0c8f..56a915d8e6 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -156,7 +156,7 @@ class CallAudioManager( if (audioManager != null && isBluetoothHeadsetConnected(audioManager!!)) { audioManager?.stopBluetoothSco() audioManager?.isBluetoothScoOn = false - audioManager?.setSpeakerphoneOn(false) + audioManager?.isSpeakerphoneOn = false } bluetoothAdapter?.closeProfileProxy(BluetoothProfile.HEADSET, it) } From ef2fcd60d712be2d2582be0e68f644c5fe52e18d Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 22 Jun 2020 09:54:45 +0200 Subject: [PATCH 79/83] code cleaning --- .../java/im/vector/riotx/features/call/CallAudioManager.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt index 56a915d8e6..b25a11b5b3 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallAudioManager.kt @@ -51,7 +51,8 @@ class CallAudioManager( private var connectedBlueToothHeadset: BluetoothProfile? = null private var wantsBluetoothConnection = false - private var bluetoothAdapter : BluetoothAdapter? = null + private var bluetoothAdapter: BluetoothAdapter? = null + init { executor.execute { audioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager @@ -179,7 +180,9 @@ class CallAudioManager( } private fun isBluetoothHeadsetConnected(audioManager: AudioManager) = - isBluetoothHeadsetOn() && !connectedBlueToothHeadset?.connectedDevices.isNullOrEmpty() && (wantsBluetoothConnection || audioManager.isBluetoothScoOn) + isBluetoothHeadsetOn() + && !connectedBlueToothHeadset?.connectedDevices.isNullOrEmpty() + && (wantsBluetoothConnection || audioManager.isBluetoothScoOn) fun setCurrentSoundDevice(device: SoundDevice) { executor.execute { From 4c34d735015958252f68c09279f54e37086e7bbe Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 22 Jun 2020 15:10:50 +0200 Subject: [PATCH 80/83] Fix / connection lost timer launched abusively --- .../features/call/VectorCallViewModel.kt | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 5c832f8f40..595aa41292 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -93,30 +93,34 @@ class VectorCallViewModel @AssistedInject constructor( var call: MxCall? = null var connectionTimoutTimer: Timer? = null + var hasBeenConnectedOnce = false private val callStateListener = object : MxCall.StateListener { override fun onStateUpdate(call: MxCall) { val callState = call.state if (callState is CallState.Connected && callState.iceConnectionState == PeerConnection.PeerConnectionState.CONNECTED) { + hasBeenConnectedOnce = true connectionTimoutTimer?.cancel() connectionTimoutTimer = null } else { // do we reset as long as it's moving? connectionTimoutTimer?.cancel() - connectionTimoutTimer = Timer().apply { - schedule(object : TimerTask() { - override fun run() { - session.callSignalingService().getTurnServer(object : MatrixCallback { - override fun onFailure(failure: Throwable) { - _viewEvents.post(VectorCallViewEvents.ConnectionTimeout(null)) - } + if (hasBeenConnectedOnce) { + connectionTimoutTimer = Timer().apply { + schedule(object : TimerTask() { + override fun run() { + session.callSignalingService().getTurnServer(object : MatrixCallback { + override fun onFailure(failure: Throwable) { + _viewEvents.post(VectorCallViewEvents.ConnectionTimeout(null)) + } - override fun onSuccess(data: TurnServerResponse) { - _viewEvents.post(VectorCallViewEvents.ConnectionTimeout(data)) - } - }) - } - }, 30_000) + override fun onSuccess(data: TurnServerResponse) { + _viewEvents.post(VectorCallViewEvents.ConnectionTimeout(data)) + } + }) + } + }, 30_000) + } } } setState { From 16f32da6478adcb24c941d84dd486b836da8605b Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 22 Jun 2020 15:36:30 +0200 Subject: [PATCH 81/83] Cleanup during review. --- matrix-sdk-android/build.gradle | 2 +- .../android/api/session/call/CallsListener.kt | 29 +++----- .../session/call/RoomConnectionParameter.kt | 40 ----------- .../api/session/call/TurnServerResponse.kt | 19 ++++++ .../android/internal/session/SessionModule.kt | 4 +- .../session/call/CallEventObserver.kt | 4 +- .../internal/session/call/CallModule.kt | 8 ++- .../session/call/GetTurnServerTask.kt | 3 +- .../{api => internal}/session/call/VoipApi.kt | 4 +- .../internal/session/room/RoomModule.kt | 6 -- .../src/main/res/values/strings.xml | 3 +- .../vector/riotx/core/di/ScreenComponent.kt | 2 +- .../vector/riotx/core/extensions/Session.kt | 4 -- .../core/ui/views/ActiveCallViewHolder.kt | 2 +- .../features/call/CallControlsBottomSheet.kt | 7 +- .../riotx/features/call/CallControlsView.kt | 4 +- .../call/SharedActiveCallViewModel.kt | 6 -- .../home/room/detail/RoomDetailFragment.kt | 1 - .../home/room/detail/RoomDetailViewModel.kt | 15 +++-- .../timeline/format/NoticeEventFormatter.kt | 12 +++- .../notifications/NotificationUtils.kt | 3 +- .../src/main/res/drawable/bg_call_actions.xml | 12 ---- vector/src/main/res/layout/activity_call.xml | 67 +------------------ .../res/layout/bottom_sheet_call_controls.xml | 12 ++-- vector/src/main/res/layout/fragment_call.xml | 63 ----------------- .../main/res/layout/fragment_home_detail.xml | 3 +- .../main/res/layout/fragment_room_detail.xml | 4 -- .../main/res/layout/view_active_call_view.xml | 16 ++--- ...ll_controls.xml => view_call_controls.xml} | 32 +++------ vector/src/main/res/menu/menu_timeline.xml | 1 + vector/src/main/res/values/colors_riotx.xml | 10 --- vector/src/main/res/values/strings.xml | 2 - 32 files changed, 97 insertions(+), 303 deletions(-) delete mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt rename matrix-sdk-android/src/main/java/im/vector/matrix/android/{api => internal}/session/call/VoipApi.kt (87%) delete mode 100644 vector/src/main/res/drawable/bg_call_actions.xml delete mode 100644 vector/src/main/res/layout/fragment_call.xml rename vector/src/main/res/layout/{fragment_call_controls.xml => view_call_controls.xml} (89%) diff --git a/matrix-sdk-android/build.gradle b/matrix-sdk-android/build.gradle index b44cb595d0..52c7485e49 100644 --- a/matrix-sdk-android/build.gradle +++ b/matrix-sdk-android/build.gradle @@ -163,7 +163,7 @@ dependencies { implementation 'com.googlecode.libphonenumber:libphonenumber:8.10.23' // Web RTC - // TODO meant for development purposes only + // TODO meant for development purposes only. See http://webrtc.github.io/webrtc-org/native-code/android/ implementation 'org.webrtc:google-webrtc:1.0.+' debugImplementation 'com.airbnb.okreplay:okreplay:1.5.0' diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt index 7ec46e7c10..1c51c10c0b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -22,32 +22,21 @@ import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent interface CallsListener { -// /** -// * Called when there is an incoming call within the room. -// * @param peerSignalingClient the incoming call -// */ -// fun onIncomingCall(peerSignalingClient: PeerSignalingClient) -// -// /** -// * An outgoing call is started. -// * -// * @param peerSignalingClient the outgoing call -// */ -// fun onOutgoingCall(peerSignalingClient: PeerSignalingClient) -// -// /** -// * Called when a called has been hung up -// * -// * @param peerSignalingClient the incoming call -// */ -// fun onCallHangUp(peerSignalingClient: PeerSignalingClient) - + /** + * Called when there is an incoming call within the room. + */ fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) + /** + * An outgoing call is started. + */ fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) + /** + * Called when a called has been hung up + */ fun onCallHangupReceived(callHangupContent: CallHangupContent) fun onCallManagedByOtherSession(callId: String) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt deleted file mode 100644 index 09769efc55..0000000000 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2020 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package im.vector.matrix.android.api.session.call - -import org.webrtc.IceCandidate -import org.webrtc.PeerConnection.IceServer -import org.webrtc.SessionDescription - -/** - * Struct holding the connection parameters of an AppRTC room. - */ -data class RoomConnectionParameters( - val callId: String, - val matrixRoomId: String -) - -/** - * Struct holding the signaling parameters of an AppRTC room. - */ -data class SignalingParameters( - val iceServers: List, - val initiator: Boolean, - val clientId: String, - val offerSdp: SessionDescription, - val iceCandidates: List -) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServerResponse.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServerResponse.kt index bee854de33..78acff5290 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServerResponse.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServerResponse.kt @@ -19,10 +19,29 @@ package im.vector.matrix.android.api.session.call import com.squareup.moshi.Json import com.squareup.moshi.JsonClass +// TODO Should not be exposed +/** + * Ref: https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-voip-turnserver + */ @JsonClass(generateAdapter = true) data class TurnServerResponse( + /** + * Required. The username to use. + */ @Json(name = "username") val username: String?, + + /** + * Required. The password to use. + */ @Json(name = "password") val password: String?, + + /** + * Required. A list of TURN URIs + */ @Json(name = "uris") val uris: List?, + + /** + * Required. The time-to-live in seconds + */ @Json(name = "ttl") val ttl: Int? ) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt index 65b79e7db7..338b93bc32 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt @@ -246,11 +246,11 @@ internal abstract class SessionModule { @Binds @IntoSet - abstract fun bindCallEventObserver(callEventObserver: CallEventObserver): LiveEntityObserver + abstract fun bindCallEventObserver(observer: CallEventObserver): LiveEntityObserver @Binds @IntoSet - abstract fun bindRoomTombstoneEventLiveObserver(roomTombstoneEventLiveObserver: RoomTombstoneEventLiveObserver): LiveEntityObserver + abstract fun bindRoomTombstoneEventLiveObserver(observer: RoomTombstoneEventLiveObserver): LiveEntityObserver @Binds @IntoSet diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt index ad70cb245d..585ecb61ca 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt @@ -34,8 +34,8 @@ import javax.inject.Inject internal class CallEventObserver @Inject constructor( @SessionDatabase realmConfiguration: RealmConfiguration, @UserId private val userId: String, - private val task: CallEventsObserverTask) : - RealmLiveEntityObserver(realmConfiguration) { + private val task: CallEventsObserverTask +) : RealmLiveEntityObserver(realmConfiguration) { override val query = Monarchy.Query { EventEntity.whereTypes(it, listOf( diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt index 6f0add43df..a25d198e83 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallModule.kt @@ -20,7 +20,6 @@ import dagger.Binds import dagger.Module import dagger.Provides import im.vector.matrix.android.api.session.call.CallSignalingService -import im.vector.matrix.android.api.session.call.VoipApi import im.vector.matrix.android.internal.session.SessionScope import retrofit2.Retrofit @@ -38,8 +37,11 @@ internal abstract class CallModule { } @Binds - abstract fun bindCallService(service:DefaultCallSignalingService): CallSignalingService + abstract fun bindCallSignalingService(service: DefaultCallSignalingService): CallSignalingService @Binds - abstract fun bindTurnServerTask(task: DefaultGetTurnServerTask): GetTurnServerTask + abstract fun bindGetTurnServerTask(task: DefaultGetTurnServerTask): GetTurnServerTask + + @Binds + abstract fun bindCallEventsObserverTask(task: DefaultCallEventsObserverTask): CallEventsObserverTask } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt index bac7a36c05..f644bb22e9 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/GetTurnServerTask.kt @@ -17,7 +17,6 @@ package im.vector.matrix.android.internal.session.call import im.vector.matrix.android.api.session.call.TurnServerResponse -import im.vector.matrix.android.api.session.call.VoipApi import im.vector.matrix.android.internal.network.executeRequest import im.vector.matrix.android.internal.task.Task import org.greenrobot.eventbus.EventBus @@ -28,7 +27,7 @@ internal abstract class GetTurnServerTask : TaskYou changed the room name to: %1$s %s placed a video call. You placed a video call. - You placed a voice call. %s placed a voice call. + You placed a voice call. %s sent data to setup the call. + You sent data to setup the call. %s answered the call. You answered the call. %s ended the call. diff --git a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt index c5fdb89ea4..7bba3cb5d4 100644 --- a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt +++ b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt @@ -149,7 +149,7 @@ interface ScreenComponent { fun inject(bottomSheet: BootstrapBottomSheet) fun inject(bottomSheet: RoomWidgetPermissionBottomSheet) fun inject(bottomSheet: RoomWidgetsBottomSheet) - fun inject(callControlsBottomSheet: CallControlsBottomSheet) + fun inject(bottomSheet: CallControlsBottomSheet) /* ========================================================================================== * Others diff --git a/vector/src/main/java/im/vector/riotx/core/extensions/Session.kt b/vector/src/main/java/im/vector/riotx/core/extensions/Session.kt index eb3fca66c6..29b169ffd4 100644 --- a/vector/src/main/java/im/vector/riotx/core/extensions/Session.kt +++ b/vector/src/main/java/im/vector/riotx/core/extensions/Session.kt @@ -38,10 +38,6 @@ fun Session.configureAndStart(context: Context, startSyncing(context) refreshPushers() pushRuleTriggerListener.startWithSession(this) - - // TODO P1 From HomeActivity - // @Inject lateinit var incomingVerificationRequestHandler: IncomingVerificationRequestHandler - // @Inject lateinit var keyRequestHandler: KeyRequestHandler } fun Session.startSyncing(context: Context) { diff --git a/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt index fdf230a55e..6a5519adbc 100644 --- a/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt +++ b/vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt @@ -27,7 +27,7 @@ import im.vector.riotx.features.call.WebRtcPeerConnectionManager import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer -class ActiveCallViewHolder() { +class ActiveCallViewHolder { private var activeCallPiP: SurfaceViewRenderer? = null private var activeCallView: ActiveCallView? = null diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt index 3d86488720..cf506a031f 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsBottomSheet.kt @@ -24,9 +24,7 @@ import androidx.core.view.isVisible import com.airbnb.mvrx.activityViewModel import im.vector.riotx.R import im.vector.riotx.core.platform.VectorBaseBottomSheetDialogFragment -import kotlinx.android.synthetic.main.activity_call.* import kotlinx.android.synthetic.main.bottom_sheet_call_controls.* -import kotlinx.android.synthetic.main.vector_preference_push_rule.view.* import me.gujun.android.span.span class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { @@ -91,6 +89,7 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { .setItems(soundDevices.toTypedArray()) { d, n -> d.cancel() when (soundDevices[n].toString()) { + // TODO Make an adapter and handle multiple Bluetooth headsets. Also do not use translations. getString(R.string.sound_device_phone) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.PHONE)) } @@ -125,11 +124,11 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { callControlsToggleSDHD.isVisible = true if (state.isHD) { callControlsToggleSDHD.title = getString(R.string.call_format_turn_hd_off) - callControlsToggleSDHD.subTitle = null + callControlsToggleSDHD.subTitle = null callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd_disabled) } else { callControlsToggleSDHD.title = getString(R.string.call_format_turn_hd_on) - callControlsToggleSDHD.subTitle = null + callControlsToggleSDHD.subTitle = null callControlsToggleSDHD.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_hd) } } else { diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index adf05184b2..6bd28ae971 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -28,7 +28,7 @@ import butterknife.ButterKnife import butterknife.OnClick import im.vector.matrix.android.api.session.call.CallState import im.vector.riotx.R -import kotlinx.android.synthetic.main.fragment_call_controls.view.* +import kotlinx.android.synthetic.main.view_call_controls.view.* import org.webrtc.PeerConnection class CallControlsView @JvmOverloads constructor( @@ -54,7 +54,7 @@ class CallControlsView @JvmOverloads constructor( lateinit var videoToggleIcon: ImageView init { - ConstraintLayout.inflate(context, R.layout.fragment_call_controls, this) + ConstraintLayout.inflate(context, R.layout.view_call_controls, this) // layoutParams = FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT) ButterKnife.bind(this) } diff --git a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt index 4900d39a32..71f5ad3877 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt @@ -19,14 +19,8 @@ package im.vector.riotx.features.call import androidx.lifecycle.MutableLiveData import androidx.lifecycle.ViewModel import im.vector.matrix.android.api.session.call.MxCall -import im.vector.riotx.core.platform.VectorSharedAction import javax.inject.Inject -sealed class CallActions : VectorSharedAction { - data class GoToCallActivity(val mxCall: MxCall) : CallActions() - data class ToggleVisibility(val visible: Boolean) : CallActions() -} - class SharedActiveCallViewModel @Inject constructor( private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager ) : ViewModel() { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index 9f2fe328d0..e697e41e0e 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -517,7 +517,6 @@ class RoomDetailFragment @Inject constructor( } R.id.voice_call, R.id.video_call -> { - // TODO CALL We should check/ask for permission here first val activeCall = sharedCallActionViewModel.activeCall.value val isVideoCall = item.itemId == R.id.video_call if (activeCall != null) { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index 42b7d67f2a..4cbdd0b8d2 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -383,15 +383,16 @@ class RoomDetailViewModel @AssistedInject constructor( } fun isMenuItemVisible(@IdRes itemId: Int) = when (itemId) { - R.id.clear_message_queue -> + R.id.clear_message_queue -> // For now always disable when not in developer mode, worker cancellation is not working properly timeline.pendingEventCount() > 0 && vectorPreferences.developerMode() - R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 - R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 - R.id.open_matrix_apps -> true - R.id.voice_call, R.id.video_call -> room.canStartCall() && webRtcPeerConnectionManager.currentCall == null - R.id.hangup_call -> webRtcPeerConnectionManager.currentCall != null - else -> false + R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 + R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 + R.id.open_matrix_apps -> true + R.id.voice_call, + R.id.video_call -> room.canStartCall() && webRtcPeerConnectionManager.currentCall == null + R.id.hangup_call -> webRtcPeerConnectionManager.currentCall != null + else -> false } // PRIVATE METHODS ***************************************************************************** diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt index 3d3581e03b..37debace89 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt @@ -255,19 +255,25 @@ class NoticeEventFormatter @Inject constructor(private val sessionHolder: Active } } } - EventType.CALL_ANSWER -> + EventType.CALL_ANSWER -> if (event.isSentByCurrentUser()) { sp.getString(R.string.notice_answered_call_by_you) } else { sp.getString(R.string.notice_answered_call, senderName) } - EventType.CALL_HANGUP -> + EventType.CALL_HANGUP -> if (event.isSentByCurrentUser()) { sp.getString(R.string.notice_ended_call_by_you) } else { sp.getString(R.string.notice_ended_call, senderName) } - else -> null + EventType.CALL_CANDIDATES -> + if (event.isSentByCurrentUser()) { + sp.getString(R.string.notice_call_candidates_by_you) + } else { + sp.getString(R.string.notice_call_candidates, senderName) + } + else -> null } } diff --git a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt index d936935330..9dc518bbc9 100755 --- a/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt +++ b/vector/src/main/java/im/vector/riotx/features/notifications/NotificationUtils.kt @@ -420,7 +420,8 @@ class NotificationUtils @Inject constructor(private val context: Context, roomName: String, roomId: String, matrixId: String, - callId: String, fromBg: Boolean = false): Notification { + callId: String, + fromBg: Boolean = false): Notification { val builder = NotificationCompat.Builder(context, if (fromBg) CALL_NOTIFICATION_CHANNEL_ID else SILENT_NOTIFICATION_CHANNEL_ID) .setContentTitle(ensureTitleNotEmpty(roomName)) .apply { diff --git a/vector/src/main/res/drawable/bg_call_actions.xml b/vector/src/main/res/drawable/bg_call_actions.xml deleted file mode 100644 index f074beb8f9..0000000000 --- a/vector/src/main/res/drawable/bg_call_actions.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index d05ac9d0d2..ee1bafe0cd 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -106,71 +106,6 @@ android:layout_width="match_parent" android:layout_height="wrap_content" app:layout_constraintBottom_toBottomOf="parent" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + \ No newline at end of file diff --git a/vector/src/main/res/layout/bottom_sheet_call_controls.xml b/vector/src/main/res/layout/bottom_sheet_call_controls.xml index 275d50f457..04cb2af20d 100644 --- a/vector/src/main/res/layout/bottom_sheet_call_controls.xml +++ b/vector/src/main/res/layout/bottom_sheet_call_controls.xml @@ -12,26 +12,26 @@ android:layout_width="match_parent" android:layout_height="wrap_content" app:actionTitle="@string/call_select_sound_device" - tools:actionDescription="Speaker" app:leftIcon="@drawable/ic_call_speaker_default" - app:tint="?attr/riotx_text_primary" /> + app:tint="?attr/riotx_text_primary" + tools:actionDescription="Speaker" /> + app:tint="?attr/riotx_text_primary" + tools:actionDescription="Front" /> + app:tint="?attr/riotx_text_primary" + tools:actionDescription="Front" /> diff --git a/vector/src/main/res/layout/fragment_call.xml b/vector/src/main/res/layout/fragment_call.xml deleted file mode 100644 index 2ab342d68a..0000000000 --- a/vector/src/main/res/layout/fragment_call.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/vector/src/main/res/layout/fragment_home_detail.xml b/vector/src/main/res/layout/fragment_home_detail.xml index f8ddd40ad2..f90422dff9 100644 --- a/vector/src/main/res/layout/fragment_home_detail.xml +++ b/vector/src/main/res/layout/fragment_home_detail.xml @@ -63,7 +63,6 @@ app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toBottomOf="@id/syncStateView" /> - - + - @@ -116,7 +115,6 @@ app:layout_constraintTop_toBottomOf="@id/activeCallView" tools:listitem="@layout/item_timeline_event_base" /> - - - diff --git a/vector/src/main/res/layout/view_active_call_view.xml b/vector/src/main/res/layout/view_active_call_view.xml index 4328691ee1..62ab2ae60e 100644 --- a/vector/src/main/res/layout/view_active_call_view.xml +++ b/vector/src/main/res/layout/view_active_call_view.xml @@ -1,5 +1,6 @@ + android:textColor="@color/white" + app:drawableTint="@color/white" /> + android:textStyle="bold" /> diff --git a/vector/src/main/res/layout/fragment_call_controls.xml b/vector/src/main/res/layout/view_call_controls.xml similarity index 89% rename from vector/src/main/res/layout/fragment_call_controls.xml rename to vector/src/main/res/layout/view_call_controls.xml index 74da380f9e..11622a1397 100644 --- a/vector/src/main/res/layout/fragment_call_controls.xml +++ b/vector/src/main/res/layout/view_call_controls.xml @@ -1,26 +1,23 @@ - - + android:layout_height="wrap_content"> + - @@ -95,7 +88,6 @@ android:id="@+id/iv_end_call" android:layout_width="64dp" android:layout_height="64dp" - android:layout_marginBottom="32dp" android:background="@drawable/oval_destructive" android:clickable="true" android:focusable="true" @@ -108,37 +100,33 @@ android:id="@+id/iv_video_toggle" android:layout_width="64dp" android:layout_height="64dp" - android:layout_marginBottom="32dp" android:background="@drawable/oval_positive" - android:backgroundTint="?attr/riotx_background" android:clickable="true" android:focusable="true" android:padding="16dp" android:src="@drawable/ic_call_videocam_off_default" android:tint="?attr/riotx_text_primary" + app:backgroundTint="?attr/riotx_background" tools:ignore="MissingConstraints" /> + tools:ignore="MissingConstraints" /> @@ -208,4 +196,4 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index a6666d33ae..d4eb923d50 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -34,6 +34,7 @@ app:iconTint="@color/riotx_destructive_accent" app:showAsAction="always" tools:visible="true" /> + #FFF8E3 #22262E - - #000000 - #000000 - #000000 - - - @android:color/transparent - @android:color/transparent - @android:color/transparent - \ No newline at end of file diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index caabd55651..57d2a2b63b 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -225,8 +225,6 @@ Turn HD off Turn HD on - - Send files Send sticker Take photo or video From c15cc34bfd76284fcee040e9f797708011b99dfc Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 22 Jun 2020 16:14:48 +0200 Subject: [PATCH 82/83] Call: a11y --- .../riotx/features/call/CallControlsView.kt | 18 ++++++++++++++++-- vector/src/main/res/layout/activity_call.xml | 1 + .../src/main/res/layout/view_call_controls.xml | 8 +++++++- vector/src/main/res/values/strings.xml | 5 +++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt index 6bd28ae971..e3b9f12f67 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/CallControlsView.kt @@ -39,8 +39,10 @@ class CallControlsView @JvmOverloads constructor( @BindView(R.id.ringingControls) lateinit var ringingControls: ViewGroup + @BindView(R.id.iv_icr_accept_call) lateinit var ringingControlAccept: ImageView + @BindView(R.id.iv_icr_end_call) lateinit var ringingControlDecline: ImageView @@ -96,8 +98,20 @@ class CallControlsView @JvmOverloads constructor( fun updateForState(state: VectorCallViewState) { val callState = state.callState.invoke() - muteIcon.setImageResource(if (state.isAudioMuted) R.drawable.ic_microphone_off else R.drawable.ic_microphone_on) - videoToggleIcon.setImageResource(if (state.isVideoEnabled) R.drawable.ic_video else R.drawable.ic_video_off) + if (state.isAudioMuted) { + muteIcon.setImageResource(R.drawable.ic_microphone_off) + muteIcon.contentDescription = resources.getString(R.string.a11y_unmute_microphone) + } else { + muteIcon.setImageResource(R.drawable.ic_microphone_on) + muteIcon.contentDescription = resources.getString(R.string.a11y_mute_microphone) + } + if (state.isVideoEnabled) { + videoToggleIcon.setImageResource(R.drawable.ic_video) + videoToggleIcon.contentDescription = resources.getString(R.string.a11y_stop_camera) + } else { + videoToggleIcon.setImageResource(R.drawable.ic_video_off) + videoToggleIcon.contentDescription = resources.getString(R.string.a11y_start_camera) + } when (callState) { is CallState.Idle, diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index ee1bafe0cd..39d0bef790 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -53,6 +53,7 @@ android:layout_width="128dp" android:layout_height="128dp" android:layout_centerVertical="true" + android:contentDescription="@string/avatar" android:scaleType="centerCrop" app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintEnd_toEndOf="parent" diff --git a/vector/src/main/res/layout/view_call_controls.xml b/vector/src/main/res/layout/view_call_controls.xml index 11622a1397..94757c2c72 100644 --- a/vector/src/main/res/layout/view_call_controls.xml +++ b/vector/src/main/res/layout/view_call_controls.xml @@ -13,13 +13,13 @@ tools:background="@color/password_strength_bar_ok" tools:visibility="visible"> - @@ -90,6 +93,7 @@ android:layout_height="64dp" android:background="@drawable/oval_destructive" android:clickable="true" + android:contentDescription="@string/call_notification_hangup" android:focusable="true" android:padding="16dp" android:src="@drawable/ic_call_end" @@ -107,6 +111,7 @@ android:src="@drawable/ic_call_videocam_off_default" android:tint="?attr/riotx_text_primary" app:backgroundTint="?attr/riotx_background" + tools:contentDescription="@string/a11y_stop_camera" tools:ignore="MissingConstraints" /> Enter the URL of an identity server Submit Set role + Open chat + Mute the microphone + Unmute the microphone + Stop the camera + Start the camera From f4e7405d92db1dce821dc5467aef80175d76a884 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 22 Jun 2020 16:21:36 +0200 Subject: [PATCH 83/83] Cleanup --- vector/src/main/AndroidManifest.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 652534a7ff..6cfe02dd0f 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -217,10 +217,6 @@ - - - -