jarno
/
element-android
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move voip responsibilities from views to WebRtcPeerConnectionManager.

This commit is contained in:
onurays 2020-05-25 23:28:30 +03:00 committed by Valere
parent 5d476e7259
commit 743ace7e60
6 changed files with 139 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt
View File

@ -46,6 +46,11 @@ interface CallService {
*/
fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List<IceCandidate>)
/**
* Send a hangup event
*/
fun sendHangup(callId: String, roomId: String)
fun addCallListener(listener: CallsListener)
fun removeCallListener(listener: CallsListener)

10
matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt
View File

@ -127,6 +127,16 @@ internal class DefaultCallService @Inject constructor(
override fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List<IceCandidate>) {
}
override fun sendHangup(callId: String, roomId: String) {
val eventContent = CallHangupContent(
callId = callId,
version = 0
)
createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = eventContent.toContent()).let { event ->
roomEventSender.sendEvent(event)
}
}
override fun addCallListener(listener: CallsListener) {
if (!callListeners.contains(listener)) callListeners.add(listener)
}

44
vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt
View File

@ -38,17 +38,14 @@ import im.vector.riotx.core.utils.checkPermissions
import io.reactivex.android.schedulers.AndroidSchedulers
import io.reactivex.subjects.PublishSubject
import kotlinx.android.parcel.Parcelize
import org.webrtc.Camera1Enumerator
import org.webrtc.Camera2Enumerator
import kotlinx.android.synthetic.main.activity_call.*
import org.webrtc.EglBase
import org.webrtc.IceCandidate
import org.webrtc.MediaStream
import org.webrtc.PeerConnection
import org.webrtc.RendererCommon
import org.webrtc.SessionDescription
import org.webrtc.SurfaceViewRenderer
import org.webrtc.VideoTrack
import java.util.concurrent.TimeUnit
import javax.inject.Inject
@Parcelize
@ -69,6 +66,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis
}
private val callViewModel: VectorCallViewModel by viewModel()
private lateinit var callArgs: CallArgs
@Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager
@ -114,10 +112,19 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
if (intent.hasExtra(MvRx.KEY_ARG)) {
callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!!
} else {
finish()
}
rootEglBase = EglUtils.rootEglBase ?: return Unit.also {
finish()
}
iv_end_call.setOnClickListener { callViewModel.handle(VectorCallViewActions.EndCall) }
callViewModel.viewEvents
.observe()
.observeOn(AndroidSchedulers.mainThread())
@ -161,8 +168,9 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis
// setSwappedFeeds(true /* isSwappedFeeds */);
if (isFirstCreation()) {
peerConnectionManager.createPeerConnectionFactory()
//peerConnectionManager.createPeerConnectionFactory()
/*
val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false)
val frontCamera = cameraIterator.deviceNames
?.firstOrNull { cameraIterator.isFrontFacing(it) }
@ -170,19 +178,12 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis
?: return true
val videoCapturer = cameraIterator.createCapturer(frontCamera, null)
val iceServers = ArrayList<PeerConnection.IceServer>().apply {
listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach {
add(
PeerConnection.IceServer.builder(it)
.setUsername("xxxxx")
.setPassword("xxxxx")
.createIceServer()
)
}
}
peerConnectionManager.createPeerConnection(videoCapturer, iceServers)
peerConnectionManager.startCall()
*/
//peerConnectionManager.startCall()
}
// PeerConnectionFactory.initialize(PeerConnectionFactory
// .InitializationOptions.builder(applicationContext)
@ -322,15 +323,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis
// Timber.v("## VOIP onCreateFailure $p0")
// }
// }, constraints)
iceCandidateSource
.buffer(400, TimeUnit.MILLISECONDS)
.subscribe {
// omit empty :/
if (it.isNotEmpty()) {
callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it))
}
}
.disposeOnDestroy()
peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer)
return false
@ -433,6 +425,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis
}
override fun sendOffer(sessionDescription: SessionDescription) {
callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription))
}
}

33
vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt
View File

@ -16,7 +16,6 @@
package im.vector.riotx.features.call
import androidx.lifecycle.viewModelScope
import com.airbnb.mvrx.MvRxState
import com.airbnb.mvrx.MvRxViewModelFactory
import com.airbnb.mvrx.ViewModelContext
@ -27,16 +26,10 @@ import im.vector.matrix.android.api.session.call.CallsListener
import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent
import im.vector.matrix.android.api.session.room.model.call.CallHangupContent
import im.vector.matrix.android.api.session.room.model.call.CallInviteContent
import im.vector.matrix.android.internal.util.awaitCallback
import im.vector.riotx.core.extensions.exhaustive
import im.vector.riotx.core.platform.VectorViewEvents
import im.vector.riotx.core.platform.VectorViewModel
import im.vector.riotx.core.platform.VectorViewModelAction
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import org.webrtc.IceCandidate
import org.webrtc.SessionDescription
import java.util.UUID
data class VectorCallViewState(
val callId: String? = null,
@ -45,8 +38,7 @@ data class VectorCallViewState(
sealed class VectorCallViewActions : VectorViewModelAction {
data class SendOffer(val sdp: SessionDescription) : VectorCallViewActions()
data class AddLocalIceCandidate(val iceCandidates: List<IceCandidate>) : VectorCallViewActions()
object EndCall : VectorCallViewActions()
}
sealed class VectorCallViewEvents : VectorViewEvents {
@ -57,7 +49,8 @@ sealed class VectorCallViewEvents : VectorViewEvents {
class VectorCallViewModel @AssistedInject constructor(
@Assisted initialState: VectorCallViewState,
val session: Session
val session: Session,
val webRtcPeerConnectionManager: WebRtcPeerConnectionManager
) : VectorViewModel<VectorCallViewState, VectorCallViewActions, VectorCallViewEvents>(initialState) {
private val callServiceListener: CallsListener = object : CallsListener {
@ -90,25 +83,9 @@ class VectorCallViewModel @AssistedInject constructor(
super.onCleared()
}
override fun handle(action: VectorCallViewActions) = withState { state ->
override fun handle(action: VectorCallViewActions) = withState {
when (action) {
is VectorCallViewActions.SendOffer -> {
viewModelScope.launch(Dispatchers.IO) {
awaitCallback<String> {
val callId = state.callId ?: UUID.randomUUID().toString().also {
setState {
copy(callId = it)
}
}
session.callService().sendOfferSdp(callId, state.roomId, action.sdp, it)
}
}
}
is VectorCallViewActions.AddLocalIceCandidate -> {
viewModelScope.launch {
session.callService().sendLocalIceCandidates(state.callId ?: "", state.roomId, action.iceCandidates)
}
}
VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall()
}.exhaustive
}

150
vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt
View File

@ -22,6 +22,7 @@ import android.content.Intent
import android.content.ServiceConnection
import android.os.IBinder
import androidx.core.content.ContextCompat
import im.vector.matrix.android.api.MatrixCallback
import im.vector.matrix.android.api.session.call.CallsListener
import im.vector.matrix.android.api.session.call.EglUtils
import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent
@ -29,6 +30,8 @@ import im.vector.matrix.android.api.session.room.model.call.CallHangupContent
import im.vector.matrix.android.api.session.room.model.call.CallInviteContent
import im.vector.riotx.core.di.ActiveSessionHolder
import im.vector.riotx.features.call.service.CallHeadsUpService
import io.reactivex.disposables.Disposable
import io.reactivex.subjects.PublishSubject
import org.webrtc.AudioSource
import org.webrtc.AudioTrack
import org.webrtc.DefaultVideoDecoderFactory
@ -47,7 +50,9 @@ import org.webrtc.VideoSource
import org.webrtc.VideoTrack
import timber.log.Timber
import java.lang.ref.WeakReference
import java.util.UUID
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import javax.inject.Inject
import javax.inject.Singleton
@ -59,7 +64,7 @@ import javax.inject.Singleton
class WebRtcPeerConnectionManager @Inject constructor(
private val context: Context,
private val sessionHolder: ActiveSessionHolder
) : CallsListener {
) : CallsListener {
interface Listener {
fun addLocalIceCandidate(candidates: IceCandidate)
@ -98,8 +103,15 @@ class WebRtcPeerConnectionManager @Inject constructor(
var localSurfaceRenderer: WeakReference<SurfaceViewRenderer>? = null
var remoteSurfaceRenderer: WeakReference<SurfaceViewRenderer>? = null
private val iceCandidateSource: PublishSubject<IceCandidate> = PublishSubject.create()
private var iceCandidateDisposable: Disposable? = null
var callHeadsUpService: CallHeadsUpService? = null
private var callId: String? = null
private var signalingRoomId: String? = null
private var participantUserId: String? = null
private val serviceConnection = object : ServiceConnection {
override fun onServiceDisconnected(name: ComponentName?) {
}
@ -109,7 +121,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
fun createPeerConnectionFactory() {
private fun createPeerConnectionFactory() {
executor.execute {
if (peerConnectionFactory == null) {
Timber.v("## VOIP createPeerConnectionFactory")
@ -142,7 +154,56 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
fun createPeerConnection(videoCapturer: VideoCapturer, iceServers: List<PeerConnection.IceServer>) {
private fun createPeerConnection() {
val iceServers = ArrayList<PeerConnection.IceServer>().apply {
listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach {
add(
PeerConnection.IceServer.builder(it)
.setUsername("xxxxx")
.setPassword("xxxxx")
.createIceServer()
)
}
}
Timber.v("## VOIP creating peer connection... ")
peerConnection = peerConnectionFactory?.createPeerConnection(
iceServers,
object : PeerConnectionObserverAdapter() {
override fun onIceCandidate(p0: IceCandidate?) {
Timber.v("## VOIP onIceCandidate local $p0")
p0?.let { iceCandidateSource.onNext(it) }
}
override fun onAddStream(mediaStream: MediaStream?) {
Timber.v("## VOIP onAddStream remote $mediaStream")
mediaStream?.videoTracks?.firstOrNull()?.let {
listener?.addRemoteVideoTrack(it)
remoteVideoTrack = it
}
}
override fun onRemoveStream(mediaStream: MediaStream?) {
mediaStream?.let {
listener?.removeRemoteVideoStream(it)
}
remoteSurfaceRenderer?.get()?.let {
remoteVideoTrack?.removeSink(it)
}
remoteVideoTrack = null
}
override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {
Timber.v("## VOIP onIceConnectionChange $p0")
if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) {
listener?.onDisconnect()
}
}
}
)
}
// TODO REMOVE THIS FUNCTION
private fun createPeerConnection(videoCapturer: VideoCapturer) {
executor.execute {
Timber.v("## VOIP PeerConnectionFactory.createPeerConnection $peerConnectionFactory...")
// Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object
@ -183,55 +244,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
// }
// .disposeOnDestroy()
Timber.v("## VOIP creating peer connection... ")
peerConnection = peerConnectionFactory?.createPeerConnection(
iceServers,
object : PeerConnectionObserverAdapter() {
override fun onIceCandidate(p0: IceCandidate?) {
Timber.v("## VOIP onIceCandidate local $p0")
p0?.let {
// iceCandidateSource.onNext(it)
listener?.addLocalIceCandidate(it)
}
}
override fun onAddStream(mediaStream: MediaStream?) {
Timber.v("## VOIP onAddStream remote $mediaStream")
mediaStream?.videoTracks?.firstOrNull()?.let {
listener?.addRemoteVideoTrack(it)
remoteVideoTrack = it
// remoteSurfaceRenderer?.get()?.let { surface ->
// it.setEnabled(true)
// it.addSink(surface)
// }
}
// runOnUiThread {
// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack ->
// remoteVideoTrack = videoTrack
// remoteVideoTrack?.setEnabled(true)
// remoteVideoTrack?.addSink(fullscreenRenderer)
// }
// }
}
override fun onRemoveStream(mediaStream: MediaStream?) {
mediaStream?.let {
listener?.removeRemoteVideoStream(it)
}
remoteSurfaceRenderer?.get()?.let {
remoteVideoTrack?.removeSink(it)
}
remoteVideoTrack = null
}
override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {
Timber.v("## VOIP onIceConnectionChange $p0")
if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) {
listener?.onDisconnect()
}
}
}
)
localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value?
localMediaStream?.addTrack(localVideoTrack)
@ -253,7 +266,22 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
fun startCall() {
private fun startCall() {
createPeerConnectionFactory()
createPeerConnection()
iceCandidateDisposable = iceCandidateSource
.buffer(400, TimeUnit.MILLISECONDS)
.subscribe {
// omit empty :/
if (it.isNotEmpty()) {
sessionHolder
.getActiveSession()
.callService()
.sendLocalIceCandidates(callId ?: "", signalingRoomId ?: "", it)
}
}
executor.execute {
val constraints = MediaConstraints()
constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"))
@ -273,8 +301,8 @@ class WebRtcPeerConnectionManager @Inject constructor(
Timber.v("## VOIP onCreateSuccess $sessionDescription")
peerConnection?.setLocalDescription(object : SdpObserverAdapter() {
override fun onSetSuccess() {
listener?.sendOffer(sessionDescription)
// callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription))
callId = UUID.randomUUID().toString()
sessionHolder.getActiveSession().callService().sendOfferSdp(callId!!, signalingRoomId!!, sessionDescription, object : MatrixCallback<String> {})
}
}, sessionDescription)
}
@ -319,6 +347,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
peerConnectionFactory?.stopAecDump()
peerConnectionFactory = null
}
iceCandidateDisposable?.dispose()
context.stopService(Intent(context, CallHeadsUpService::class.java))
}
@ -346,12 +375,18 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
fun startOutgoingCall(context: Context, signalingRoomId: String, participantUserId: String, isVideoCall: Boolean) {
this.signalingRoomId = signalingRoomId
this.participantUserId = participantUserId
startHeadsUpService(signalingRoomId, sessionHolder.getActiveSession().myUserId, false, isVideoCall)
context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, isVideoCall))
startCall()
}
override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) {
startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo())
startCall()
}
private fun startHeadsUpService(roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean) {
@ -361,6 +396,13 @@ class WebRtcPeerConnectionManager @Inject constructor(
context.bindService(Intent(context, CallHeadsUpService::class.java), serviceConnection, 0)
}
fun endCall() {
if (callId != null && signalingRoomId != null) {
sessionHolder.getActiveSession().callService().sendHangup(callId!!, signalingRoomId!!)
close()
}
}
override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) {
}

12
vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt
View File

@ -21,14 +21,10 @@ import android.os.Build
import android.telecom.Connection
import android.telecom.DisconnectCause
import androidx.annotation.RequiresApi
import im.vector.riotx.features.call.VectorCallViewActions
import im.vector.riotx.features.call.VectorCallViewModel
import im.vector.riotx.features.call.WebRtcPeerConnectionManager
import org.webrtc.Camera1Enumerator
import org.webrtc.Camera2Enumerator
import org.webrtc.IceCandidate
import org.webrtc.MediaStream
import org.webrtc.PeerConnection
import org.webrtc.SessionDescription
import org.webrtc.VideoTrack
import timber.log.Timber
@ -91,7 +87,8 @@ import javax.inject.Inject
}
private fun startCall() {
peerConnectionManager.createPeerConnectionFactory()
/*
//peerConnectionManager.createPeerConnectionFactory()
peerConnectionManager.listener = this
val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false)
@ -113,7 +110,8 @@ import javax.inject.Inject
}
peerConnectionManager.createPeerConnection(videoCapturer, iceServers)
peerConnectionManager.startCall()
//peerConnectionManager.startCall()
*/
}
override fun addLocalIceCandidate(candidates: IceCandidate) {
@ -129,6 +127,6 @@ import javax.inject.Inject
}
override fun sendOffer(sessionDescription: SessionDescription) {
callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription))
}
}