# HG changeset patch # User Andreas Pehrson # Date 1513610373 -3600 # Mon Dec 18 16:19:33 2017 +0100 # Node ID abd22366d81e48c08efa3940f2b726ea125e6c10 # Parent 0c6af5adf3aae8737e78ca55aafefdd777cd6e84 Bug 1299515 - Allow MediaEngines to Start() and Stop() without affecting tracks. r=jib, r=padenot Checked in under Bug 1299516. This is the larger change for this bug. In order to turn off a device on disabling we want to Stop() it without ending the attached track. To allow this, this patch breaks out track-creation from Start() to SetTrack() and moves track-ending logic from Stop() to Deallocate(). It is a programming error to Start() or Stop() a MediaEngineSource that hasn't seen a SetTrack(). MozReview-Commit-ID: 3KzmuDjCAH0 diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -838,38 +838,42 @@ nsresult MediaDevice::Allocate(const dom return mSource->Allocate(aConstraints, aPrefs, mID, aPrincipalInfo, getter_AddRefs(mAllocationHandle), aOutBadConstraint); } -nsresult MediaDevice::Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipal) - +nsresult MediaDevice::SetTrack(const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipalHandle) { - return mSource->Start(aStream, aTrackID, aPrincipal); + return mSource->SetTrack(mAllocationHandle, aStream, aTrackID, aPrincipalHandle); +} + +nsresult MediaDevice::Start() +{ + return mSource->Start(mAllocationHandle); } nsresult MediaDevice::Reconfigure(const dom::MediaTrackConstraints &aConstraints, const MediaEnginePrefs &aPrefs, const char** aOutBadConstraint) { return mSource->Reconfigure(mAllocationHandle, aConstraints, aPrefs, mID, aOutBadConstraint); } -nsresult MediaDevice::Stop(SourceMediaStream* aStream, TrackID aTrackID) +nsresult MediaDevice::Stop() { - return mSource->Stop(aStream, aTrackID); + return mSource->Stop(mAllocationHandle); } nsresult MediaDevice::Deallocate() { return mSource->Deallocate(mAllocationHandle); } void MediaDevice::Pull(const RefPtr& aStream, @@ -1224,35 +1228,40 @@ public: // notification lambda to ensure it's kept alive until that lambda runs or is discarded. RefPtr self = this; MediaManager::PostTask(NewTaskFrom([self, domStream, callback]() mutable { MOZ_ASSERT(MediaManager::IsInMediaThread()); SourceMediaStream* source = self->mSourceListener->GetSourceStream(); RefPtr error = nullptr; if (self->mAudioDevice) { - nsresult rv = self->mAudioDevice->Start(source, - kAudioTrack, - self->mSourceListener->GetPrincipalHandle()); - if (NS_FAILED(rv)) { + nsresult rv = self->mAudioDevice->SetTrack(source, + kAudioTrack, + self->mSourceListener->GetPrincipalHandle()); + if (NS_SUCCEEDED(rv)) { + rv = self->mAudioDevice->Start(); + } else { nsString log; if (rv == NS_ERROR_NOT_AVAILABLE) { log.AssignASCII("Concurrent mic process limit."); error = new MediaMgrError(NS_LITERAL_STRING("NotReadableError"), log); } else { log.AssignASCII("Starting audio failed"); error = new MediaMgrError(NS_LITERAL_STRING("InternalError"), log); } } } if (!error && self->mVideoDevice) { - nsresult rv = self->mVideoDevice->Start(source, - kVideoTrack, - self->mSourceListener->GetPrincipalHandle()); + nsresult rv = self->mVideoDevice->SetTrack(source, + kVideoTrack, + self->mSourceListener->GetPrincipalHandle()); + if (NS_SUCCEEDED(rv)) { + rv = self->mVideoDevice->Start(); + } if (NS_FAILED(rv)) { nsString log; log.AssignASCII("Starting video failed"); error = new MediaMgrError(NS_LITERAL_STRING("InternalError"), log); } } if (error) { @@ -3811,18 +3820,18 @@ SourceListener::StopTrack(TrackID aTrack break; } default: { MOZ_ASSERT(false, "Unknown track id"); return; } } - MediaManager::PostTask(NewTaskFrom([device, stream = mStream, aTrackID]() { - device->Stop(stream, aTrackID); + MediaManager::PostTask(NewTaskFrom([device]() { + device->Stop(); device->Deallocate(); })); if ((!mAudioDevice || mAudioStopped) && (!mVideoDevice || mVideoStopped)) { LOG(("SourceListener %p this was the last track stopped", this)); Stop(); } diff --git a/dom/media/MediaManager.h b/dom/media/MediaManager.h --- a/dom/media/MediaManager.h +++ b/dom/media/MediaManager.h @@ -77,24 +77,24 @@ public: uint32_t GetBestFitnessDistance( const nsTArray& aConstraintSets, bool aIsChrome); nsresult Allocate(const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const mozilla::ipc::PrincipalInfo& aPrincipalInfo, const char** aOutBadConstraint); - nsresult Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipal); + nsresult SetTrack(const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal); + nsresult Start(); nsresult Reconfigure(const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const char** aOutBadConstraint); - nsresult Stop(SourceMediaStream* aStream, - TrackID aTrackID); + nsresult Stop(); nsresult Deallocate(); void Pull(const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, const PrincipalHandle& aPrincipal); void GetSettings(dom::MediaTrackSettings& aOutSettings) const; diff --git a/dom/media/webrtc/MediaEngineDefault.cpp b/dom/media/webrtc/MediaEngineDefault.cpp --- a/dom/media/webrtc/MediaEngineDefault.cpp +++ b/dom/media/webrtc/MediaEngineDefault.cpp @@ -122,18 +122,24 @@ MediaEngineDefaultVideoSource::Deallocat { AssertIsOnOwningThread(); MOZ_ASSERT(!aHandle); MOZ_ASSERT(!mImage); MOZ_ASSERT(mState == kStopped || mState == kAllocated); MutexAutoLock lock(mMutex); + if (mStream && IsTrackIDExplicit(mTrackID)) { + mStream->EndTrack(mTrackID); + mStream = nullptr; + mTrackID = TRACK_NONE; + } mState = kReleased; mImageContainer = nullptr; + return NS_OK; } static void AllocateSolidColorFrame(layers::PlanarYCbCrData& aData, int aWidth, int aHeight, int aY, int aCb, int aCr) { MOZ_ASSERT(!(aWidth&1)); @@ -161,23 +167,45 @@ static void AllocateSolidColorFrame(laye } static void ReleaseFrame(layers::PlanarYCbCrData& aData) { free(aData.mYChannel); } nsresult -MediaEngineDefaultVideoSource::Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipalHandle) +MediaEngineDefaultVideoSource::SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) { AssertIsOnOwningThread(); - MOZ_ASSERT(mState == kAllocated, "Allocate() must happen before Start()"); + MOZ_ASSERT(mState == kAllocated); + MOZ_ASSERT(!mStream); + MOZ_ASSERT(mTrackID == TRACK_NONE); + + { + MutexAutoLock lock(mMutex); + mStream = aStream; + mTrackID = aTrackID; + } + aStream->AddTrack(aTrackID, 0, new VideoSegment(), + SourceMediaStream::ADDTRACK_QUEUED); + return NS_OK; +} + +nsresult +MediaEngineDefaultVideoSource::Start(const RefPtr& aHandle) +{ + AssertIsOnOwningThread(); + + MOZ_ASSERT(mState == kAllocated || mState == kStopped); + MOZ_ASSERT(mStream, "SetTrack() must happen before Start()"); + MOZ_ASSERT(IsTrackIDExplicit(mTrackID), "SetTrack() must happen before Start()"); mTimer = NS_NewTimer(); if (!mTimer) { return NS_ERROR_FAILURE; } if (!mImageContainer) { mImageContainer = @@ -194,48 +222,39 @@ MediaEngineDefaultVideoSource::Start(Sou #endif mTimer->InitWithNamedFuncCallback([](nsITimer* aTimer, void* aClosure) { RefPtr source = static_cast(aClosure); source->GenerateFrame(); }, this, interval, nsITimer::TYPE_REPEATING_SLACK, "MediaEngineDefaultVideoSource::GenerateFrame"); - aStream->AddTrack(aTrackID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED); - MutexAutoLock lock(mMutex); - // Remember Stream and TrackID so we can end it later - mStream = aStream; - mTrackID = aTrackID; - mState = kStarted; return NS_OK; } nsresult -MediaEngineDefaultVideoSource::Stop(SourceMediaStream *aStream, TrackID aTrackID) +MediaEngineDefaultVideoSource::Stop(const RefPtr& aHandle) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == kStarted); MOZ_ASSERT(mTimer); + MOZ_ASSERT(mStream); + MOZ_ASSERT(IsTrackIDExplicit(mTrackID)); mTimer->Cancel(); mTimer = nullptr; - aStream->EndTrack(aTrackID); MutexAutoLock lock(mMutex); - MOZ_ASSERT(mStream == aStream); - MOZ_ASSERT(mTrackID == aTrackID); - mStream = nullptr; - mTrackID = TRACK_NONE; mImage = nullptr; + mState = kStopped; - mState = kStopped; return NS_OK; } nsresult MediaEngineDefaultVideoSource::Reconfigure( const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs &aPrefs, @@ -308,23 +327,29 @@ MediaEngineDefaultVideoSource::Pull(cons const PrincipalHandle& aPrincipalHandle) { // AppendFrame takes ownership of `segment` VideoSegment segment; RefPtr image; { MutexAutoLock lock(mMutex); - if (mState != kStarted) { + // Started - append real image + // Stopped - append null + // Released - Track is ended, safe to ignore + // Can happen because NotifyPull comes from a stream listener + if (mState == kReleased) { return; } - - MOZ_ASSERT(mStream == aStream); - MOZ_ASSERT(mTrackID == aTrackID); - image = mImage; + MOZ_ASSERT(mState != kAllocated); + if (mState == kStarted) { + MOZ_ASSERT(mStream == aStream); + MOZ_ASSERT(mTrackID == aTrackID); + image = mImage; + } } StreamTime delta = aDesiredTime - aStream->GetEndOfAppendedData(aTrackID); if (delta > 0) { // nullptr images are allowed IntSize size(image ? mOpts.mWidth : 0, image ? mOpts.mHeight : 0); segment.AppendFrame(image.forget(), delta, size, aPrincipalHandle); // This can fail if either a) we haven't added the track yet, or b) @@ -400,60 +425,76 @@ nsresult MediaEngineDefaultAudioSource::Deallocate(const RefPtr& aHandle) { AssertIsOnOwningThread(); MOZ_ASSERT(!aHandle); MOZ_ASSERT(mState == kStopped || mState == kAllocated); MutexAutoLock lock(mMutex); + if (mStream && IsTrackIDExplicit(mTrackID)) { + mStream->EndTrack(mTrackID); + mStream = nullptr; + mTrackID = TRACK_NONE; + } mState = kReleased; return NS_OK; } nsresult -MediaEngineDefaultAudioSource::Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipalHandle) +MediaEngineDefaultAudioSource::SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) { - AssertIsOnOwningThread(); MOZ_ASSERT(mState == kAllocated); + MOZ_ASSERT(!mStream); + MOZ_ASSERT(mTrackID == TRACK_NONE); // AddAudioTrack will take ownership of segment mStream = aStream; mTrackID = aTrackID; aStream->AddAudioTrack(aTrackID, aStream->GraphRate(), 0, new AudioSegment(), SourceMediaStream::ADDTRACK_QUEUED); + return NS_OK; +} + +nsresult +MediaEngineDefaultAudioSource::Start(const RefPtr& aHandle) +{ + AssertIsOnOwningThread(); + + MOZ_ASSERT(mState == kAllocated || mState == kStopped); + MOZ_ASSERT(mStream, "SetTrack() must happen before Start()"); + MOZ_ASSERT(IsTrackIDExplicit(mTrackID), "SetTrack() must happen before Start()"); if (!mSineGenerator) { // generate sine wave (default 1KHz) - mSineGenerator = new SineWaveGenerator(aStream->GraphRate(), mFreq); + mSineGenerator = new SineWaveGenerator(mStream->GraphRate(), mFreq); } mLastNotify = 0; MutexAutoLock lock(mMutex); mState = kStarted; return NS_OK; } nsresult -MediaEngineDefaultAudioSource::Stop(SourceMediaStream *aStream, - TrackID aTrackID) +MediaEngineDefaultAudioSource::Stop(const RefPtr& aHandle) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == kStarted); - aStream->EndTrack(aTrackID); MutexAutoLock lock(mMutex); mState = kStopped; return NS_OK; } nsresult MediaEngineDefaultAudioSource::Reconfigure( diff --git a/dom/media/webrtc/MediaEngineDefault.h b/dom/media/webrtc/MediaEngineDefault.h --- a/dom/media/webrtc/MediaEngineDefault.h +++ b/dom/media/webrtc/MediaEngineDefault.h @@ -49,23 +49,27 @@ public: nsCString GetUUID() const override; nsresult Allocate(const dom::MediaTrackConstraints &aConstraints, const MediaEnginePrefs &aPrefs, const nsString& aDeviceId, const ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle, const char** aOutBadConstraint) override; - nsresult Start(SourceMediaStream*, TrackID, const PrincipalHandle&) override; + nsresult SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) override; + nsresult Start(const RefPtr& aHandle) override; nsresult Reconfigure(const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const char** aOutBadConstraint) override; - nsresult Stop(SourceMediaStream*, TrackID) override; + nsresult Stop(const RefPtr& aHandle) override; nsresult Deallocate(const RefPtr& aHandle) override; void Pull(const RefPtr& aHandle, const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, const PrincipalHandle& aPrincipalHandle) override; uint32_t GetBestFitnessDistance( @@ -125,23 +129,27 @@ public: nsCString GetUUID() const override; nsresult Allocate(const dom::MediaTrackConstraints &aConstraints, const MediaEnginePrefs &aPrefs, const nsString& aDeviceId, const ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle, const char** aOutBadConstraint) override; - nsresult Start(SourceMediaStream*, TrackID, const PrincipalHandle&) override; + nsresult SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) override; + nsresult Start(const RefPtr& aHandle) override; nsresult Reconfigure(const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const char** aOutBadConstraint) override; - nsresult Stop(SourceMediaStream*, TrackID) override; + nsresult Stop(const RefPtr& aHandle) override; nsresult Deallocate(const RefPtr& aHandle) override; void inline AppendToSegment(AudioSegment& aSegment, TrackTicks aSamples, const PrincipalHandle& aPrincipalHandle); void Pull(const RefPtr& aHandle, const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp --- a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp @@ -76,17 +76,17 @@ MediaEngineRemoteVideoSource::Shutdown() if (!mInitDone) { // Already shut down return; } // Allocate always returns a null AllocationHandle. // We can safely pass nullptr here. if (mState == kStarted) { - Stop(mStream, mTrackID); + Stop(nullptr); } if (mState == kAllocated || mState == kStopped) { Deallocate(nullptr); } MOZ_ASSERT(mState == kReleased); mInitDone = false; } @@ -174,21 +174,17 @@ MediaEngineRemoteVideoSource::Allocate( const nsString& aDeviceId, const mozilla::ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle, const char** aOutBadConstraint) { LOG((__PRETTY_FUNCTION__)); AssertIsOnOwningThread(); - if (!mInitDone) { - LOG(("Init not done")); - return NS_ERROR_FAILURE; - } - + MOZ_ASSERT(mInitDone); MOZ_ASSERT(mState == kReleased); NormalizedConstraints constraints(aConstraints); LOG(("ChooseCapability(kFitness) for mTargetCapability and mCapability (Allocate) ++")); if (!ChooseCapability(constraints, aPrefs, aDeviceId, mCapability, kFitness)) { *aOutBadConstraint = MediaConstraintsHelper::FindBadConstraint(constraints, this, aDeviceId); return NS_ERROR_FAILURE; @@ -218,16 +214,18 @@ MediaEngineRemoteVideoSource::Deallocate { LOG((__PRETTY_FUNCTION__)); AssertIsOnOwningThread(); MOZ_ASSERT(mState == kStopped || mState == kAllocated); MOZ_ASSERT(mStream); MOZ_ASSERT(IsTrackIDExplicit(mTrackID)); + mStream->EndTrack(mTrackID); + { MutexAutoLock lock(mMutex); mStream = nullptr; mTrackID = TRACK_NONE; mPrincipal = PRINCIPAL_HANDLE_NONE; mState = kReleased; } @@ -242,88 +240,97 @@ MediaEngineRemoteVideoSource::Deallocate if (camera::GetChildAndCall(&camera::CamerasChild::ReleaseCaptureDevice, mCapEngine, mCaptureIndex)) { MOZ_ASSERT_UNREACHABLE("Couldn't release allocated device"); } return NS_OK; } nsresult -MediaEngineRemoteVideoSource::Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipal) +MediaEngineRemoteVideoSource::SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) { LOG((__PRETTY_FUNCTION__)); AssertIsOnOwningThread(); + MOZ_ASSERT(mState == kAllocated); + MOZ_ASSERT(!mStream); + MOZ_ASSERT(mTrackID == TRACK_NONE); MOZ_ASSERT(aStream); MOZ_ASSERT(IsTrackIDExplicit(aTrackID)); if (!mImageContainer) { mImageContainer = layers::LayerManager::CreateImageContainer( layers::ImageContainer::ASYNCHRONOUS); } { MutexAutoLock lock(mMutex); mStream = aStream; mTrackID = aTrackID; mPrincipal = aPrincipal; + } + aStream->AddTrack(aTrackID, 0, new VideoSegment(), + SourceMediaStream::ADDTRACK_QUEUED); + return NS_OK; +} + +nsresult +MediaEngineRemoteVideoSource::Start(const RefPtr& aHandle) +{ + LOG((__PRETTY_FUNCTION__)); + AssertIsOnOwningThread(); + + MOZ_ASSERT(mInitDone); + MOZ_ASSERT(mState == kAllocated || mState == kStopped); + MOZ_ASSERT(mStream); + MOZ_ASSERT(IsTrackIDExplicit(mTrackID)); + + { + MutexAutoLock lock(mMutex); mState = kStarted; } if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture, mCapEngine, mCaptureIndex, mCapability, this)) { LOG(("StartCapture failed")); MutexAutoLock lock(mMutex); - mStream = nullptr; - mTrackID = TRACK_NONE; - mPrincipal = PRINCIPAL_HANDLE_NONE; mState = kStopped; return NS_ERROR_FAILURE; } - NS_DispatchToMainThread(media::NewRunnableFrom([settings = mSettings]() mutable { - settings->mWidth.Construct(0); - settings->mHeight.Construct(0); - settings->mFrameRate.Construct(0); - return NS_OK; - })); - - mStream->AddTrack(mTrackID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED); return NS_OK; } nsresult -MediaEngineRemoteVideoSource::Stop(SourceMediaStream* aStream, - TrackID aTrackID) +MediaEngineRemoteVideoSource::Stop(const RefPtr& aHandle) { LOG((__PRETTY_FUNCTION__)); AssertIsOnOwningThread(); MOZ_ASSERT(mState == kStarted); - aStream->EndTrack(aTrackID); + if (camera::GetChildAndCall(&camera::CamerasChild::StopCapture, + mCapEngine, mCaptureIndex)) { + MOZ_DIAGNOSTIC_ASSERT(false, "Stopping a started capture failed"); + } { MutexAutoLock lock(mMutex); mState = kStopped; // Drop any cached image so we don't start with a stale image on next // usage. Also, gfx gets very upset if these are held until this object // is gc'd in final-cc during shutdown (bug 1374164) mImage = nullptr; } - if (camera::GetChildAndCall(&camera::CamerasChild::StopCapture, - mCapEngine, mCaptureIndex)) { - MOZ_DIAGNOSTIC_ASSERT(false, "Stopping a started capture failed"); - } - return NS_OK; } nsresult MediaEngineRemoteVideoSource::Reconfigure(const RefPtr& aHandle, const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, @@ -350,22 +357,22 @@ MediaEngineRemoteVideoSource::Reconfigur // Start() applies mCapability on the device. mCapability = newCapability; if (mState == kStarted) { // Allocate always returns a null AllocationHandle. // We can safely pass nullptr below. - nsresult rv = Stop(mStream, mTrackID); + nsresult rv = Stop(nullptr); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } - rv = Start(mStream, mTrackID, mPrincipal); + rv = Start(nullptr); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } } return NS_OK; } @@ -405,27 +412,32 @@ MediaEngineRemoteVideoSource::GetCapabil void MediaEngineRemoteVideoSource::Pull(const RefPtr& aHandle, const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, const PrincipalHandle& aPrincipalHandle) { MutexAutoLock lock(mMutex); + if (mState == kReleased) { + // We end the track before deallocating, so this is safe. + return; + } + MOZ_ASSERT(mState == kStarted || mState == kStopped); StreamTime delta = aDesiredTime - aStream->GetEndOfAppendedData(aTrackID); if (delta <= 0) { return; } VideoSegment segment; RefPtr image = mImage; - if (image) { - MOZ_ASSERT(mImageSize == image->GetSize()); + if (mState == kStarted) { + MOZ_ASSERT(!image || mImageSize == image->GetSize()); segment.AppendFrame(image.forget(), delta, mImageSize, aPrincipalHandle); } else { // nullptr images are allowed, but we force it to black and retain the size. segment.AppendFrame(image.forget(), delta, mImageSize, aPrincipalHandle, true); } // This is safe from any thread, and is safe if the track is Finished // or Destroyed. diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.h b/dom/media/webrtc/MediaEngineRemoteVideoSource.h --- a/dom/media/webrtc/MediaEngineRemoteVideoSource.h +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.h @@ -123,23 +123,27 @@ public: } nsresult Allocate(const dom::MediaTrackConstraints &aConstraints, const MediaEnginePrefs &aPrefs, const nsString& aDeviceId, const ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle, const char** aOutBadConstraint) override; nsresult Deallocate(const RefPtr& aHandle) override; - nsresult Start(SourceMediaStream*, TrackID, const PrincipalHandle&) override; + nsresult SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) override; + nsresult Start(const RefPtr& aHandle) override; nsresult Reconfigure(const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const char** aOutBadConstraint) override; - nsresult Stop(SourceMediaStream*, TrackID) override; + nsresult Stop(const RefPtr& aHandle) override; void Pull(const RefPtr& aHandle, const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, const PrincipalHandle& aPrincipalHandle) override; void GetSettings(dom::MediaTrackSettings& aOutSettings) const override; diff --git a/dom/media/webrtc/MediaEngineSource.h b/dom/media/webrtc/MediaEngineSource.h --- a/dom/media/webrtc/MediaEngineSource.h +++ b/dom/media/webrtc/MediaEngineSource.h @@ -132,40 +132,57 @@ public: virtual nsresult Allocate(const dom::MediaTrackConstraints &aConstraints, const MediaEnginePrefs &aPrefs, const nsString& aDeviceId, const mozilla::ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle, const char** aOutBadConstraint) = 0; /** - * Start the device and add the track to the provided SourceMediaStream, with - * the provided TrackID. You may start appending data to the track - * immediately after. + * Called by MediaEngine when a SourceMediaStream and TrackID have been + * provided for the given AllocationHandle to feed data to. + * + * This must be called before Start for the given AllocationHandle. */ - virtual nsresult Start(SourceMediaStream*, TrackID, const PrincipalHandle&) = 0; + virtual nsresult SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) = 0; + + /** + * Called by MediaEngine to start feeding data to the track associated with + * the given AllocationHandle. + * + * If this is the first AllocationHandle to start, the underlying device + * will be started. + */ + virtual nsresult Start(const RefPtr& aHandle) = 0; /** * Applies new constraints to the capability selection for the underlying * device. * * Should the constraints lead to choosing a new capability while the device * is actively being captured, the device will restart using the new * capability. */ virtual nsresult Reconfigure(const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const char** aOutBadConstraint) = 0; /** - * Stop the device and release the corresponding MediaStream. + * Called by MediaEngine to stop feeding data to the track associated with + * the given AllocationHandle. + * + * If this was the last AllocationHandle that had been started, + * the underlying device will be stopped. */ - virtual nsresult Stop(SourceMediaStream *aSource, TrackID aID) = 0; + virtual nsresult Stop(const RefPtr& aHandle) = 0; /** * Called by MediaEngine to deallocate a handle to this source. * * If this was the last registered AllocationHandle, the underlying device * will be deallocated. */ virtual nsresult Deallocate(const RefPtr& aHandle) = 0; diff --git a/dom/media/webrtc/MediaEngineTabVideoSource.cpp b/dom/media/webrtc/MediaEngineTabVideoSource.cpp --- a/dom/media/webrtc/MediaEngineTabVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineTabVideoSource.cpp @@ -205,38 +205,47 @@ MediaEngineTabVideoSource::Deallocate(co MutexAutoLock lock(mMutex); mState = kReleased; } return NS_OK; } nsresult -MediaEngineTabVideoSource::Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipalHandle) +MediaEngineTabVideoSource::SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const mozilla::PrincipalHandle& aPrincipal) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == kAllocated); MOZ_ASSERT(!mStream); MOZ_ASSERT(mTrackID == TRACK_NONE); MOZ_ASSERT(aStream); MOZ_ASSERT(IsTrackIDExplicit(aTrackID)); + mStream = aStream; + mTrackID = aTrackID; + mStream->AddTrack(mTrackID, 0, new VideoSegment()); + return NS_OK; +} + +nsresult +MediaEngineTabVideoSource::Start(const RefPtr& aHandle) +{ + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == kAllocated); nsCOMPtr runnable; if (!mWindow) { runnable = new InitRunnable(this); } else { runnable = new StartRunnable(this); } NS_DispatchToMainThread(runnable); - mStream = aStream; - mTrackID = aTrackID; - mStream->AddTrack(mTrackID, 0, new VideoSegment()); { MutexAutoLock lock(mMutex); mState = kStarted; } return NS_OK; } @@ -245,36 +254,37 @@ void MediaEngineTabVideoSource::Pull(const RefPtr& aHandle, const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, const PrincipalHandle& aPrincipalHandle) { VideoSegment segment; RefPtr image; + gfx::IntSize imageSize; { MutexAutoLock lock(mMutex); if (mState == kReleased) { // We end the track before setting the state to released. return; } - image = mImage; + if (mState == kStarted) { + image = mImage; + imageSize = mImageSize; + } } - // Note: we're not giving up mImage here StreamTime delta = aDesiredTime - aStream->GetEndOfAppendedData(aTrackID); if (delta <= 0) { return; } // nullptr images are allowed - gfx::IntSize size = image ? image->GetSize() : IntSize(0, 0); - segment.AppendFrame(image.forget(), delta, size, - aPrincipalHandle); + segment.AppendFrame(image.forget(), delta, imageSize, aPrincipalHandle); // This can fail if either a) we haven't added the track yet, or b) // we've removed or ended the track. aStream->AppendToTrack(aTrackID, &(segment)); } void MediaEngineTabVideoSource::Draw() { if (!mWindow && !mBlackedoutWindow) { @@ -378,38 +388,33 @@ MediaEngineTabVideoSource::Draw() { if (!surface) { return; } RefPtr image = new layers::SourceSurfaceImage(size, surface); MutexAutoLock lock(mMutex); mImage = image; + mImageSize = size; } nsresult -MediaEngineTabVideoSource::Stop(SourceMediaStream* aStream, - TrackID aTrackID) +MediaEngineTabVideoSource::Stop(const RefPtr& aHandle) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == kStarted); - MOZ_ASSERT(mStream == aStream); - MOZ_ASSERT(mTrackID == aTrackID); // If mBlackedoutWindow is true, we may be running // despite mWindow == nullptr. if (!mWindow && !mBlackedoutWindow) { return NS_OK; } NS_DispatchToMainThread(new StopRunnable(this)); { MutexAutoLock lock(mMutex); mState = kStopped; - mStream->EndTrack(mTrackID); - mStream = nullptr; - mTrackID = TRACK_NONE; } return NS_OK; } } diff --git a/dom/media/webrtc/MediaEngineTabVideoSource.h b/dom/media/webrtc/MediaEngineTabVideoSource.h --- a/dom/media/webrtc/MediaEngineTabVideoSource.h +++ b/dom/media/webrtc/MediaEngineTabVideoSource.h @@ -37,23 +37,27 @@ public: nsresult Allocate(const dom::MediaTrackConstraints &aConstraints, const MediaEnginePrefs &aPrefs, const nsString& aDeviceId, const ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle, const char** aOutBadConstraint) override; nsresult Deallocate(const RefPtr& aHandle) override; - nsresult Start(SourceMediaStream*, TrackID, const PrincipalHandle&) override; + nsresult SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) override; + nsresult Start(const RefPtr& aHandle) override; nsresult Reconfigure(const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const char** aOutBadConstraint) override; - nsresult Stop(SourceMediaStream*, TrackID) override; + nsresult Stop(const RefPtr& aHandle) override; void Pull(const RefPtr& aHandle, const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, const PrincipalHandle& aPrincipalHandle) override; uint32_t GetBestFitnessDistance( @@ -122,21 +126,23 @@ private: size_t mDataSize = 0; nsCOMPtr mWindow; // If this is set, we will run despite mWindow == nullptr. bool mBlackedoutWindow = false; // Current state of this source. // Written on owning thread *and* under mMutex. // Can be read on owning thread *or* under mMutex. MediaEngineSourceState mState = kReleased; - // mStream and mTrackID are set in SetSource() to keep track of what to end + // mStream and mTrackID are set in SetTrack() to keep track of what to end // in Deallocate(). // Owning thread only. RefPtr mStream; TrackID mTrackID = TRACK_NONE; + // mImage and mImageSize is Protected by mMutex. RefPtr mImage; + gfx::IntSize mImageSize; nsCOMPtr mTimer; Mutex mMutex; nsCOMPtr mTabSource; }; } // namespace mozilla diff --git a/dom/media/webrtc/MediaEngineWebRTC.h b/dom/media/webrtc/MediaEngineWebRTC.h --- a/dom/media/webrtc/MediaEngineWebRTC.h +++ b/dom/media/webrtc/MediaEngineWebRTC.h @@ -84,42 +84,34 @@ public: return NS_OK; } nsresult Deallocate(const RefPtr& aHandle) override { // Nothing to do here, everything is managed in MediaManager.cpp MOZ_ASSERT(!aHandle); return NS_OK; } - nsresult Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipalHandle) override; - nsresult Stop(SourceMediaStream* aStream, TrackID aTrackID) override; + nsresult SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) override; + nsresult Start(const RefPtr& aHandle) override; + nsresult Stop(const RefPtr& aHandle) override; nsresult Reconfigure(const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const char** aOutBadConstraint) override; void Pull(const RefPtr& aHandle, const RefPtr& aStream, TrackID aTrackID, StreamTime aDesiredTime, const PrincipalHandle& aPrincipalHandle) override - { - // The AudioCapture setup code in MediaManager creates a dummy - // SourceMediaStream that is not actually exposed to content. - // We append null data here just to keep the MediaStreamGraph happy. - StreamTime delta = aDesiredTime - aStream->GetEndOfAppendedData(aTrackID); - if (delta > 0) { - AudioSegment segment; - segment.AppendNullData(delta); - aStream->AppendToTrack(aTrackID, &segment); - } - } + {} dom::MediaSourceEnum GetMediaSource() const override { return dom::MediaSourceEnum::AudioCapture; } nsresult TakePhoto(MediaEnginePhotoCallback* aCallback) override { @@ -418,20 +410,22 @@ public: nsresult Allocate(const dom::MediaTrackConstraints &aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle, const char** aOutBadConstraint) override; nsresult Deallocate(const RefPtr& aHandle) override; - nsresult Start(SourceMediaStream* aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipalHandle) override; - nsresult Stop(SourceMediaStream* aStream, TrackID aTrackID) override; + nsresult SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) override; + nsresult Start(const RefPtr& aHandle) override; + nsresult Stop(const RefPtr& aHandle) override; nsresult Reconfigure(const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs, const nsString& aDeviceId, const char** aOutBadConstraint) override; void Pull(const RefPtr& aHandle, const RefPtr& aStream, @@ -476,16 +470,17 @@ private: Allocation() = delete; explicit Allocation(const RefPtr& aHandle); ~Allocation(); const RefPtr mHandle; RefPtr mStream; TrackID mTrackID = TRACK_NONE; PrincipalHandle mPrincipal = PRINCIPAL_HANDLE_NONE; + bool mEnabled = false; }; /** * Used with nsTArray::IndexOf to locate an Allocation by a handle. */ class AllocationHandleComparator { public: bool Equals(const Allocation& aAllocation, @@ -526,16 +521,18 @@ private: void UpdateNSSettingsIfNeeded(bool aEnable, webrtc::NsModes aMode); void SetLastPrefs(const MediaEnginePrefs& aPrefs); // These allocate/configure and release the channel bool AllocChannel(); void FreeChannel(); + bool HasEnabledTrack() const; + template void InsertInGraph(const T* aBuffer, size_t aFrames, uint32_t aChannels); void PacketizeAndProcess(MediaStreamGraph* aGraph, const AudioDataValue* aBuffer, size_t aFrames, @@ -546,22 +543,23 @@ private: // This is true when all processing is disabled, we can skip // packetization, resampling and other processing passes. // Graph thread only. bool PassThrough() const; // Graph thread only. void SetPassThrough(bool aPassThrough); - RefPtr mAudioInput; + // Owning thread only. RefPtr mListener; // Note: shared across all microphone sources static int sChannelsOpen; + const RefPtr mAudioInput; const UniquePtr mAudioProcessing; // accessed from the GraphDriver thread except for deletion. nsAutoPtr> mPacketizerInput; nsAutoPtr> mPacketizerOutput; // mMutex protects some of our members off the owning thread. Mutex mMutex; diff --git a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp --- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp +++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -580,16 +580,19 @@ MediaEngineWebRTCMicrophoneSource::Alloc nsresult MediaEngineWebRTCMicrophoneSource::Deallocate(const RefPtr& aHandle) { AssertIsOnOwningThread(); size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator()); MOZ_ASSERT(i != mAllocations.NoIndex); + MOZ_ASSERT(!mAllocations[i].mEnabled, + "Source should be stopped for the track before removing"); + mAllocations[i].mStream->EndTrack(mAllocations[i].mTrackID); { MutexAutoLock lock(mMutex); mAllocations.RemoveElementAt(i); } if (mAllocations.IsEmpty()) { // If empty, no callbacks to deliver data should be occuring MOZ_ASSERT(mState != kReleased, "Source not allocated"); @@ -598,29 +601,31 @@ MediaEngineWebRTCMicrophoneSource::Deall LOG(("Audio device %d deallocated", mCapIndex)); } else { LOG(("Audio device %d deallocated but still in use", mCapIndex)); } return NS_OK; } nsresult -MediaEngineWebRTCMicrophoneSource::Start(SourceMediaStream *aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipal) +MediaEngineWebRTCMicrophoneSource::SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipal) { AssertIsOnOwningThread(); MOZ_ASSERT(aStream); MOZ_ASSERT(IsTrackIDExplicit(aTrackID)); // Until we fix bug 1400488 we need to block a second tab (OuterWindow) // from opening an already-open device. If it's the same tab, they // will share a Graph(), and we can allow it. if (!mAllocations.IsEmpty() && - aStream->Graph() != mAllocations[0].mStream->Graph()) { + mAllocations[0].mStream && + mAllocations[0].mStream->Graph() != aStream->Graph()) { return NS_ERROR_NOT_AVAILABLE; } Allocation* allocation = nullptr; for (Allocation& a : mAllocations) { if (!a.mStream) { // This assumes Allocate() is always followed by Start() before another // Allocate(). But this is changing in one of the coming patches anyway. @@ -644,68 +649,87 @@ MediaEngineWebRTCMicrophoneSource::Start aStream->GraphRate(), 0, segment, SourceMediaStream::ADDTRACK_QUEUED); // XXX Make this based on the pref. aStream->RegisterForAudioMixing(); - if (!mListener) { - mListener = new WebRTCAudioDataListener(this); + LOG(("Stream %p registered for microphone capture", aStream.get())); + return NS_OK; +} + +nsresult +MediaEngineWebRTCMicrophoneSource::Start(const RefPtr& aHandle) +{ + AssertIsOnOwningThread(); + + if (sChannelsOpen == 0) { + return NS_ERROR_FAILURE; } - // Make sure logger starts before capture - AsyncLatencyLogger::Get(true); + size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator()); + MOZ_ASSERT(i != mAllocations.NoIndex, "Can't start track that hasn't been added"); + Allocation& allocation = mAllocations[i]; - // Must be *before* StartSend() so it will notice we selected external input (full_duplex) - mAudioInput->StartRecording(allocation->mStream, mListener); + MOZ_ASSERT(!allocation.mEnabled, "Source already started"); + { + // This spans setting both the enabled state and mState. + MutexAutoLock lock(mMutex); + allocation.mEnabled = true; - if (mState == kStarted) { - return NS_OK; - } - MOZ_ASSERT(mState == kAllocated || mState == kStopped); + if (!mListener) { + mListener = new WebRTCAudioDataListener(this); + } + + // Make sure logger starts before capture + AsyncLatencyLogger::Get(true); - { - MutexAutoLock lock(mMutex); + // Must be *before* StartSend() so it will notice we selected external input (full_duplex) + mAudioInput->StartRecording(allocation.mStream, mListener); + + if (mState == kStarted) { + return NS_OK; + } + MOZ_ASSERT(mState == kAllocated || mState == kStopped); + mState = kStarted; } return NS_OK; } nsresult -MediaEngineWebRTCMicrophoneSource::Stop(SourceMediaStream *aStream, TrackID aTrackID) +MediaEngineWebRTCMicrophoneSource::Stop(const RefPtr& aHandle) { AssertIsOnOwningThread(); - aStream->EndTrack(aTrackID); + size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator()); + MOZ_ASSERT(i != mAllocations.NoIndex, "Cannot stop track that we don't know about"); + Allocation& allocation = mAllocations[i]; - class StreamComparator { - public: - bool Equals(const Allocation& aItem, - const RefPtr& aStream) const - { - return aItem.mStream == aStream; - } - }; - - MutexAutoLock lock(mMutex); - MOZ_ASSERT(mAllocations.RemoveElement(aStream, StreamComparator())); - - mAudioInput->StopRecording(aStream); - - if (!mAllocations.IsEmpty()) { - // Another track is keeping us from stopping + if (!allocation.mEnabled) { + // Already stopped - this is allowed return NS_OK; } - MOZ_ASSERT(mState == kStarted, "Should be started when stopping"); { + // This spans setting both the enabled state and mState. MutexAutoLock lock(mMutex); + allocation.mEnabled = false; + + mAudioInput->StopRecording(allocation.mStream); + + if (HasEnabledTrack()) { + // Another track is keeping us from stopping + return NS_OK; + } + + MOZ_ASSERT(mState == kStarted, "Should be started when stopping"); mState = kStopped; } if (mListener) { // breaks a cycle, since the WebRTCAudioDataListener has a RefPtr to us mListener->Shutdown(); mListener = nullptr; } @@ -1098,17 +1122,19 @@ MediaEngineWebRTCMicrophoneSource::Shutd // breaks a cycle, since the WebRTCAudioDataListener has a RefPtr to us mListener->Shutdown(); // Don't release the webrtc.org pointers yet until the Listener is (async) shutdown mListener = nullptr; } if (mState == kStarted) { for (const Allocation& allocation : mAllocations) { - Stop(allocation.mStream, allocation.mTrackID); + if (allocation.mEnabled) { + Stop(allocation.mHandle); + } } MOZ_ASSERT(mState == kStopped); } while (!mAllocations.IsEmpty()) { MOZ_ASSERT(mState == kAllocated || mState == kStopped); // on last Deallocate(), FreeChannel()s and DeInit()s if all channels are released Deallocate(mAllocations[0].mHandle); @@ -1137,32 +1163,50 @@ MediaEngineWebRTCAudioCaptureSource::Get uuid.ToProvidedString(uuidBuffer); asciiString.AssignASCII(uuidBuffer); // Remove {} and the null terminator return nsCString(Substring(asciiString, 1, NSID_LENGTH - 3)); } -nsresult -MediaEngineWebRTCAudioCaptureSource::Start(SourceMediaStream *aStream, - TrackID aTrackID, - const PrincipalHandle& aPrincipal) +bool +MediaEngineWebRTCMicrophoneSource::HasEnabledTrack() const { AssertIsOnOwningThread(); - aStream->AddTrack(aTrackID, 0, new AudioSegment()); + for (const Allocation& allocation : mAllocations) { + if (allocation.mEnabled) { + return true; + } + } + return false; +} + +nsresult +MediaEngineWebRTCAudioCaptureSource::SetTrack(const RefPtr& aHandle, + const RefPtr& aStream, + TrackID aTrackID, + const PrincipalHandle& aPrincipalHandle) +{ + AssertIsOnOwningThread(); + // Nothing to do here. aStream is a placeholder dummy and not exposed. return NS_OK; } nsresult -MediaEngineWebRTCAudioCaptureSource::Stop(SourceMediaStream *aStream, - TrackID aTrackID) +MediaEngineWebRTCAudioCaptureSource::Start(const RefPtr& aHandle) { AssertIsOnOwningThread(); - aStream->EndAllTrackAndFinish(); + return NS_OK; +} + +nsresult +MediaEngineWebRTCAudioCaptureSource::Stop(const RefPtr& aHandle) +{ + AssertIsOnOwningThread(); return NS_OK; } nsresult MediaEngineWebRTCAudioCaptureSource::Reconfigure( const RefPtr& aHandle, const dom::MediaTrackConstraints& aConstraints, const MediaEnginePrefs &aPrefs,