/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include #include "mozilla/dom/AudioContext.h" #include "mozilla/SharedThreadPool.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/Unused.h" #include "CubebUtils.h" #ifdef MOZ_WEBRTC #include "webrtc/MediaEngineWebRTC.h" #endif #ifdef XP_MACOSX #include #endif extern mozilla::LazyLogModule gMediaStreamGraphLog; #define STREAM_LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg) // We don't use NSPR log here because we want this interleaved with adb logcat // on Android/B2G // #define ENABLE_LIFECYCLE_LOG #ifdef ENABLE_LIFECYCLE_LOG #ifdef ANDROID #include "android/log.h" #define LIFECYCLE_LOG(...) __android_log_print(ANDROID_LOG_INFO, "Gecko - MSG" , __VA_ARGS__); printf(__VA_ARGS__);printf("\n"); #else #define LIFECYCLE_LOG(...) printf(__VA_ARGS__);printf("\n"); #endif #else #define LIFECYCLE_LOG(...) #endif namespace mozilla { StaticRefPtr AsyncCubebTask::sThreadPool; struct AutoProfilerUnregisterThread { // The empty ctor is used to silence a pre-4.8.0 GCC unused variable warning. AutoProfilerUnregisterThread() { } ~AutoProfilerUnregisterThread() { profiler_unregister_thread(); } }; GraphDriver::GraphDriver(MediaStreamGraphImpl* aGraphImpl) : mIterationStart(0), mIterationEnd(0), mGraphImpl(aGraphImpl), mWaitState(WAITSTATE_RUNNING), mCurrentTimeStamp(TimeStamp::Now()), mPreviousDriver(nullptr), mNextDriver(nullptr) { } void GraphDriver::SetGraphTime(GraphDriver* aPreviousDriver, GraphTime aLastSwitchNextIterationStart, GraphTime aLastSwitchNextIterationEnd) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); // We set mIterationEnd here, because the first thing a driver do when it // does an iteration is to update graph times, so we are in fact setting // mIterationStart of the next iteration by setting the end of the previous // iteration. mIterationStart = aLastSwitchNextIterationStart; mIterationEnd = aLastSwitchNextIterationEnd; MOZ_ASSERT(!PreviousDriver()); MOZ_ASSERT(aPreviousDriver); STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)", aPreviousDriver, aPreviousDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver")); SetPreviousDriver(aPreviousDriver); } void GraphDriver::SwitchAtNextIteration(GraphDriver* aNextDriver) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); LIFECYCLE_LOG("Switching to new driver: %p (%s)", aNextDriver, aNextDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"); if (mNextDriver && mNextDriver != GraphImpl()->CurrentDriver()) { LIFECYCLE_LOG("Discarding previous next driver: %p (%s)", mNextDriver.get(), mNextDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"); } SetNextDriver(aNextDriver); } GraphTime GraphDriver::StateComputedTime() const { return mGraphImpl->mStateComputedTime; } void GraphDriver::EnsureNextIteration() { mGraphImpl->EnsureNextIteration(); } void GraphDriver::Shutdown() { if (AsAudioCallbackDriver()) { LIFECYCLE_LOG("Releasing audio driver off main thread (GraphDriver::Shutdown).\n"); RefPtr releaseEvent = new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN); releaseEvent->Dispatch(NS_DISPATCH_SYNC); } else { Stop(); } } bool GraphDriver::Switching() { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); return mNextDriver || mPreviousDriver; } GraphDriver* GraphDriver::NextDriver() { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); return mNextDriver; } GraphDriver* GraphDriver::PreviousDriver() { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); return mPreviousDriver; } void GraphDriver::SetNextDriver(GraphDriver* aNextDriver) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); mNextDriver = aNextDriver; } void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); mPreviousDriver = aPreviousDriver; } ThreadedDriver::ThreadedDriver(MediaStreamGraphImpl* aGraphImpl) : GraphDriver(aGraphImpl) { } class MediaStreamGraphShutdownThreadRunnable : public Runnable { public: explicit MediaStreamGraphShutdownThreadRunnable(already_AddRefed aThread) : mThread(aThread) { } NS_IMETHOD Run() override { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mThread); mThread->Shutdown(); mThread = nullptr; return NS_OK; } private: nsCOMPtr mThread; }; ThreadedDriver::~ThreadedDriver() { if (mThread) { if (NS_IsMainThread()) { mThread->Shutdown(); } else { nsCOMPtr event = new MediaStreamGraphShutdownThreadRunnable(mThread.forget()); NS_DispatchToMainThread(event); } } } class MediaStreamGraphInitThreadRunnable : public Runnable { public: explicit MediaStreamGraphInitThreadRunnable(ThreadedDriver* aDriver) : mDriver(aDriver) { } NS_IMETHOD Run() override { char aLocal; STREAM_LOG(LogLevel::Debug, ("Starting system thread")); profiler_register_thread("MediaStreamGraph", &aLocal); LIFECYCLE_LOG("Starting a new system driver for graph %p\n", mDriver->mGraphImpl.get()); RefPtr previousDriver; { MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor()); previousDriver = mDriver->PreviousDriver(); } if (previousDriver) { LIFECYCLE_LOG("%p releasing an AudioCallbackDriver(%p), for graph %p\n", mDriver, previousDriver, mDriver->GraphImpl()); MOZ_ASSERT(!mDriver->AsAudioCallbackDriver()); RefPtr releaseEvent = new AsyncCubebTask(previousDriver->AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN); releaseEvent->Dispatch(); MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor()); mDriver->SetPreviousDriver(nullptr); } else { MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor()); MOZ_ASSERT(mDriver->mGraphImpl->MessagesQueued() || mDriver->mGraphImpl->mForceShutDown, "Don't start a graph without messages queued."); mDriver->mGraphImpl->SwapMessageQueues(); } mDriver->RunThread(); return NS_OK; } private: RefPtr mDriver; }; void ThreadedDriver::Start() { LIFECYCLE_LOG("Starting thread for a SystemClockDriver %p\n", mGraphImpl.get()); Unused << NS_WARN_IF(mThread); if (!mThread) { // Ensure we haven't already started it nsCOMPtr event = new MediaStreamGraphInitThreadRunnable(this); // Note: mThread may be null during event->Run() if we pass to NewNamedThread! See AudioInitTask nsresult rv = NS_NewNamedThread("MediaStreamGrph", getter_AddRefs(mThread)); if (NS_SUCCEEDED(rv)) { mThread->Dispatch(event, NS_DISPATCH_NORMAL); } } } void ThreadedDriver::Resume() { Start(); } void ThreadedDriver::Revive() { // Note: only called on MainThread, without monitor // We know were weren't in a running state STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver reviving.")); // If we were switching, switch now. Otherwise, tell thread to run the main // loop again. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (NextDriver()) { NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); } else { nsCOMPtr event = new MediaStreamGraphInitThreadRunnable(this); mThread->Dispatch(event, NS_DISPATCH_NORMAL); } } void ThreadedDriver::RemoveCallback() { } void ThreadedDriver::Stop() { NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread"); // mGraph's thread is not running so it's OK to do whatever here STREAM_LOG(LogLevel::Debug, ("Stopping threads for MediaStreamGraph %p", this)); if (mThread) { mThread->Shutdown(); mThread = nullptr; } } SystemClockDriver::SystemClockDriver(MediaStreamGraphImpl* aGraphImpl) : ThreadedDriver(aGraphImpl), mInitialTimeStamp(TimeStamp::Now()), mLastTimeStamp(TimeStamp::Now()), mIsFallback(false) {} SystemClockDriver::~SystemClockDriver() { } void SystemClockDriver::MarkAsFallback() { mIsFallback = true; } bool SystemClockDriver::IsFallback() { return mIsFallback; } void ThreadedDriver::RunThread() { AutoProfilerUnregisterThread autoUnregister; while (true) { mIterationStart = IterationEnd(); mIterationEnd += GetIntervalForIteration(); GraphTime stateComputedTime = StateComputedTime(); if (stateComputedTime < mIterationEnd) { STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected")); mIterationEnd = stateComputedTime; } if (mIterationStart >= mIterationEnd) { NS_ASSERTION(mIterationStart == mIterationEnd , "Time can't go backwards!"); // This could happen due to low clock resolution, maybe? STREAM_LOG(LogLevel::Debug, ("Time did not advance")); } GraphTime nextStateComputedTime = mGraphImpl->RoundUpToNextAudioBlock( mIterationEnd + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS)); if (nextStateComputedTime < stateComputedTime) { // A previous driver may have been processing further ahead of // iterationEnd. STREAM_LOG(LogLevel::Warning, ("Prevent state from going backwards. interval[%ld; %ld] state[%ld; %ld]", (long)mIterationStart, (long)mIterationEnd, (long)stateComputedTime, (long)nextStateComputedTime)); nextStateComputedTime = stateComputedTime; } STREAM_LOG(LogLevel::Verbose, ("interval[%ld; %ld] state[%ld; %ld]", (long)mIterationStart, (long)mIterationEnd, (long)stateComputedTime, (long)nextStateComputedTime)); bool stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime); if (!stillProcessing) { // Enter shutdown mode. The stable-state handler will detect this // and complete shutdown if the graph does not get restarted. mGraphImpl->SignalMainThreadCleanup(); return; } MonitorAutoLock lock(GraphImpl()->GetMonitor()); if (NextDriver()) { STREAM_LOG(LogLevel::Debug, ("Switching to AudioCallbackDriver")); RemoveCallback(); NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); return; } } } MediaTime SystemClockDriver::GetIntervalForIteration() { TimeStamp now = TimeStamp::Now(); MediaTime interval = mGraphImpl->SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds()); mCurrentTimeStamp = now; MOZ_LOG(gMediaStreamGraphLog, LogLevel::Verbose, ("Updating current time to %f (real %f, StateComputedTime() %f)", mGraphImpl->MediaTimeToSeconds(IterationEnd() + interval), (now - mInitialTimeStamp).ToSeconds(), mGraphImpl->MediaTimeToSeconds(StateComputedTime()))); return interval; } TimeStamp OfflineClockDriver::GetCurrentTimeStamp() { MOZ_CRASH("This driver does not support getting the current timestamp."); return TimeStamp(); } void SystemClockDriver::WaitForNextIteration() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); PRIntervalTime timeout = PR_INTERVAL_NO_TIMEOUT; TimeStamp now = TimeStamp::Now(); // This lets us avoid hitting the Atomic twice when we know we won't sleep bool another = mGraphImpl->mNeedAnotherIteration; // atomic if (!another) { mGraphImpl->mGraphDriverAsleep = true; // atomic mWaitState = WAITSTATE_WAITING_INDEFINITELY; } // NOTE: mNeedAnotherIteration while also atomic may have changed before // we could set mGraphDriverAsleep, so we must re-test it. // (EnsureNextIteration sets mNeedAnotherIteration, then tests // mGraphDriverAsleep if (another || mGraphImpl->mNeedAnotherIteration) { // atomic int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS - int64_t((now - mCurrentTimeStamp).ToMilliseconds()); // Make sure timeoutMS doesn't overflow 32 bits by waking up at // least once a minute, if we need to wake up at all timeoutMS = std::max(0, std::min(timeoutMS, 60*1000)); timeout = PR_MillisecondsToInterval(uint32_t(timeoutMS)); STREAM_LOG(LogLevel::Verbose, ("Waiting for next iteration; at %f, timeout=%f", (now - mInitialTimeStamp).ToSeconds(), timeoutMS/1000.0)); if (mWaitState == WAITSTATE_WAITING_INDEFINITELY) { mGraphImpl->mGraphDriverAsleep = false; // atomic } mWaitState = WAITSTATE_WAITING_FOR_NEXT_ITERATION; } if (timeout > 0) { mGraphImpl->GetMonitor().Wait(timeout); STREAM_LOG(LogLevel::Verbose, ("Resuming after timeout; at %f, elapsed=%f", (TimeStamp::Now() - mInitialTimeStamp).ToSeconds(), (TimeStamp::Now() - now).ToSeconds())); } if (mWaitState == WAITSTATE_WAITING_INDEFINITELY) { mGraphImpl->mGraphDriverAsleep = false; // atomic } // Note: this can race against the EnsureNextIteration setting // WAITSTATE_RUNNING and setting mGraphDriverAsleep to false, so you can // have an iteration with WAITSTATE_WAKING_UP instead of RUNNING. mWaitState = WAITSTATE_RUNNING; mGraphImpl->mNeedAnotherIteration = false; // atomic } void SystemClockDriver::WakeUp() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); // Note: this can race against the thread setting WAITSTATE_RUNNING and // setting mGraphDriverAsleep to false, so you can have an iteration // with WAITSTATE_WAKING_UP instead of RUNNING. mWaitState = WAITSTATE_WAKING_UP; mGraphImpl->mGraphDriverAsleep = false; // atomic mGraphImpl->GetMonitor().Notify(); } OfflineClockDriver::OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice) : ThreadedDriver(aGraphImpl), mSlice(aSlice) { } OfflineClockDriver::~OfflineClockDriver() { } MediaTime OfflineClockDriver::GetIntervalForIteration() { return mGraphImpl->MillisecondsToMediaTime(mSlice); } void OfflineClockDriver::WaitForNextIteration() { // No op: we want to go as fast as possible when we are offline } void OfflineClockDriver::WakeUp() { MOZ_ASSERT(false, "An offline graph should not have to wake up."); } AsyncCubebTask::AsyncCubebTask(AudioCallbackDriver* aDriver, AsyncCubebOperation aOperation) : mDriver(aDriver), mOperation(aOperation), mShutdownGrip(aDriver->GraphImpl()) { NS_WARNING_ASSERTION(mDriver->mAudioStream || aOperation == INIT, "No audio stream!"); } AsyncCubebTask::~AsyncCubebTask() { } /* static */ nsresult AsyncCubebTask::EnsureThread() { if (!sThreadPool) { nsCOMPtr threadPool = SharedThreadPool::Get(NS_LITERAL_CSTRING("CubebOperation"), 1); sThreadPool = threadPool; // Need to null this out before xpcom-shutdown-threads Observers run // since we don't know the order that the shutdown-threads observers // will run. ClearOnShutdown guarantees it runs first. if (!NS_IsMainThread()) { NS_DispatchToMainThread(NS_NewRunnableFunction([]() -> void { ClearOnShutdown(&sThreadPool, ShutdownPhase::ShutdownThreads); })); } else { ClearOnShutdown(&sThreadPool, ShutdownPhase::ShutdownThreads); } const uint32_t kIdleThreadTimeoutMs = 2000; nsresult rv = sThreadPool->SetIdleThreadTimeout(PR_MillisecondsToInterval(kIdleThreadTimeoutMs)); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } } return NS_OK; } NS_IMETHODIMP AsyncCubebTask::Run() { MOZ_ASSERT(mDriver); switch(mOperation) { case AsyncCubebOperation::INIT: { LIFECYCLE_LOG("AsyncCubebOperation::INIT driver=%p\n", mDriver.get()); mDriver->Init(); mDriver->CompleteAudioContextOperations(mOperation); break; } case AsyncCubebOperation::SHUTDOWN: { LIFECYCLE_LOG("AsyncCubebOperation::SHUTDOWN driver=%p\n", mDriver.get()); mDriver->Stop(); mDriver->CompleteAudioContextOperations(mOperation); mDriver = nullptr; mShutdownGrip = nullptr; break; } default: MOZ_CRASH("Operation not implemented."); } // The thread will kill itself after a bit return NS_OK; } StreamAndPromiseForOperation::StreamAndPromiseForOperation(MediaStream* aStream, void* aPromise, dom::AudioContextOperation aOperation) : mStream(aStream) , mPromise(aPromise) , mOperation(aOperation) { // MOZ_ASSERT(aPromise); } AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl) : GraphDriver(aGraphImpl) , mSampleRate(0) , mInputChannels(1) , mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS) , mStarted(false) , mAudioInput(nullptr) , mAudioChannel(aGraphImpl->AudioChannel()) , mAddedMixer(false) , mInCallback(false) , mMicrophoneActive(false) , mShouldFallbackIfError(false) , mFromFallback(false) { STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver ctor for graph %p", aGraphImpl)); } AudioCallbackDriver::~AudioCallbackDriver() { MOZ_ASSERT(mPromisesForOperation.IsEmpty()); } bool IsMacbookOrMacbookAir() { #ifdef XP_MACOSX size_t len = 0; sysctlbyname("hw.model", NULL, &len, NULL, 0); if (len) { UniquePtr model(new char[len]); // This string can be // MacBook%d,%d for a normal MacBook // MacBookPro%d,%d for a MacBook Pro // MacBookAir%d,%d for a Macbook Air sysctlbyname("hw.model", model.get(), &len, NULL, 0); char* substring = strstr(model.get(), "MacBook"); if (substring) { const size_t offset = strlen("MacBook"); if (strncmp(model.get() + offset, "Air", len - offset) || isdigit(model[offset + 1])) { return true; } } return false; } #endif return false; } void AudioCallbackDriver::Init() { cubeb* cubebContext = CubebUtils::GetCubebContext(); if (!cubebContext) { NS_WARNING("Could not get cubeb context."); return; } cubeb_stream_params output; cubeb_stream_params input; uint32_t latency_frames; MOZ_ASSERT(!NS_IsMainThread(), "This is blocking and should never run on the main thread."); mSampleRate = output.rate = CubebUtils::PreferredSampleRate(); #if defined(__ANDROID__) output.stream_type = CUBEB_STREAM_TYPE_MUSIC; if (output.stream_type == CUBEB_STREAM_TYPE_MAX) { NS_WARNING("Bad stream type"); return; } #else (void)mAudioChannel; #endif output.channels = mGraphImpl->AudioChannelCount(); if (AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_S16) { output.format = CUBEB_SAMPLE_S16NE; } else { output.format = CUBEB_SAMPLE_FLOAT32NE; } Maybe latencyPref = CubebUtils::GetCubebMSGLatencyInFrames(); if (latencyPref) { latency_frames = latencyPref.value(); } else { if (cubeb_get_min_latency(cubebContext, output, &latency_frames) != CUBEB_OK) { NS_WARNING("Could not get minimal latency from cubeb."); } } // Macbook and MacBook air don't have enough CPU to run very low latency // MediaStreamGraphs, cap the minimal latency to 512 frames int this case. if (IsMacbookOrMacbookAir()) { latency_frames = std::max((uint32_t) 512, latency_frames); } input = output; input.channels = mInputChannels; // change to support optional stereo capture cubeb_stream* stream = nullptr; CubebUtils::AudioDeviceID input_id = nullptr, output_id = nullptr; // We have to translate the deviceID values to cubeb devid's since those can be // freed whenever enumerate is called. { #ifdef MOZ_WEBRTC StaticMutexAutoLock lock(AudioInputCubeb::Mutex()); #endif if ((!mGraphImpl->mInputWanted #ifdef MOZ_WEBRTC || AudioInputCubeb::GetDeviceID(mGraphImpl->mInputDeviceID, input_id) #endif ) && (mGraphImpl->mOutputDeviceID == -1 // pass nullptr for ID for default output #ifdef MOZ_WEBRTC // XXX we should figure out how we would use a deviceID for output without webrtc. // Currently we don't set this though, so it's ok || AudioInputCubeb::GetDeviceID(mGraphImpl->mOutputDeviceID, output_id) #endif ) && // XXX Only pass input input if we have an input listener. Always // set up output because it's easier, and it will just get silence. // XXX Add support for adding/removing an input listener later. cubeb_stream_init(cubebContext, &stream, "AudioCallbackDriver", input_id, mGraphImpl->mInputWanted ? &input : nullptr, output_id, mGraphImpl->mOutputWanted ? &output : nullptr, latency_frames, DataCallback_s, StateCallback_s, this) == CUBEB_OK) { mAudioStream.own(stream); DebugOnly rv = cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale()); NS_WARNING_ASSERTION( rv == CUBEB_OK, "Could not set the audio stream volume in GraphDriver.cpp"); } else { #ifdef MOZ_WEBRTC StaticMutexAutoUnlock unlock(AudioInputCubeb::Mutex()); #endif NS_WARNING("Could not create a cubeb stream for MediaStreamGraph, falling back to a SystemClockDriver"); // Fall back to a driver using a normal thread. If needed, // the graph will try to re-open an audio stream later. MonitorAutoLock lock(GraphImpl()->GetMonitor()); SystemClockDriver* nextDriver = new SystemClockDriver(GraphImpl()); SetNextDriver(nextDriver); nextDriver->MarkAsFallback(); nextDriver->SetGraphTime(this, mIterationStart, mIterationEnd); // We're not using SwitchAtNextIteration here, because there // won't be a next iteration if we don't restart things manually: // the audio stream just signaled that it's in error state. mGraphImpl->SetCurrentDriver(nextDriver); nextDriver->Start(); return; } } bool aec; Unused << mGraphImpl->AudioTrackPresent(aec); SetMicrophoneActive(aec); cubeb_stream_register_device_changed_callback(mAudioStream, AudioCallbackDriver::DeviceChangedCallback_s); StartStream(); STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver started.")); } void AudioCallbackDriver::Destroy() { STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver destroyed.")); mAudioInput = nullptr; mAudioStream.reset(); } void AudioCallbackDriver::Resume() { STREAM_LOG(LogLevel::Debug, ("Resuming audio threads for MediaStreamGraph %p", mGraphImpl.get())); if (cubeb_stream_start(mAudioStream) != CUBEB_OK) { NS_WARNING("Could not start cubeb stream for MSG."); } } void AudioCallbackDriver::Start() { if (mPreviousDriver) { if (mPreviousDriver->AsAudioCallbackDriver()) { LIFECYCLE_LOG("Releasing audio driver off main thread."); RefPtr releaseEvent = new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN); releaseEvent->Dispatch(); mPreviousDriver = nullptr; } else { LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver."); MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver()); mFromFallback = mPreviousDriver->AsSystemClockDriver()->IsFallback(); mPreviousDriver = nullptr; } } LIFECYCLE_LOG("Starting new audio driver off main thread, " "to ensure it runs after previous shutdown."); RefPtr initEvent = new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT); initEvent->Dispatch(); } void AudioCallbackDriver::StartStream() { mShouldFallbackIfError = true; if (cubeb_stream_start(mAudioStream) != CUBEB_OK) { MOZ_CRASH("Could not start cubeb stream for MSG."); } { MonitorAutoLock mon(mGraphImpl->GetMonitor()); mStarted = true; mWaitState = WAITSTATE_RUNNING; } } void AudioCallbackDriver::Stop() { if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) { NS_WARNING("Could not stop cubeb stream for MSG."); } } void AudioCallbackDriver::Revive() { // Note: only called on MainThread, without monitor // We know were weren't in a running state STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver reviving.")); // If we were switching, switch now. Otherwise, start the audio thread again. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (NextDriver()) { RemoveCallback(); NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); } else { STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from a new thread.", mGraphImpl.get())); RefPtr initEvent = new AsyncCubebTask(this, AsyncCubebOperation::INIT); initEvent->Dispatch(); } } void AudioCallbackDriver::RemoveCallback() { if (mAddedMixer) { mGraphImpl->mMixer.RemoveCallback(this); mAddedMixer = false; } } void AudioCallbackDriver::WaitForNextIteration() { } void AudioCallbackDriver::WakeUp() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); mGraphImpl->GetMonitor().Notify(); } /* static */ long AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream, void* aUser, const void* aInputBuffer, void* aOutputBuffer, long aFrames) { AudioCallbackDriver* driver = reinterpret_cast(aUser); return driver->DataCallback(static_cast(aInputBuffer), static_cast(aOutputBuffer), aFrames); } /* static */ void AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void * aUser, cubeb_state aState) { AudioCallbackDriver* driver = reinterpret_cast(aUser); driver->StateCallback(aState); } /* static */ void AudioCallbackDriver::DeviceChangedCallback_s(void* aUser) { AudioCallbackDriver* driver = reinterpret_cast(aUser); driver->DeviceChangedCallback(); } bool AudioCallbackDriver::InCallback() { return mInCallback; } AudioCallbackDriver::AutoInCallback::AutoInCallback(AudioCallbackDriver* aDriver) : mDriver(aDriver) { mDriver->mInCallback = true; } AudioCallbackDriver::AutoInCallback::~AutoInCallback() { mDriver->mInCallback = false; } long AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer, AudioDataValue* aOutputBuffer, long aFrames) { bool stillProcessing; // Don't add the callback until we're inited and ready if (!mAddedMixer) { mGraphImpl->mMixer.AddCallback(this); mAddedMixer = true; } #ifdef DEBUG // DebugOnly<> doesn't work here... it forces an initialization that will cause // mInCallback to be set back to false before we exit the statement. Do it by // hand instead. AutoInCallback aic(this); #endif GraphTime stateComputedTime = StateComputedTime(); if (stateComputedTime == 0) { MonitorAutoLock mon(mGraphImpl->GetMonitor()); // Because this function is called during cubeb_stream_init (to prefill the // audio buffers), it can be that we don't have a message here (because this // driver is the first one for this graph), and the graph would exit. Simply // return here until we have messages. if (!mGraphImpl->MessagesQueued()) { PodZero(aOutputBuffer, aFrames * mGraphImpl->AudioChannelCount()); return aFrames; } mGraphImpl->SwapMessageQueues(); } uint32_t durationMS = aFrames * 1000 / mSampleRate; // For now, simply average the duration with the previous // duration so there is some damping against sudden changes. if (!mIterationDurationMS) { mIterationDurationMS = durationMS; } else { mIterationDurationMS = (mIterationDurationMS*3) + durationMS; mIterationDurationMS /= 4; } // Process mic data if any/needed if (aInputBuffer) { if (mAudioInput) { // for this specific input-only or full-duplex stream mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer, static_cast(aFrames), mSampleRate, mInputChannels); } } mBuffer.SetBuffer(aOutputBuffer, aFrames); // fill part or all with leftover data from last iteration (since we // align to Audio blocks) mScratchBuffer.Empty(mBuffer); // if we totally filled the buffer (and mScratchBuffer isn't empty), // we don't need to run an iteration and if we do so we may overflow. if (mBuffer.Available()) { // State computed time is decided by the audio callback's buffer length. We // compute the iteration start and end from there, trying to keep the amount // of buffering in the graph constant. GraphTime nextStateComputedTime = mGraphImpl->RoundUpToNextAudioBlock(stateComputedTime + mBuffer.Available()); mIterationStart = mIterationEnd; // inGraph is the number of audio frames there is between the state time and // the current time, i.e. the maximum theoretical length of the interval we // could use as [mIterationStart; mIterationEnd]. GraphTime inGraph = stateComputedTime - mIterationStart; // We want the interval [mIterationStart; mIterationEnd] to be before the // interval [stateComputedTime; nextStateComputedTime]. We also want // the distance between these intervals to be roughly equivalent each time, to // ensure there is no clock drift between current time and state time. Since // we can't act on the state time because we have to fill the audio buffer, we // reclock the current time against the state time, here. mIterationEnd = mIterationStart + 0.8 * inGraph; STREAM_LOG(LogLevel::Verbose, ("interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) (duration ticks: %ld)\n", (long)mIterationStart, (long)mIterationEnd, (long)stateComputedTime, (long)nextStateComputedTime, (long)aFrames, (uint32_t)durationMS, (long)(nextStateComputedTime - stateComputedTime))); mCurrentTimeStamp = TimeStamp::Now(); if (stateComputedTime < mIterationEnd) { STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected")); mIterationEnd = stateComputedTime; } stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime); } else { STREAM_LOG(LogLevel::Verbose, ("DataCallback buffer filled entirely from scratch buffer, skipping iteration.")); stillProcessing = true; } mBuffer.BufferFilled(); // Callback any observers for the AEC speaker data. Note that one // (maybe) of these will be full-duplex, the others will get their input // data off separate cubeb callbacks. Take care with how stuff is // removed/added to this list and TSAN issues, but input and output will // use separate callback methods. mGraphImpl->NotifyOutputData(aOutputBuffer, static_cast(aFrames), mSampleRate, ChannelCount); if (!stillProcessing) { // About to hand over control of the graph. Do not start a new driver if // StateCallback() receives an error for this stream while the main thread // or another driver has control of the graph. mShouldFallbackIfError = false; // Enter shutdown mode. The stable-state handler will detect this // and complete shutdown if the graph does not get restarted. mGraphImpl->SignalMainThreadCleanup(); return aFrames - 1; } bool switching = false; { MonitorAutoLock mon(mGraphImpl->GetMonitor()); switching = !!NextDriver(); } if (switching) { mShouldFallbackIfError = false; // If the audio stream has not been started by the previous driver or // the graph itself, keep it alive. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (!IsStarted()) { return aFrames; } STREAM_LOG(LogLevel::Debug, ("Switching to system driver.")); RemoveCallback(); NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); // Returning less than aFrames starts the draining and eventually stops the // audio thread. This function will never get called again. return aFrames - 1; } return aFrames; } void AudioCallbackDriver::StateCallback(cubeb_state aState) { STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver State: %d", aState)); if (aState == CUBEB_STATE_ERROR && mShouldFallbackIfError) { mShouldFallbackIfError = false; MonitorAutoLock lock(GraphImpl()->GetMonitor()); // Fall back to a driver using a normal thread. If needed, // the graph will try to re-open an audio stream later. SystemClockDriver* nextDriver = new SystemClockDriver(GraphImpl()); SetNextDriver(nextDriver); RemoveCallback(); nextDriver->MarkAsFallback(); nextDriver->SetGraphTime(this, mIterationStart, mIterationEnd); // We're not using SwitchAtNextIteration here, because there // won't be a next iteration if we don't restart things manually: // the audio stream just signaled that it's in error state. mGraphImpl->SetCurrentDriver(nextDriver); nextDriver->Start(); } } void AudioCallbackDriver::MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat, uint32_t aChannels, uint32_t aFrames, uint32_t aSampleRate) { uint32_t toWrite = mBuffer.Available(); if (!mBuffer.Available()) { NS_WARNING("DataCallback buffer full, expect frame drops."); } MOZ_ASSERT(mBuffer.Available() <= aFrames); mBuffer.WriteFrames(aMixedBuffer, mBuffer.Available()); MOZ_ASSERT(mBuffer.Available() == 0, "Missing frames to fill audio callback's buffer."); DebugOnly written = mScratchBuffer.Fill(aMixedBuffer + toWrite * aChannels, aFrames - toWrite); NS_WARNING_ASSERTION(written == aFrames - toWrite, "Dropping frames."); }; void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive) { #ifdef XP_MACOSX cubeb_device* out; int rv; char name[128]; size_t length = sizeof(name); rv = sysctlbyname("hw.model", name, &length, NULL, 0); if (rv) { return; } if (!strncmp(name, "MacBookPro", 10)) { if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) { // Check if we are currently outputing sound on external speakers. if (!strcmp(out->output_name, "ispk")) { // Pan everything to the right speaker. if (aMicrophoneActive) { if (cubeb_stream_set_panning(mAudioStream, 1.0) != CUBEB_OK) { NS_WARNING("Could not pan audio output to the right."); } } else { if (cubeb_stream_set_panning(mAudioStream, 0.0) != CUBEB_OK) { NS_WARNING("Could not pan audio output to the center."); } } } else { if (cubeb_stream_set_panning(mAudioStream, 0.0) != CUBEB_OK) { NS_WARNING("Could not pan audio output to the center."); } } cubeb_stream_device_destroy(mAudioStream, out); } } #endif } void AudioCallbackDriver::DeviceChangedCallback() { // Tell the audio engine the device has changed, it might want to reset some // state. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (mAudioInput) { mAudioInput->DeviceChanged(); } #ifdef XP_MACOSX PanOutputIfNeeded(mMicrophoneActive); #endif } void AudioCallbackDriver::SetMicrophoneActive(bool aActive) { MonitorAutoLock mon(mGraphImpl->GetMonitor()); mMicrophoneActive = aActive; #ifdef XP_MACOSX PanOutputIfNeeded(mMicrophoneActive); #endif } uint32_t AudioCallbackDriver::IterationDuration() { // The real fix would be to have an API in cubeb to give us the number. Short // of that, we approximate it here. bug 1019507 return mIterationDurationMS; } bool AudioCallbackDriver::IsStarted() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); return mStarted; } void AudioCallbackDriver::EnqueueStreamAndPromiseForOperation(MediaStream* aStream, void* aPromise, dom::AudioContextOperation aOperation) { MonitorAutoLock mon(mGraphImpl->GetMonitor()); mPromisesForOperation.AppendElement(StreamAndPromiseForOperation(aStream, aPromise, aOperation)); } void AudioCallbackDriver::CompleteAudioContextOperations(AsyncCubebOperation aOperation) { AutoTArray array; // We can't lock for the whole function because AudioContextOperationCompleted // will grab the monitor { MonitorAutoLock mon(GraphImpl()->GetMonitor()); array.SwapElements(mPromisesForOperation); } for (uint32_t i = 0; i < array.Length(); i++) { StreamAndPromiseForOperation& s = array[i]; if ((aOperation == AsyncCubebOperation::INIT && s.mOperation == dom::AudioContextOperation::Resume) || (aOperation == AsyncCubebOperation::SHUTDOWN && s.mOperation != dom::AudioContextOperation::Resume)) { GraphImpl()->AudioContextOperationCompleted(s.mStream, s.mPromise, s.mOperation); array.RemoveElementAt(i); i--; } } if (!array.IsEmpty()) { MonitorAutoLock mon(GraphImpl()->GetMonitor()); mPromisesForOperation.AppendElements(array); } } } // namespace mozilla