jp9000
/
obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

libobs: Implement audio monitoring 

Adds functions to turn on audio monitoring to allow the user to hear
playback of an audio source over the user's speaker.  It can be set to
turn off monitoring and only output to stream, or it can be set to
output only to monitoring, or it can be set to both.

On windows, audio monitoring uses WASAPI.  Windows also is capable of
syncing the audio to the video according to when the video frame itself
was played.

On mac, it uses AudioQueue.

On linux, it's not currently implemented and won't do anything (to be
implemented).
master
jp9000 2017-02-05 21:37:35 -08:00
parent 74f9c389cb
commit d2934eca7e
13 changed files with 1170 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
libobs/CMakeLists.txt
View File

@ -68,6 +68,13 @@ if(WIN32)
util/windows/CoTaskMemPtr.hpp
util/windows/HRError.hpp
util/windows/WinHandle.hpp)
set(libobs_audio_monitoring_SOURCES
audio-monitoring/win32/wasapi-output.c
audio-monitoring/win32/wasapi-enum-devices.c
)
set(libobs_audio_monitoring_HEADERS
audio-monitoring/win32/wasapi-output.h
)
set(libobs_PLATFORM_DEPS winmm)
if(MSVC)
set(libobs_PLATFORM_DEPS
@ -83,6 +90,13 @@ elseif(APPLE)
util/platform-cocoa.m)
set(libobs_PLATFORM_HEADERS
util/threading-posix.h)
set(libobs_audio_monitoring_SOURCES
audio-monitoring/osx/coreaudio-enum-devices.c
audio-monitoring/osx/coreaudio-output.c
)
set(libobs_audio_monitoring_HEADERS
audio-monitoring/osx/mac-helpers.h
)
set_source_files_properties(${libobs_PLATFORM_SOURCES}
PROPERTIES
@ -93,6 +107,18 @@ elseif(APPLE)
mark_as_advanced(COCOA)
include_directories(${COCOA})
find_library(COREAUDIO CoreAudio)
mark_as_advanced(COREAUDIO)
include_directories(${COREAUDIO})
find_library(AUDIOTOOLBOX AudioToolbox)
mark_as_advanced(AUDIOTOOLBOX)
include_directories(${AUDIOTOOLBOX})
find_library(AUDIOUNIT AudioUnit)
mark_as_advanced(AUDIOUNIT)
include_directories(${AUDIOUNIT})
find_library(APPKIT AppKit)
mark_as_advanced(APPKIT)
include_directories(${APPKIT})
@ -107,6 +133,9 @@ elseif(APPLE)
set(libobs_PLATFORM_DEPS
${COCOA}
${COREAUDIO}
${AUDIOUNIT}
${AUDIOTOOLBOX}
${APPKIT}
${IOKIT}
${CARBON})
@ -118,6 +147,9 @@ elseif(UNIX)
util/platform-nix.c)
set(libobs_PLATFORM_HEADERS
util/threading-posix.h)
set(libobs_audio_monitoring_SOURCES
audio-monitoring/null/null-audio-monitoring.c
)
if(DBUS_FOUND)
set(libobs_PLATFORM_SOURCES ${libobs_PLATFORM_SOURCES}
@ -334,7 +366,10 @@ set(libobs_HEADERS
${libobs_graphics_HEADERS}
${libobs_mediaio_HEADERS}
${libobs_util_HEADERS}
${libobs_libobs_HEADERS})
${libobs_libobs_HEADERS}
${libobs_audio_monitoring_SOURCES}
${libobs_audio_monitoring_HEADERS}
)
source_group("callback\\Source Files" FILES ${libobs_callback_SOURCES})
source_group("callback\\Header Files" FILES ${libobs_callback_HEADERS})
@ -346,6 +381,8 @@ source_group("media-io\\Source Files" FILES ${libobs_mediaio_SOURCES})
source_group("media-io\\Header Files" FILES ${libobs_mediaio_HEADERS})
source_group("util\\Source Files" FILES ${libobs_util_SOURCES})
source_group("util\\Header Files" FILES ${libobs_util_HEADERS})
source_group("audio-monitoring\\Source Files" FILES ${libobs_audio_monitoring_SOURCES})
source_group("audio-monitoring\\Header Files" FILES ${libobs_audio_monitoring_HEADERS})
if(BUILD_CAPTIONS)
include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption)

23
libobs/audio-monitoring/null/null-audio-monitoring.c Normal file
View File

@ -0,0 +1,23 @@
#include "../../obs-internal.h"
void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb, void *data)
{
UNUSED_PARAMETER(cb);
UNUSED_PARAMETER(data);
}
struct audio_monitor *audio_monitor_create(obs_source_t *source)
{
UNUSED_PARAMETER(source);
return NULL;
}
void audio_monitor_reset(struct audio_monitor *monitor)
{
UNUSED_PARAMETER(monitor);
}
void audio_monitor_destroy(struct audio_monitor *monitor)
{
UNUSED_PARAMETER(monitor);
}

96
libobs/audio-monitoring/osx/coreaudio-enum-devices.c Normal file
View File

@ -0,0 +1,96 @@
#include <CoreFoundation/CFString.h>
#include <CoreAudio/CoreAudio.h>
#include "../../obs-internal.h"
#include "../../util/dstr.h"
#include "mac-helpers.h"
static inline bool cf_to_cstr(CFStringRef ref, char *buf, size_t size)
{
if (!ref) return false;
return (bool)CFStringGetCString(ref, buf, size, kCFStringEncodingUTF8);
}
static void obs_enum_audio_monitoring_device(obs_enum_audio_device_cb cb,
void *data, AudioDeviceID id)
{
UInt32 size = 0;
CFStringRef cf_name = NULL;
CFStringRef cf_uid = NULL;
char name[1024];
char uid[1024];
OSStatus stat;
AudioObjectPropertyAddress addr = {
kAudioDevicePropertyStreams,
kAudioDevicePropertyScopeInput,
kAudioObjectPropertyElementMaster
};
/* check to see if it's a mac input device */
AudioObjectGetPropertyDataSize(id, &addr, 0, NULL, &size);
if (!size)
return;
size = sizeof(CFStringRef);
addr.mSelector = kAudioDevicePropertyDeviceUID;
stat = AudioObjectGetPropertyData(id, &addr, 0, NULL, &size, &cf_uid);
if (!success(stat, "get audio device UID"))
return;
addr.mSelector = kAudioDevicePropertyDeviceNameCFString;
stat = AudioObjectGetPropertyData(id, &addr, 0, NULL, &size, &cf_name);
if (!success(stat, "get audio device name"))
goto fail;
if (!cf_to_cstr(cf_name, name, sizeof(name))) {
blog(LOG_WARNING, "%s: failed to convert name", __FUNCTION__);
goto fail;
}
if (!cf_to_cstr(cf_uid, uid, sizeof(uid))) {
blog(LOG_WARNING, "%s: failed to convert uid", __FUNCTION__);
goto fail;
}
cb(data, name, uid);
fail:
if (cf_name)
CFRelease(cf_name);
if (cf_uid)
CFRelease(cf_uid);
}
void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb, void *data)
{
AudioObjectPropertyAddress addr = {
kAudioHardwarePropertyDevices,
kAudioObjectPropertyScopeGlobal,
kAudioObjectPropertyElementMaster
};
UInt32 size = 0;
UInt32 count;
OSStatus stat;
AudioDeviceID *ids;
stat = AudioObjectGetPropertyDataSize(kAudioObjectSystemObject, &addr,
0, NULL, &size);
if (!success(stat, "get data size"))
return;
ids = malloc(size);
count = size / sizeof(AudioDeviceID);
stat = AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr,
0, NULL, &size, ids);
if (success(stat, "get data")) {
for (UInt32 i = 0; i < count; i++)
obs_enum_audio_monitoring_device(cb, data, ids[i]);
}
free(ids);
}

322
libobs/audio-monitoring/osx/coreaudio-output.c Normal file
View File

@ -0,0 +1,322 @@
#include <AudioUnit/AudioUnit.h>
#include <AudioToolBox/AudioQueue.h>
#include <CoreFoundation/CFString.h>
#include <CoreAudio/CoreAudio.h>
#include "../../media-io/audio-resampler.h"
#include "../../util/circlebuf.h"
#include "../../util/threading.h"
#include "../../util/platform.h"
#include "../../obs-internal.h"
#include "../../util/darray.h"
#include "mac-helpers.h"
struct audio_monitor {
obs_source_t *source;
AudioQueueRef queue;
AudioQueueBufferRef buffers[3];
pthread_mutex_t mutex;
struct circlebuf empty_buffers;
struct circlebuf new_data;
audio_resampler_t *resampler;
size_t buffer_size;
size_t wait_size;
uint32_t channels;
volatile bool active;
bool paused;
};
static inline bool fill_buffer(struct audio_monitor *monitor)
{
AudioQueueBufferRef buf;
OSStatus stat;
if (monitor->new_data.size < monitor->buffer_size) {
return false;
}
circlebuf_pop_front(&monitor->empty_buffers, &buf, sizeof(buf));
circlebuf_pop_front(&monitor->new_data, buf->mAudioData,
monitor->buffer_size);
buf->mAudioDataByteSize = monitor->buffer_size;
stat = AudioQueueEnqueueBuffer(monitor->queue, buf, 0, NULL);
if (!success(stat, "AudioQueueEnqueueBuffer")) {
blog(LOG_WARNING, "%s: %s", __FUNCTION__,
"Failed to enqueue buffer");
AudioQueueStop(monitor->queue, false);
}
return true;
}
static void on_audio_playback(void *param, obs_source_t *source,
const struct audio_data *audio_data, bool muted)
{
struct audio_monitor *monitor = param;
float vol = source->user_volume;
uint32_t bytes;
UNUSED_PARAMETER(source);
if (!os_atomic_load_bool(&monitor->active)) {
return;
}
uint8_t *resample_data[MAX_AV_PLANES];
uint32_t resample_frames;
uint64_t ts_offset;
bool success;
success = audio_resampler_resample(monitor->resampler, resample_data,
&resample_frames, &ts_offset,
(const uint8_t *const *)audio_data->data,
(uint32_t)audio_data->frames);
if (!success) {
return;
}
bytes = sizeof(float) * monitor->channels * resample_frames;
if (muted) {
memset(resample_data[0], 0, bytes);
} else {
/* apply volume */
if (!close_float(vol, 1.0f, EPSILON)) {
register float *cur = (float*)resample_data[0];
register float *end = cur +
resample_frames * monitor->channels;
while (cur < end)
*(cur++) *= vol;
}
}
pthread_mutex_lock(&monitor->mutex);
circlebuf_push_back(&monitor->new_data, resample_data[0], bytes);
if (monitor->new_data.size >= monitor->wait_size) {
monitor->wait_size = 0;
while (monitor->empty_buffers.size > 0) {
if (!fill_buffer(monitor)) {
break;
}
}
if (monitor->paused) {
AudioQueueStart(monitor->queue, NULL);
monitor->paused = false;
}
}
pthread_mutex_unlock(&monitor->mutex);
}
static void buffer_audio(void *data, AudioQueueRef aq, AudioQueueBufferRef buf)
{
struct audio_monitor *monitor = data;
pthread_mutex_lock(&monitor->mutex);
circlebuf_push_back(&monitor->empty_buffers, &buf, sizeof(buf));
while (monitor->empty_buffers.size > 0) {
if (!fill_buffer(monitor)) {
break;
}
}
if (monitor->empty_buffers.size == sizeof(buf) * 3) {
monitor->paused = true;
monitor->wait_size = monitor->buffer_size * 3;
AudioQueuePause(monitor->queue);
}
pthread_mutex_unlock(&monitor->mutex);
UNUSED_PARAMETER(aq);
}
static bool audio_monitor_init(struct audio_monitor *monitor)
{
const struct audio_output_info *info = audio_output_get_info(
obs->audio.audio);
uint32_t channels = get_audio_channels(info->speakers);
OSStatus stat;
AudioStreamBasicDescription desc = {
.mSampleRate = (Float64)info->samples_per_sec,
.mFormatID = kAudioFormatLinearPCM,
.mFormatFlags = kAudioFormatFlagIsFloat |
kAudioFormatFlagIsPacked,
.mBytesPerPacket = sizeof(float) * channels,
.mFramesPerPacket = 1,
.mBytesPerFrame = sizeof(float) * channels,
.mChannelsPerFrame = channels,
.mBitsPerChannel = sizeof(float) * 8
};
monitor->channels = channels;
monitor->buffer_size =
channels * sizeof(float) * info->samples_per_sec / 100 * 3;
monitor->wait_size = monitor->buffer_size * 3;
pthread_mutex_init_value(&monitor->mutex);
stat = AudioQueueNewOutput(&desc, buffer_audio, monitor, NULL, NULL, 0,
&monitor->queue);
if (!success(stat, "AudioStreamBasicDescription")) {
return false;
}
const char *uid = obs->audio.monitoring_device_id;
if (!uid || !*uid) {
return false;
}
if (strcmp(uid, "default") != 0) {
CFStringRef cf_uid = CFStringCreateWithBytesNoCopy(NULL,
(const UInt8*)uid, strlen(uid),
kCFStringEncodingUTF8,
false, NULL);
stat = AudioQueueSetProperty(monitor->queue,
kAudioQueueProperty_CurrentDevice,
cf_uid, sizeof(cf_uid));
CFRelease(cf_uid);
if (!success(stat, "set current device")) {
return false;
}
}
stat = AudioQueueSetParameter(monitor->queue,
kAudioQueueParam_Volume, 1.0);
if (!success(stat, "set volume")) {
return false;
}
for (size_t i = 0; i < 3; i++) {
stat = AudioQueueAllocateBuffer(monitor->queue,
monitor->buffer_size, &monitor->buffers[i]);
if (!success(stat, "allocation of buffer")) {
return false;
}
circlebuf_push_back(&monitor->empty_buffers,
&monitor->buffers[i],
sizeof(monitor->buffers[i]));
}
if (pthread_mutex_init(&monitor->mutex, NULL) != 0) {
blog(LOG_WARNING, "%s: %s", __FUNCTION__,
"Failed to init mutex");
return false;
}
struct resample_info from = {
.samples_per_sec = info->samples_per_sec,
.speakers = info->speakers,
.format = AUDIO_FORMAT_FLOAT_PLANAR
};
struct resample_info to = {
.samples_per_sec = info->samples_per_sec,
.speakers = info->speakers,
.format = AUDIO_FORMAT_FLOAT
};
monitor->resampler = audio_resampler_create(&to, &from);
if (!monitor->resampler) {
blog(LOG_WARNING, "%s: %s", __FUNCTION__,
"Failed to create resampler");
return false;
}
stat = AudioQueueStart(monitor->queue, NULL);
if (!success(stat, "start")) {
return false;
}
monitor->active = true;
return true;
}
static void audio_monitor_free(struct audio_monitor *monitor)
{
if (monitor->source) {
obs_source_remove_audio_capture_callback(
monitor->source, on_audio_playback, monitor);
}
if (monitor->active) {
AudioQueueStop(monitor->queue, true);
}
for (size_t i = 0; i < 3; i++) {
if (monitor->buffers[i]) {
AudioQueueFreeBuffer(monitor->queue,
monitor->buffers[i]);
}
}
if (monitor->queue) {
AudioQueueDispose(monitor->queue, true);
}
audio_resampler_destroy(monitor->resampler);
circlebuf_free(&monitor->empty_buffers);
circlebuf_free(&monitor->new_data);
pthread_mutex_destroy(&monitor->mutex);
}
static void audio_monitor_init_final(struct audio_monitor *monitor,
obs_source_t *source)
{
monitor->source = source;
obs_source_add_audio_capture_callback(source, on_audio_playback,
monitor);
}
struct audio_monitor *audio_monitor_create(obs_source_t *source)
{
struct audio_monitor *monitor = bzalloc(sizeof(*monitor));
if (!audio_monitor_init(monitor)) {
goto fail;
}
pthread_mutex_lock(&obs->audio.monitoring_mutex);
da_push_back(obs->audio.monitors, &monitor);
pthread_mutex_unlock(&obs->audio.monitoring_mutex);
audio_monitor_init_final(monitor, source);
return monitor;
fail:
audio_monitor_free(monitor);
bfree(monitor);
return NULL;
}
void audio_monitor_reset(struct audio_monitor *monitor)
{
bool success;
obs_source_t *source = monitor->source;
audio_monitor_free(monitor);
memset(monitor, 0, sizeof(*monitor));
success = audio_monitor_init(monitor);
if (success)
audio_monitor_init_final(monitor, source);
}
void audio_monitor_destroy(struct audio_monitor *monitor)
{
if (monitor) {
audio_monitor_free(monitor);
pthread_mutex_lock(&obs->audio.monitoring_mutex);
da_erase_item(obs->audio.monitors, &monitor);
pthread_mutex_unlock(&obs->audio.monitoring_mutex);
bfree(monitor);
}
}

15
libobs/audio-monitoring/osx/mac-helpers.h Normal file
View File

@ -0,0 +1,15 @@
#pragma once
static bool success_(OSStatus stat, const char *func, const char *call)
{
if (stat != noErr) {
blog(LOG_WARNING, "%s: %s failed: %d",
func, call, (int)stat);
return false;
}
return true;
}
#define success(stat, call) \
success_(stat, __FUNCTION__, call)

105
libobs/audio-monitoring/win32/wasapi-enum-devices.c Normal file
View File

@ -0,0 +1,105 @@
#include "../../obs-internal.h"
#include "wasapi-output.h"
#include <propsys.h>
#ifdef __MINGW32__
#ifdef DEFINE_PROPERTYKEY
#undef DEFINE_PROPERTYKEY
#endif
#define DEFINE_PROPERTYKEY(id, a, b, c, d, e, f, g, h, i, j, k, l) \
const PROPERTYKEY id = { { a,b,c, { d,e,f,g,h,i,j,k, } }, l };
DEFINE_PROPERTYKEY(PKEY_Device_FriendlyName, \
0xa45c254e, 0xdf1c, 0x4efd, 0x80, \
0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0, 14);
#else
#include <functiondiscoverykeys_devpkey.h>
#endif
static bool get_device_info(obs_enum_audio_device_cb cb, void *data,
IMMDeviceCollection *collection, UINT idx)
{
IPropertyStore *store = NULL;
IMMDevice *device = NULL;
PROPVARIANT name_var;
char utf8_name[512];
WCHAR *w_id = NULL;
char utf8_id[512];
bool cont = true;
HRESULT hr;
hr = collection->lpVtbl->Item(collection, idx, &device);
if (FAILED(hr)) {
goto fail;
}
hr = device->lpVtbl->GetId(device, &w_id);
if (FAILED(hr)) {
goto fail;
}
hr = device->lpVtbl->OpenPropertyStore(device, STGM_READ, &store);
if (FAILED(hr)) {
goto fail;
}
PropVariantInit(&name_var);
hr = store->lpVtbl->GetValue(store, &PKEY_Device_FriendlyName,
&name_var);
if (FAILED(hr)) {
goto fail;
}
os_wcs_to_utf8(w_id, 0, utf8_id, 512);
os_wcs_to_utf8(name_var.pwszVal, 0, utf8_name, 512);
cont = cb(data, utf8_name, utf8_id);
fail:
safe_release(store);
safe_release(device);
if (w_id)
CoTaskMemFree(w_id);
return cont;
}
void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb,
void *data)
{
IMMDeviceEnumerator *enumerator = NULL;
IMMDeviceCollection *collection = NULL;
UINT count;
HRESULT hr;
hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL,
&IID_IMMDeviceEnumerator, &enumerator);
if (FAILED(hr)) {
goto fail;
}
hr = enumerator->lpVtbl->EnumAudioEndpoints(enumerator, eRender,
DEVICE_STATE_ACTIVE, &collection);
if (FAILED(hr)) {
goto fail;
}
hr = collection->lpVtbl->GetCount(collection, &count);
if (FAILED(hr)) {
goto fail;
}
for (UINT i = 0; i < count; i++) {
if (!get_device_info(cb, data, collection, i)) {
break;
}
}
fail:
safe_release(enumerator);
safe_release(collection);
}

402
libobs/audio-monitoring/win32/wasapi-output.c Normal file
View File

@ -0,0 +1,402 @@
#include "../../media-io/audio-resampler.h"
#include "../../util/circlebuf.h"
#include "../../util/platform.h"
#include "../../util/darray.h"
#include "../../obs-internal.h"
#include "wasapi-output.h"
#define ACTUALLY_DEFINE_GUID(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8) \
EXTERN_C const GUID DECLSPEC_SELECTANY name \
= { l, w1, w2, { b1, b2, b3, b4, b5, b6, b7, b8 } }
ACTUALLY_DEFINE_GUID(CLSID_MMDeviceEnumerator,
0xBCDE0395, 0xE52F, 0x467C,
0x8E, 0x3D, 0xC4, 0x57, 0x92, 0x91, 0x69, 0x2E);
ACTUALLY_DEFINE_GUID(IID_IMMDeviceEnumerator,
0xA95664D2, 0x9614, 0x4F35,
0xA7, 0x46, 0xDE, 0x8D, 0xB6, 0x36, 0x17, 0xE6);
ACTUALLY_DEFINE_GUID(IID_IAudioClient,
0x1CB9AD4C, 0xDBFA, 0x4C32,
0xB1, 0x78, 0xC2, 0xF5, 0x68, 0xA7, 0x03, 0xB2);
ACTUALLY_DEFINE_GUID(IID_IAudioRenderClient,
0xF294ACFC, 0x3146, 0x4483,
0xA7, 0xBF, 0xAD, 0xDC, 0xA7, 0xC2, 0x60, 0xE2);
struct audio_monitor {
obs_source_t *source;
IMMDevice *device;
IAudioClient *client;
IAudioRenderClient *render;
uint64_t last_recv_time;
audio_resampler_t *resampler;
uint32_t sample_rate;
uint32_t channels;
bool source_has_video : 1;
int64_t lowest_audio_offset;
struct circlebuf delay_buffer;
uint32_t delay_size;
DARRAY(float) buf;
pthread_mutex_t playback_mutex;
};
/* #define DEBUG_AUDIO */
static bool process_audio_delay(struct audio_monitor *monitor,
float **data, uint32_t *frames, uint64_t ts, uint32_t pad)
{
obs_source_t *s = monitor->source;
uint64_t last_frame_ts = s->last_frame_ts;
uint64_t cur_time = os_gettime_ns();
uint64_t front_ts;
uint64_t cur_ts;
int64_t diff;
uint32_t blocksize = monitor->channels * sizeof(float);
/* cut off audio if long-since leftover audio in delay buffer */
if (cur_time - monitor->last_recv_time > 1000000000)
circlebuf_free(&monitor->delay_buffer);
monitor->last_recv_time = cur_time;
circlebuf_push_back(&monitor->delay_buffer, &ts, sizeof(ts));
circlebuf_push_back(&monitor->delay_buffer, frames, sizeof(*frames));
circlebuf_push_back(&monitor->delay_buffer, *data,
*frames * blocksize);
while (monitor->delay_buffer.size != 0) {
size_t size;
bool bad_diff;
circlebuf_peek_front(&monitor->delay_buffer, &cur_ts,
sizeof(ts));
front_ts = cur_ts -
((uint64_t)pad * 1000000000ULL /
(uint64_t)monitor->sample_rate);
diff = (int64_t)front_ts - (int64_t)last_frame_ts;
bad_diff = llabs(diff) > 5000000000;
/* delay audio if rushing */
if (!bad_diff && diff > 75000000) {
#ifdef DEBUG_AUDIO
blog(LOG_INFO, "audio rushing, cutting audio, "
"diff: %lld, delay buffer size: %lu, "
"v: %llu: a: %llu",
diff, (int)monitor->delay_buffer.size,
last_frame_ts, front_ts);
#endif
return false;
}
circlebuf_pop_front(&monitor->delay_buffer, NULL, sizeof(ts));
circlebuf_pop_front(&monitor->delay_buffer, frames,
sizeof(*frames));
size = *frames * blocksize;
da_resize(monitor->buf, size);
circlebuf_pop_front(&monitor->delay_buffer,
monitor->buf.array, size);
/* cut audio if dragging */
if (!bad_diff && diff < -75000000 && monitor->delay_buffer.size > 0) {
#ifdef DEBUG_AUDIO
blog(LOG_INFO, "audio dragging, cutting audio, "
"diff: %lld, delay buffer size: %lu, "
"v: %llu: a: %llu",
diff, (int)monitor->delay_buffer.size,
last_frame_ts, front_ts);
#endif
continue;
}
*data = monitor->buf.array;
return true;
}
return false;
}
static void on_audio_playback(void *param, obs_source_t *source,
const struct audio_data *audio_data, bool muted)
{
struct audio_monitor *monitor = param;
IAudioRenderClient *render = monitor->render;
uint8_t *resample_data[MAX_AV_PLANES];
float vol = source->user_volume;
uint32_t resample_frames;
uint64_t ts_offset;
bool success;
BYTE *output;
if (pthread_mutex_trylock(&monitor->playback_mutex) != 0) {
return;
}
if (os_atomic_load_long(&source->activate_refs) == 0) {
goto unlock;
}
success = audio_resampler_resample(monitor->resampler, resample_data,
&resample_frames, &ts_offset,
(const uint8_t *const *)audio_data->data,
(uint32_t)audio_data->frames);
if (!success) {
goto unlock;
}
UINT32 pad = 0;
monitor->client->lpVtbl->GetCurrentPadding(monitor->client, &pad);
if (monitor->source_has_video) {
uint64_t ts = audio_data->timestamp - ts_offset;
if (!process_audio_delay(monitor, (float**)(&resample_data[0]),
&resample_frames, ts, pad)) {
goto unlock;
}
}
HRESULT hr = render->lpVtbl->GetBuffer(render, resample_frames,
&output);
if (FAILED(hr)) {
goto unlock;
}
if (!muted) {
/* apply volume */
if (!close_float(vol, 1.0f, EPSILON)) {
register float *cur = (float*)resample_data[0];
register float *end = cur +
resample_frames * monitor->channels;
while (cur < end)
*(cur++) *= vol;
}
memcpy(output, resample_data[0],
resample_frames * monitor->channels *
sizeof(float));
}
render->lpVtbl->ReleaseBuffer(render, resample_frames,
muted ? AUDCLNT_BUFFERFLAGS_SILENT : 0);
unlock:
pthread_mutex_unlock(&monitor->playback_mutex);
}
static inline void audio_monitor_free(struct audio_monitor *monitor)
{
if (monitor->source) {
obs_source_remove_audio_capture_callback(
monitor->source, on_audio_playback, monitor);
}
if (monitor->client)
monitor->client->lpVtbl->Stop(monitor->client);
safe_release(monitor->device);
safe_release(monitor->client);
safe_release(monitor->render);
audio_resampler_destroy(monitor->resampler);
circlebuf_free(&monitor->delay_buffer);
da_free(monitor->buf);
}
static enum speaker_layout convert_speaker_layout(DWORD layout, WORD channels)
{
switch (layout) {
case KSAUDIO_SPEAKER_QUAD: return SPEAKERS_QUAD;
case KSAUDIO_SPEAKER_2POINT1: return SPEAKERS_2POINT1;
case KSAUDIO_SPEAKER_4POINT1: return SPEAKERS_4POINT1;
case KSAUDIO_SPEAKER_SURROUND: return SPEAKERS_SURROUND;
case KSAUDIO_SPEAKER_5POINT1: return SPEAKERS_5POINT1;
case KSAUDIO_SPEAKER_5POINT1_SURROUND: return SPEAKERS_5POINT1_SURROUND;
case KSAUDIO_SPEAKER_7POINT1: return SPEAKERS_7POINT1;
case KSAUDIO_SPEAKER_7POINT1_SURROUND: return SPEAKERS_7POINT1_SURROUND;
}
return (enum speaker_layout)channels;
}
static bool audio_monitor_init(struct audio_monitor *monitor)
{
IMMDeviceEnumerator *immde = NULL;
WAVEFORMATEX *wfex = NULL;
bool success = false;
UINT32 frames;
HRESULT hr;
const char *id = obs->audio.monitoring_device_id;
if (!id) {
return false;
}
pthread_mutex_init_value(&monitor->playback_mutex);
/* ------------------------------------------ *
* Init device */
hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL,
&IID_IMMDeviceEnumerator, (void**)&immde);
if (FAILED(hr)) {
return false;
}
if (strcmp(id, "default") == 0) {
hr = immde->lpVtbl->GetDefaultAudioEndpoint(immde,
eRender, eConsole, &monitor->device);
} else {
wchar_t w_id[512];
os_utf8_to_wcs(id, 0, w_id, 512);
hr = immde->lpVtbl->GetDevice(immde, w_id, &monitor->device);
}
if (FAILED(hr)) {
goto fail;
}
/* ------------------------------------------ *
* Init client */
hr = monitor->device->lpVtbl->Activate(monitor->device,
&IID_IAudioClient, CLSCTX_ALL, NULL,
(void**)&monitor->client);
if (FAILED(hr)) {
goto fail;
}
hr = monitor->client->lpVtbl->GetMixFormat(monitor->client, &wfex);
if (FAILED(hr)) {
goto fail;
}
hr = monitor->client->lpVtbl->Initialize(monitor->client,
AUDCLNT_SHAREMODE_SHARED, 0,
10000000, 0, wfex, NULL);
if (FAILED(hr)) {
goto fail;
}
/* ------------------------------------------ *
* Init resampler */
const struct audio_output_info *info = audio_output_get_info(
obs->audio.audio);
WAVEFORMATEXTENSIBLE *ext = (WAVEFORMATEXTENSIBLE*)wfex;
struct resample_info from;
struct resample_info to;
from.samples_per_sec = info->samples_per_sec;
from.speakers = info->speakers;
from.format = AUDIO_FORMAT_FLOAT_PLANAR;
to.samples_per_sec = (uint32_t)wfex->nSamplesPerSec;
to.speakers = convert_speaker_layout(ext->dwChannelMask,
wfex->nChannels);
to.format = AUDIO_FORMAT_FLOAT;
monitor->sample_rate = (uint32_t)wfex->nSamplesPerSec;
monitor->channels = wfex->nChannels;
monitor->resampler = audio_resampler_create(&to, &from);
if (!monitor->resampler) {
goto fail;
}
/* ------------------------------------------ *
* Init client */
hr = monitor->client->lpVtbl->GetBufferSize(monitor->client, &frames);
if (FAILED(hr)) {
goto fail;
}
hr = monitor->client->lpVtbl->GetService(monitor->client,
&IID_IAudioRenderClient, (void**)&monitor->render);
if (FAILED(hr)) {
goto fail;
}
if (pthread_mutex_init(&monitor->playback_mutex, NULL) != 0) {
goto fail;
}
hr = monitor->client->lpVtbl->Start(monitor->client);
if (FAILED(hr)) {
goto fail;
}
success = true;
fail:
safe_release(immde);
if (wfex)
CoTaskMemFree(wfex);
return success;
}
static void audio_monitor_init_final(struct audio_monitor *monitor,
obs_source_t *source)
{
monitor->source = source;
monitor->source_has_video =
(source->info.output_flags & OBS_SOURCE_VIDEO) != 0;
obs_source_add_audio_capture_callback(source, on_audio_playback,
monitor);
}
struct audio_monitor *audio_monitor_create(obs_source_t *source)
{
struct audio_monitor monitor = {0};
struct audio_monitor *out;
if (!audio_monitor_init(&monitor)) {
goto fail;
}
out = bmemdup(&monitor, sizeof(monitor));
pthread_mutex_lock(&obs->audio.monitoring_mutex);
da_push_back(obs->audio.monitors, &out);
pthread_mutex_unlock(&obs->audio.monitoring_mutex);
audio_monitor_init_final(out, source);
return out;
fail:
audio_monitor_free(&monitor);
return NULL;
}
void audio_monitor_reset(struct audio_monitor *monitor)
{
struct audio_monitor new_monitor = {0};
bool success;
pthread_mutex_lock(&monitor->playback_mutex);
success = audio_monitor_init(&new_monitor);
pthread_mutex_unlock(&monitor->playback_mutex);
if (success) {
obs_source_t *source = monitor->source;
audio_monitor_free(monitor);
*monitor = new_monitor;
audio_monitor_init_final(monitor, source);
} else {
audio_monitor_free(&new_monitor);
}
}
void audio_monitor_destroy(struct audio_monitor *monitor)
{
if (monitor) {
audio_monitor_free(monitor);
pthread_mutex_lock(&obs->audio.monitoring_mutex);
da_erase_item(obs->audio.monitors, &monitor);
pthread_mutex_unlock(&obs->audio.monitoring_mutex);
bfree(monitor);
}
}

13
libobs/audio-monitoring/win32/wasapi-output.h Normal file
View File

@ -0,0 +1,13 @@
#include <windows.h>
#include <mmdeviceapi.h>
#include <audioclient.h>
#define KSAUDIO_SPEAKER_4POINT1 (KSAUDIO_SPEAKER_QUAD|SPEAKER_LOW_FREQUENCY)
#define KSAUDIO_SPEAKER_2POINT1 (KSAUDIO_SPEAKER_STEREO|SPEAKER_LOW_FREQUENCY)
#define safe_release(ptr) \
do { \
if (ptr) { \
ptr->lpVtbl->Release(ptr); \
} \
} while (false)

16
libobs/obs-internal.h
View File

@ -277,8 +277,9 @@ struct obs_core_video {
gs_effect_t *deinterlace_yadif_2x_effect;
};
struct audio_monitor;
struct obs_core_audio {
/* TODO: sound output subsystem */
audio_t *audio;
DARRAY(struct obs_source*) render_order;
@ -290,6 +291,11 @@ struct obs_core_audio {
int total_buffering_ticks;
float user_volume;
pthread_mutex_t monitoring_mutex;
DARRAY(struct audio_monitor*) monitors;
char *monitoring_device_name;
char *monitoring_device_id;
};
/* user sources, output channels, and displays */
@ -546,6 +552,7 @@ struct obs_source {
volatile bool timing_set;
volatile uint64_t timing_adjust;
uint64_t resample_offset;
uint64_t last_audio_ts;
uint64_t next_audio_ts_min;
uint64_t next_audio_sys_ts_min;
uint64_t last_frame_ts;
@ -661,6 +668,9 @@ struct obs_source {
enum obs_transition_mode transition_mode;
enum obs_transition_scale_type transition_scale_type;
struct matrix4 transition_matrices[2];
struct audio_monitor *monitor;
enum obs_monitoring_type monitoring_type;
};
extern const struct obs_source_info *get_source_info(const char *id);
@ -679,6 +689,10 @@ extern void obs_transition_enum_sources(obs_source_t *transition,
extern void obs_transition_save(obs_source_t *source, obs_data_t *data);
extern void obs_transition_load(obs_source_t *source, obs_data_t *data);
struct audio_monitor *audio_monitor_create(obs_source_t *source);
void audio_monitor_reset(struct audio_monitor *monitor);
extern void audio_monitor_destroy(struct audio_monitor *monitor);
extern void obs_source_destroy(struct obs_source *source);
enum view_type {

48
libobs/obs-source.c
View File

@ -503,6 +503,8 @@ void obs_source_destroy(struct obs_source *source)
source->context.data = NULL;
}
audio_monitor_destroy(source->monitor);
obs_hotkey_unregister(source->push_to_talk_key);
obs_hotkey_unregister(source->push_to_mute_key);
obs_hotkey_pair_unregister(source->mute_unmute_key);
@ -1184,6 +1186,7 @@ static void source_output_audio_data(obs_source_t *source,
in.timestamp = source->next_audio_ts_min;
}
source->last_audio_ts = in.timestamp;
source->next_audio_ts_min = in.timestamp +
conv_frames_to_time(sample_rate, in.frames);
@ -1226,10 +1229,12 @@ static void source_output_audio_data(obs_source_t *source,
source->last_sync_offset = sync_offset;
}
if (push_back && source->audio_ts)
source_output_audio_push_back(source, &in);
else
source_output_audio_place(source, &in);
if (source->monitoring_type != OBS_MONITORING_TYPE_MONITOR_ONLY) {
if (push_back && source->audio_ts)
source_output_audio_push_back(source, &in);
else
source_output_audio_place(source, &in);
}
pthread_mutex_unlock(&source->audio_buf_mutex);
@ -3891,3 +3896,38 @@ void obs_source_remove_audio_capture_callback(obs_source_t *source,
da_erase_item(source->audio_cb_list, &info);
pthread_mutex_unlock(&source->audio_cb_mutex);
}
void obs_source_set_monitoring_type(obs_source_t *source,
enum obs_monitoring_type type)
{
bool was_on;
bool now_on;
if (!obs_source_valid(source, "obs_source_set_monitoring_type"))
return;
if (source->info.output_flags & OBS_SOURCE_DO_NOT_MONITOR)
return;
if (source->monitoring_type == type)
return;
was_on = source->monitoring_type != OBS_MONITORING_TYPE_NONE;
now_on = type != OBS_MONITORING_TYPE_NONE;
if (was_on != now_on) {
if (!was_on) {
source->monitor = audio_monitor_create(source);
} else {
audio_monitor_destroy(source->monitor);
source->monitor = NULL;
}
}
source->monitoring_type = type;
}
enum obs_monitoring_type obs_source_get_monitoring_type(
const obs_source_t *source)
{
return obs_source_valid(source, "obs_source_get_monitoring_type") ?
source->monitoring_type : OBS_MONITORING_TYPE_NONE;
}

8
libobs/obs-source.h
View File

@ -120,6 +120,14 @@ enum obs_source_type {
*/
#define OBS_SOURCE_DEPRECATED (1<<8)
/**
* Source cannot have its audio monitored
*
* Specifies that this source may cause a feedback loop if audio is monitored.
* This is used primarily with desktop audio capture sources.
*/
#define OBS_SOURCE_DO_NOT_MONITOR (1<<9)
/** @} */
typedef void (*obs_source_enum_proc_t)(obs_source_t *parent,

70
libobs/obs.c
View File

@ -488,10 +488,22 @@ static bool obs_init_audio(struct audio_output_info *ai)
struct obs_core_audio *audio = &obs->audio;
int errorcode;
/* TODO: sound subsystem */
pthread_mutexattr_t attr;
pthread_mutex_init_value(&audio->monitoring_mutex);
if (pthread_mutexattr_init(&attr) != 0)
return false;
if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0)
return false;
if (pthread_mutex_init(&audio->monitoring_mutex, &attr) != 0)
return false;
audio->user_volume = 1.0f;
audio->monitoring_device_name = bstrdup("Default");
audio->monitoring_device_id = bstrdup("default");
errorcode = audio_output_open(&audio->audio, ai);
if (errorcode == AUDIO_OUTPUT_SUCCESS)
return true;
@ -513,6 +525,11 @@ static void obs_free_audio(void)
da_free(audio->render_order);
da_free(audio->root_nodes);
da_free(audio->monitors);
bfree(audio->monitoring_device_name);
bfree(audio->monitoring_device_id);
pthread_mutex_destroy(&audio->monitoring_mutex);
memset(audio, 0, sizeof(struct obs_core_audio));
}
@ -725,6 +742,8 @@ static bool obs_init(const char *locale, const char *module_config_path,
{
obs = bzalloc(sizeof(struct obs_core));
pthread_mutex_init_value(&obs->audio.monitoring_mutex);
obs->name_store_owned = !store;
obs->name_store = store ? store : profiler_name_store_create();
if (!obs->name_store) {
@ -1454,6 +1473,7 @@ static obs_source_t *obs_load_source_type(obs_data_t *source_data)
uint32_t mixers;
int di_order;
int di_mode;
int monitoring_type;
source = obs_source_create(id, name, settings, hotkeys);
@ -1505,6 +1525,10 @@ static obs_source_t *obs_load_source_type(obs_data_t *source_data)
obs_source_set_deinterlace_field_order(source,
(enum obs_deinterlace_field_order)di_order);
monitoring_type = (int)obs_data_get_int(source_data, "monitoring_type");
obs_source_set_monitoring_type(source,
(enum obs_monitoring_type)monitoring_type);
if (filters) {
size_t count = obs_data_array_count(filters);
@ -1601,6 +1625,7 @@ obs_data_t *obs_save_source(obs_source_t *source)
uint64_t ptm_delay = obs_source_get_push_to_mute_delay(source);
bool push_to_talk= obs_source_push_to_talk_enabled(source);
uint64_t ptt_delay = obs_source_get_push_to_talk_delay(source);
int m_type = (int)obs_source_get_monitoring_type(source);
int di_mode = (int)obs_source_get_deinterlace_mode(source);
int di_order =
(int)obs_source_get_deinterlace_field_order(source);
@ -1630,6 +1655,7 @@ obs_data_t *obs_save_source(obs_source_t *source)
obs_data_set_obj (source_data, "hotkeys", hotkey_data);
obs_data_set_int (source_data, "deinterlace_mode", di_mode);
obs_data_set_int (source_data, "deinterlace_field_order", di_order);
obs_data_set_int (source_data, "monitoring_type", m_type);
if (source->info.type == OBS_SOURCE_TYPE_TRANSITION)
obs_transition_save(source, source_data);
@ -1876,3 +1902,45 @@ bool obs_obj_invalid(void *obj)
return !context->data;
}
bool obs_set_audio_monitoring_device(const char *name, const char *id)
{
if (!obs || !name || !id || !*name || !*id)
return false;
#ifdef _WIN32
pthread_mutex_lock(&obs->audio.monitoring_mutex);
if (strcmp(id, obs->audio.monitoring_device_id) == 0)
return true;
if (obs->audio.monitoring_device_name)
bfree(obs->audio.monitoring_device_name);
if (obs->audio.monitoring_device_id)
bfree(obs->audio.monitoring_device_id);
obs->audio.monitoring_device_name = bstrdup(name);
obs->audio.monitoring_device_id = bstrdup(id);
for (size_t i = 0; i < obs->audio.monitors.num; i++) {
struct audio_monitor *monitor = obs->audio.monitors.array[i];
audio_monitor_reset(monitor);
}
pthread_mutex_unlock(&obs->audio.monitoring_mutex);
return true;
#else
return false;
#endif
}
void obs_get_audio_monitoring_device(const char **name, const char **id)
{
if (!obs)
return;
if (name)
*name = obs->audio.monitoring_device_name;
if (id)
*id = obs->audio.monitoring_device_id;
}

20
libobs/obs.h
View File

@ -582,6 +582,15 @@ EXPORT enum obs_obj_type obs_obj_get_type(void *obj);
EXPORT const char *obs_obj_get_id(void *obj);
EXPORT bool obs_obj_invalid(void *obj);
typedef bool (*obs_enum_audio_device_cb)(void *data, const char *name,
const char *id);
EXPORT void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb,
void *data);
EXPORT bool obs_set_audio_monitoring_device(const char *name, const char *id);
EXPORT void obs_get_audio_monitoring_device(const char **name, const char **id);
/* ------------------------------------------------------------------------- */
/* View context */
@ -914,6 +923,17 @@ EXPORT void obs_source_set_deinterlace_field_order(obs_source_t *source,
EXPORT enum obs_deinterlace_field_order obs_source_get_deinterlace_field_order(
const obs_source_t *source);
enum obs_monitoring_type {
OBS_MONITORING_TYPE_NONE,
OBS_MONITORING_TYPE_MONITOR_ONLY,
OBS_MONITORING_TYPE_MONITOR_AND_OUTPUT
};
EXPORT void obs_source_set_monitoring_type(obs_source_t *source,
enum obs_monitoring_type type);
EXPORT enum obs_monitoring_type obs_source_get_monitoring_type(
const obs_source_t *source);
/* ------------------------------------------------------------------------- */
/* Functions used by sources */