- Add planar audio support. FFmpeg and libav use planar audio for many encoders, so it was somewhat necessary to add support in libobs itself. - Improve/adjust FFmpeg test output plugin. The exports were somewhat messed up (making me rethink how exports should be done). Not yet functional; it handles video properly, but it still does not handle audio properly. - Improve planar video code. The planar video code was not properly accounting for row sizes for each plane. Specifying row sizes for each plane has now been added. This will also make it more compatible with FFmpeg/libav. - Fixed a bug where callbacks wouldn't create properly in audio-io and video-io code. - Implement 'blogva' function to allow for va_list usage with libobs logging.
567 lines
15 KiB
C
567 lines
15 KiB
C
/******************************************************************************
|
|
Copyright (C) 2013 by Hugh Bailey <obs.jim@gmail.com>
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
******************************************************************************/
|
|
|
|
#include "../util/threading.h"
|
|
#include "../util/darray.h"
|
|
#include "../util/circlebuf.h"
|
|
#include "../util/platform.h"
|
|
|
|
#include "audio-io.h"
|
|
|
|
struct audio_input {
|
|
struct audio_convert_info conversion;
|
|
void (*callback)(void *param, const struct audio_data *data);
|
|
void *param;
|
|
};
|
|
|
|
struct audio_line {
|
|
char *name;
|
|
|
|
struct audio_output *audio;
|
|
struct circlebuf buffers[MAX_AUDIO_PLANES];
|
|
pthread_mutex_t mutex;
|
|
DARRAY(uint8_t) volume_buffers[MAX_AUDIO_PLANES];
|
|
uint64_t base_timestamp;
|
|
uint64_t last_timestamp;
|
|
|
|
/* states whether this line is still being used. if not, then when the
|
|
* buffer is depleted, it's destroyed */
|
|
bool alive;
|
|
|
|
struct audio_line **prev_next;
|
|
struct audio_line *next;
|
|
};
|
|
|
|
static inline void audio_line_destroy_data(struct audio_line *line)
|
|
{
|
|
for (size_t i = 0; i < MAX_AUDIO_PLANES; i++) {
|
|
circlebuf_free(&line->buffers[i]);
|
|
da_free(line->volume_buffers[i]);
|
|
}
|
|
|
|
pthread_mutex_destroy(&line->mutex);
|
|
bfree(line->name);
|
|
bfree(line);
|
|
}
|
|
|
|
struct audio_output {
|
|
struct audio_output_info info;
|
|
size_t block_size;
|
|
size_t channels;
|
|
size_t planes;
|
|
|
|
pthread_t thread;
|
|
event_t stop_event;
|
|
|
|
DARRAY(uint8_t) mix_buffers[MAX_AUDIO_PLANES];
|
|
|
|
bool initialized;
|
|
|
|
pthread_mutex_t line_mutex;
|
|
struct audio_line *first_line;
|
|
|
|
pthread_mutex_t input_mutex;
|
|
DARRAY(struct audio_input) inputs;
|
|
};
|
|
|
|
static inline void audio_output_removeline(struct audio_output *audio,
|
|
struct audio_line *line)
|
|
{
|
|
pthread_mutex_lock(&audio->line_mutex);
|
|
*line->prev_next = line->next;
|
|
if (line->next)
|
|
line->next->prev_next = line->prev_next;
|
|
pthread_mutex_unlock(&audio->line_mutex);
|
|
|
|
audio_line_destroy_data(line);
|
|
}
|
|
|
|
static inline uint32_t time_to_frames(audio_t audio, uint64_t offset)
|
|
{
|
|
double audio_offset_d = (double)offset;
|
|
audio_offset_d /= 1000000000.0;
|
|
audio_offset_d *= (double)audio->info.samples_per_sec;
|
|
|
|
return (uint32_t)audio_offset_d;
|
|
}
|
|
|
|
static inline size_t time_to_bytes(audio_t audio, uint64_t offset)
|
|
{
|
|
return time_to_frames(audio, offset) * audio->block_size;
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------- */
|
|
|
|
static inline void clear_excess_audio_data(struct audio_line *line,
|
|
uint64_t size)
|
|
{
|
|
for (size_t i = 0; i < line->audio->planes; i++) {
|
|
size_t clear_size = (size > line->buffers[i].size) ?
|
|
(size_t)size : line->buffers[i].size;
|
|
|
|
circlebuf_pop_front(&line->buffers[i], NULL, clear_size);
|
|
}
|
|
|
|
blog(LOG_WARNING, "Excess audio data for audio line '%s', somehow "
|
|
"audio data went back in time by %llu bytes",
|
|
line->name, size);
|
|
}
|
|
|
|
static inline uint64_t min_uint64(uint64_t a, uint64_t b)
|
|
{
|
|
return a < b ? a : b;
|
|
}
|
|
|
|
static inline void mix_audio_line(struct audio_output *audio,
|
|
struct audio_line *line, size_t size, uint64_t timestamp)
|
|
{
|
|
/* TODO: this just overwrites. handle actual mixing */
|
|
if (!line->buffers[0].size) {
|
|
if (!line->alive)
|
|
audio_output_removeline(audio, line);
|
|
return;
|
|
}
|
|
|
|
size_t time_offset = time_to_bytes(audio,
|
|
line->base_timestamp - timestamp);
|
|
if (time_offset > size)
|
|
return;
|
|
|
|
size -= time_offset;
|
|
|
|
for (size_t i = 0; i < audio->planes; i++) {
|
|
size_t pop_size;
|
|
pop_size = (size_t)min_uint64(size, line->buffers[i].size);
|
|
|
|
circlebuf_pop_front(&line->buffers[i],
|
|
audio->mix_buffers[i].array + time_offset,
|
|
pop_size);
|
|
}
|
|
}
|
|
|
|
static inline void do_audio_output(struct audio_output *audio,
|
|
uint64_t timestamp, uint32_t frames)
|
|
{
|
|
struct audio_data data;
|
|
for (size_t i = 0; i < MAX_AUDIO_PLANES; i++)
|
|
data.data[i] = audio->mix_buffers[i].array;
|
|
data.frames = frames;
|
|
data.timestamp = timestamp;
|
|
data.volume = 1.0f;
|
|
|
|
/* TODO: conversion */
|
|
pthread_mutex_lock(&audio->input_mutex);
|
|
for (size_t i = 0; i < audio->inputs.num; i++) {
|
|
struct audio_input *input = audio->inputs.array+i;
|
|
input->callback(input->param, &data);
|
|
}
|
|
pthread_mutex_unlock(&audio->input_mutex);
|
|
}
|
|
|
|
static void mix_and_output(struct audio_output *audio, uint64_t audio_time,
|
|
uint64_t prev_time)
|
|
{
|
|
struct audio_line *line = audio->first_line;
|
|
uint64_t time_offset = audio_time - prev_time;
|
|
uint32_t frames = time_to_frames(audio, time_offset);
|
|
size_t bytes = frames * audio->block_size;
|
|
|
|
for (size_t i = 0; i < audio->planes; i++) {
|
|
da_resize(audio->mix_buffers[i], bytes);
|
|
memset(audio->mix_buffers[i].array, 0, bytes);
|
|
}
|
|
|
|
while (line) {
|
|
struct audio_line *next = line->next;
|
|
|
|
if (line->buffers[0].size && line->base_timestamp < prev_time) {
|
|
clear_excess_audio_data(line,
|
|
prev_time - line->base_timestamp);
|
|
line->base_timestamp = prev_time;
|
|
}
|
|
|
|
mix_audio_line(audio, line, bytes, prev_time);
|
|
line->base_timestamp = audio_time;
|
|
line = next;
|
|
}
|
|
|
|
do_audio_output(audio, prev_time, frames);
|
|
}
|
|
|
|
/* sample audio 40 times a second */
|
|
#define AUDIO_WAIT_TIME (1000/40)
|
|
|
|
static void *audio_thread(void *param)
|
|
{
|
|
struct audio_output *audio = param;
|
|
uint64_t buffer_time = audio->info.buffer_ms * 1000000;
|
|
uint64_t prev_time = os_gettime_ns() - buffer_time;
|
|
uint64_t audio_time;
|
|
|
|
while (event_try(&audio->stop_event) == EAGAIN) {
|
|
os_sleep_ms(AUDIO_WAIT_TIME);
|
|
|
|
pthread_mutex_lock(&audio->line_mutex);
|
|
|
|
audio_time = os_gettime_ns() - buffer_time;
|
|
mix_and_output(audio, audio_time, prev_time);
|
|
prev_time = audio_time;
|
|
|
|
pthread_mutex_unlock(&audio->line_mutex);
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------- */
|
|
|
|
static size_t audio_get_input_idx(audio_t video,
|
|
void (*callback)(void *param, const struct audio_data *data),
|
|
void *param)
|
|
{
|
|
for (size_t i = 0; i < video->inputs.num; i++) {
|
|
struct audio_input *input = video->inputs.array+i;
|
|
if (input->callback == callback && input->param == param)
|
|
return i;
|
|
}
|
|
|
|
return DARRAY_INVALID;
|
|
}
|
|
|
|
void audio_output_connect(audio_t audio,
|
|
struct audio_convert_info *conversion,
|
|
void (*callback)(void *param, const struct audio_data *data),
|
|
void *param)
|
|
{
|
|
pthread_mutex_lock(&audio->input_mutex);
|
|
|
|
if (audio_get_input_idx(audio, callback, param) == DARRAY_INVALID) {
|
|
struct audio_input input;
|
|
input.callback = callback;
|
|
input.param = param;
|
|
|
|
/* TODO: conversion */
|
|
if (conversion) {
|
|
input.conversion = *conversion;
|
|
} else {
|
|
input.conversion.format = audio->info.format;
|
|
input.conversion.speakers = audio->info.speakers;
|
|
input.conversion.samples_per_sec =
|
|
audio->info.samples_per_sec;
|
|
}
|
|
|
|
da_push_back(audio->inputs, &input);
|
|
}
|
|
|
|
pthread_mutex_unlock(&audio->input_mutex);
|
|
}
|
|
|
|
void audio_output_disconnect(audio_t audio,
|
|
void (*callback)(void *param, const struct audio_data *data),
|
|
void *param)
|
|
{
|
|
pthread_mutex_lock(&audio->input_mutex);
|
|
|
|
size_t idx = audio_get_input_idx(audio, callback, param);
|
|
if (idx != DARRAY_INVALID)
|
|
da_erase(audio->inputs, idx);
|
|
|
|
pthread_mutex_unlock(&audio->input_mutex);
|
|
}
|
|
|
|
static inline bool valid_audio_params(struct audio_output_info *info)
|
|
{
|
|
return info->format && info->name && info->samples_per_sec > 0 &&
|
|
info->speakers > 0;
|
|
}
|
|
|
|
int audio_output_open(audio_t *audio, struct audio_output_info *info)
|
|
{
|
|
struct audio_output *out;
|
|
pthread_mutexattr_t attr;
|
|
bool planar = is_audio_planar(info->format);
|
|
|
|
if (!valid_audio_params(info))
|
|
return AUDIO_OUTPUT_INVALIDPARAM;
|
|
|
|
out = bmalloc(sizeof(struct audio_output));
|
|
memset(out, 0, sizeof(struct audio_output));
|
|
|
|
memcpy(&out->info, info, sizeof(struct audio_output_info));
|
|
pthread_mutex_init_value(&out->line_mutex);
|
|
out->channels = get_audio_channels(info->speakers);
|
|
out->planes = planar ? out->channels : 1;
|
|
out->block_size = (planar ? 1 : out->channels) *
|
|
get_audio_bytes_per_channel(info->format);
|
|
|
|
if (pthread_mutexattr_init(&attr) != 0)
|
|
goto fail;
|
|
if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0)
|
|
goto fail;
|
|
if (pthread_mutex_init(&out->line_mutex, &attr) != 0)
|
|
goto fail;
|
|
if (pthread_mutex_init(&out->input_mutex, NULL) != 0)
|
|
goto fail;
|
|
if (event_init(&out->stop_event, EVENT_TYPE_MANUAL) != 0)
|
|
goto fail;
|
|
if (pthread_create(&out->thread, NULL, audio_thread, out) != 0)
|
|
goto fail;
|
|
|
|
out->initialized = true;
|
|
*audio = out;
|
|
return AUDIO_OUTPUT_SUCCESS;
|
|
|
|
fail:
|
|
audio_output_close(out);
|
|
return AUDIO_OUTPUT_FAIL;
|
|
}
|
|
|
|
void audio_output_close(audio_t audio)
|
|
{
|
|
void *thread_ret;
|
|
struct audio_line *line;
|
|
|
|
if (!audio)
|
|
return;
|
|
|
|
if (audio->initialized) {
|
|
event_signal(&audio->stop_event);
|
|
pthread_join(audio->thread, &thread_ret);
|
|
}
|
|
|
|
line = audio->first_line;
|
|
while (line) {
|
|
struct audio_line *next = line->next;
|
|
audio_line_destroy_data(line);
|
|
line = next;
|
|
}
|
|
|
|
for (size_t i = 0; i < MAX_AUDIO_PLANES; i++)
|
|
da_free(audio->mix_buffers[i]);
|
|
|
|
event_destroy(&audio->stop_event);
|
|
pthread_mutex_destroy(&audio->line_mutex);
|
|
bfree(audio);
|
|
}
|
|
|
|
audio_line_t audio_output_createline(audio_t audio, const char *name)
|
|
{
|
|
struct audio_line *line = bmalloc(sizeof(struct audio_line));
|
|
memset(line, 0, sizeof(struct audio_line));
|
|
line->alive = true;
|
|
line->audio = audio;
|
|
|
|
if (pthread_mutex_init(&line->mutex, NULL) != 0) {
|
|
blog(LOG_ERROR, "audio_output_createline: Failed to create "
|
|
"mutex");
|
|
bfree(line);
|
|
return NULL;
|
|
}
|
|
|
|
pthread_mutex_lock(&audio->line_mutex);
|
|
|
|
if (audio->first_line) {
|
|
audio->first_line->prev_next = &line->next;
|
|
line->next = audio->first_line;
|
|
}
|
|
|
|
line->prev_next = &audio->first_line;
|
|
audio->first_line = line;
|
|
|
|
pthread_mutex_unlock(&audio->line_mutex);
|
|
|
|
line->name = bstrdup(name ? name : "(unnamed audio line)");
|
|
return line;
|
|
}
|
|
|
|
const struct audio_output_info *audio_output_getinfo(audio_t audio)
|
|
{
|
|
return &audio->info;
|
|
}
|
|
|
|
void audio_line_destroy(struct audio_line *line)
|
|
{
|
|
if (line) {
|
|
if (!line->buffers[0].size)
|
|
audio_output_removeline(line->audio, line);
|
|
else
|
|
line->alive = false;
|
|
}
|
|
}
|
|
|
|
size_t audio_output_blocksize(audio_t audio)
|
|
{
|
|
return audio->block_size;
|
|
}
|
|
|
|
size_t audio_output_planes(audio_t audio)
|
|
{
|
|
return audio->planes;
|
|
}
|
|
|
|
size_t audio_output_channels(audio_t audio)
|
|
{
|
|
return audio->channels;
|
|
}
|
|
|
|
/* TODO: Optimization of volume multiplication functions */
|
|
|
|
static inline void mul_vol_u8bit(void *array, float volume, size_t total_num)
|
|
{
|
|
uint8_t *vals = array;
|
|
int16_t vol = (int16_t)(volume * 127.0f);
|
|
|
|
for (size_t i = 0; i < total_num; i++) {
|
|
int16_t val = (int16_t)(vals[i] ^ 0x80) << 8;
|
|
vals[i] = (uint8_t)((val * vol / 127) + 128);
|
|
}
|
|
}
|
|
|
|
static inline void mul_vol_16bit(void *array, float volume, size_t total_num)
|
|
{
|
|
uint16_t *vals = array;
|
|
int32_t vol = (int32_t)(volume * 32767.0f);
|
|
|
|
for (size_t i = 0; i < total_num; i++)
|
|
vals[i] = (int32_t)((int32_t)vals[i] * vol / 32767);
|
|
}
|
|
|
|
static inline float conv_24bit_to_float(uint8_t *vals)
|
|
{
|
|
int32_t val = ((int32_t)vals[0]) |
|
|
((int32_t)vals[1] << 8) |
|
|
((int32_t)vals[2] << 16);
|
|
|
|
if ((val & 0x800000) != 0)
|
|
val |= 0xFF000000;
|
|
|
|
return (float)val / 8388607.0f;
|
|
}
|
|
|
|
static inline void conv_float_to_24bit(float fval, uint8_t *vals)
|
|
{
|
|
int32_t val = (int32_t)(fval * 8388607.0f);
|
|
vals[0] = (val) & 0xFF;
|
|
vals[1] = (val >> 8) & 0xFF;
|
|
vals[2] = (val >> 16) & 0xFF;
|
|
}
|
|
|
|
static inline void mul_vol_24bit(void *array, float volume, size_t total_num)
|
|
{
|
|
uint8_t *vals = array;
|
|
|
|
for (size_t i = 0; i < total_num; i++) {
|
|
float val = conv_24bit_to_float(vals) * volume;
|
|
conv_float_to_24bit(val, vals);
|
|
vals += 3;
|
|
}
|
|
}
|
|
|
|
static inline void mul_vol_32bit(void *array, float volume, size_t total_num)
|
|
{
|
|
int32_t *vals = array;
|
|
|
|
for (size_t i = 0; i < total_num; i++) {
|
|
float val = (float)vals[i] / 2147483647.0f;
|
|
vals[i] = (int32_t)(val * volume / 2147483647.0f);
|
|
}
|
|
}
|
|
|
|
static inline void mul_vol_float(void *array, float volume, size_t total_num)
|
|
{
|
|
float *vals = array;
|
|
|
|
for (size_t i = 0; i < total_num; i++)
|
|
vals[i] *= volume;
|
|
}
|
|
|
|
static void audio_line_place_data_pos(struct audio_line *line,
|
|
const struct audio_data *data, size_t position)
|
|
{
|
|
bool planar = line->audio->planes > 1;
|
|
size_t total_num = data->frames * planar ? 1 : line->audio->channels;
|
|
size_t total_size = data->frames * line->audio->block_size;
|
|
|
|
for (size_t i = 0; i < line->audio->planes; i++) {
|
|
da_copy_array(line->volume_buffers[i], data->data[i],
|
|
total_size);
|
|
|
|
uint8_t *array = line->volume_buffers[i].array;
|
|
|
|
switch (line->audio->info.format) {
|
|
case AUDIO_FORMAT_U8BIT:
|
|
case AUDIO_FORMAT_U8BIT_PLANAR:
|
|
mul_vol_u8bit(array, data->volume, total_num);
|
|
break;
|
|
case AUDIO_FORMAT_16BIT:
|
|
case AUDIO_FORMAT_16BIT_PLANAR:
|
|
mul_vol_16bit(array, data->volume, total_num);
|
|
break;
|
|
case AUDIO_FORMAT_32BIT:
|
|
case AUDIO_FORMAT_32BIT_PLANAR:
|
|
mul_vol_32bit(array, data->volume, total_num);
|
|
break;
|
|
case AUDIO_FORMAT_FLOAT:
|
|
case AUDIO_FORMAT_FLOAT_PLANAR:
|
|
mul_vol_float(array, data->volume, total_num);
|
|
break;
|
|
case AUDIO_FORMAT_UNKNOWN:
|
|
blog(LOG_ERROR, "audio_line_place_data_pos: "
|
|
"Unknown format");
|
|
break;
|
|
}
|
|
|
|
circlebuf_place(&line->buffers[i], position,
|
|
line->volume_buffers[i].array, total_size);
|
|
}
|
|
}
|
|
|
|
static inline void audio_line_place_data(struct audio_line *line,
|
|
const struct audio_data *data)
|
|
{
|
|
uint64_t time_offset = data->timestamp - line->base_timestamp;
|
|
size_t pos = time_to_bytes(line->audio, time_offset);
|
|
|
|
audio_line_place_data_pos(line, data, pos);
|
|
}
|
|
|
|
void audio_line_output(audio_line_t line, const struct audio_data *data)
|
|
{
|
|
/* TODO: prevent insertation of data too far away from expected
|
|
* audio timing */
|
|
|
|
pthread_mutex_lock(&line->mutex);
|
|
|
|
if (!line->buffers[0].size) {
|
|
line->base_timestamp = data->timestamp;
|
|
audio_line_place_data_pos(line, data, 0);
|
|
|
|
} else if (line->base_timestamp <= data->timestamp) {
|
|
audio_line_place_data(line, data);
|
|
|
|
} else {
|
|
blog(LOG_DEBUG, "Bad timestamp for audio line '%s', "
|
|
"data->timestamp: %llu, "
|
|
"line->base_timestamp: %llu. This can "
|
|
"sometimes happen when there's a pause in "
|
|
"the threads.", line->name, data->timestamp,
|
|
line->base_timestamp);
|
|
}
|
|
|
|
pthread_mutex_unlock(&line->mutex);
|
|
}
|