jp9000/obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement output, improve video/audio subsystems

Browse Source 
- Fill in the rest of the FFmpeg test output code for testing so it
   actually properly outputs data.

 - Improve the main video subsystem to be a bit more optimal and
   automatically output I420 or NV12 if needed.

 - Fix audio subsystem insertation and byte calculation.  Now it will
   seamlessly insert new audio data in to the audio stream based upon
   its timestamp value.  (Be extremely cautious when using floating
   point calculations for important things like this, and always round
   your values and check your values)

 - Use 32 byte alignment in case of future optimizations and export a
   function to get the current alignment.

 - Make os_sleepto_ns return true if slept, false if the time has
   already been passed before the call.

 - Fix sinewave output so that it actually properly calculates a middle
   C sinewave.

 - Change the use of row_bytes to linesize (also makes it a bit more
   consistent with FFmpeg's naming as well)
This commit is contained in:
jp9000
2014-02-09 05:51:06 -07:00
parent 4461281a3b
commit 6c92cf5841
30 changed files with 500 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
libobs/media-io/audio-io.c
View File

@@ -15,6 +15,8 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
******************************************************************************/
#include <math.h>
#include "../util/threading.h"
#include "../util/darray.h"
#include "../util/circlebuf.h"
@@ -22,6 +24,10 @@
#include "audio-io.h"
/* #define DEBUG_AUDIO */
#define nop() do {int invalid = 0;} while(0)
struct audio_input {
struct audio_convert_info conversion;
void (*callback)(void *param, const struct audio_data *data);
@@ -90,35 +96,63 @@ static inline void audio_output_removeline(struct audio_output *audio,
audio_line_destroy_data(line);
}
static inline uint32_t time_to_frames(audio_t audio, uint64_t offset)
/* ------------------------------------------------------------------------- */
/* the following functions are used to calculate frame offsets based upon
* timestamps. this will actually work accurately as long as you handle the
* values correctly */
static inline double ts_to_frames(audio_t audio, uint64_t ts)
{
double audio_offset_d = (double)offset;
double audio_offset_d = (double)ts;
audio_offset_d /= 1000000000.0;
audio_offset_d *= (double)audio->info.samples_per_sec;
return (uint32_t)audio_offset_d;
return audio_offset_d;
}
static inline size_t time_to_bytes(audio_t audio, uint64_t offset)
static inline double positive_round(double val)
{
return time_to_frames(audio, offset) * audio->block_size;
return floor(val+0.5);
}
static size_t ts_diff_frames(audio_t audio, uint64_t ts1, uint64_t ts2)
{
double diff = ts_to_frames(audio, ts1) - ts_to_frames(audio, ts2);
return (size_t)positive_round(diff);
}
static size_t ts_diff_bytes(audio_t audio, uint64_t ts1, uint64_t ts2)
{
return ts_diff_frames(audio, ts1, ts2) * audio->block_size;
}
/* unless the value is 3+ hours worth of frames, this won't overflow */
static inline uint64_t conv_frames_to_time(audio_t audio, uint32_t frames)
{
return (uint64_t)frames * 1000000000ULL /
(uint64_t)audio->info.samples_per_sec;
}
/* ------------------------------------------------------------------------- */
static inline void clear_excess_audio_data(struct audio_line *line,
uint64_t size)
uint64_t prev_time)
{
size_t size = ts_diff_bytes(line->audio, prev_time,
line->base_timestamp);
blog(LOG_WARNING, "Excess audio data for audio line '%s', somehow "
"audio data went back in time by %lu bytes. "
"prev_time: %llu, line->base_timestamp: %llu",
line->name, (uint32_t)size,
prev_time, line->base_timestamp);
for (size_t i = 0; i < line->audio->planes; i++) {
size_t clear_size = (size > line->buffers[i].size) ?
(size_t)size : line->buffers[i].size;
circlebuf_pop_front(&line->buffers[i], NULL, clear_size);
}
blog(LOG_WARNING, "Excess audio data for audio line '%s', somehow "
"audio data went back in time by %llu bytes",
line->name, size);
}
static inline uint64_t min_uint64(uint64_t a, uint64_t b)
@@ -126,31 +160,35 @@ static inline uint64_t min_uint64(uint64_t a, uint64_t b)
return a < b ? a : b;
}
static inline void mix_audio_line(struct audio_output *audio,
static inline size_t min_size(size_t a, size_t b)
{
return a < b ? a : b;
}
/* TODO: this just overwrites. handle actual mixing */
static inline bool mix_audio_line(struct audio_output *audio,
struct audio_line *line, size_t size, uint64_t timestamp)
{
/* TODO: this just overwrites. handle actual mixing */
if (!line->buffers[0].size) {
if (!line->alive)
audio_output_removeline(audio, line);
return;
}
size_t time_offset = time_to_bytes(audio,
line->base_timestamp - timestamp);
size_t time_offset = ts_diff_bytes(audio,
line->base_timestamp, timestamp);
if (time_offset > size)
return;
return false;
size -= time_offset;
#ifdef DEBUG_AUDIO
blog(LOG_DEBUG, "shaved off %lu bytes", size);
#endif
for (size_t i = 0; i < audio->planes; i++) {
size_t pop_size;
pop_size = (size_t)min_uint64(size, line->buffers[i].size);
size_t pop_size = min_size(size, line->buffers[i].size);
circlebuf_pop_front(&line->buffers[i],
audio->mix_buffers[i].array + time_offset,
pop_size);
}
return true;
}
static inline void do_audio_output(struct audio_output *audio,
@@ -172,34 +210,61 @@ static inline void do_audio_output(struct audio_output *audio,
pthread_mutex_unlock(&audio->input_mutex);
}
static void mix_and_output(struct audio_output *audio, uint64_t audio_time,
static uint64_t mix_and_output(struct audio_output *audio, uint64_t audio_time,
uint64_t prev_time)
{
struct audio_line *line = audio->first_line;
uint64_t time_offset = audio_time - prev_time;
uint32_t frames = time_to_frames(audio, time_offset);
uint32_t frames = (uint32_t)ts_diff_frames(audio, audio_time,
prev_time);
size_t bytes = frames * audio->block_size;
#ifdef DEBUG_AUDIO
blog(LOG_DEBUG, "audio_time: %llu, prev_time: %llu, bytes: %lu",
audio_time, prev_time, bytes);
#endif
/* return an adjusted audio_time according to the amount
* of data that was sampled to ensure seamless transmission */
audio_time = prev_time + conv_frames_to_time(audio, frames);
/* resize and clear mix buffers */
for (size_t i = 0; i < audio->planes; i++) {
da_resize(audio->mix_buffers[i], bytes);
memset(audio->mix_buffers[i].array, 0, bytes);
}
/* mix audio lines */
while (line) {
struct audio_line *next = line->next;
/* if line marked for removal, destroy and move to the next */
if (!line->buffers[0].size) {
if (!line->alive) {
audio_output_removeline(audio, line);
line = next;
continue;
}
}
pthread_mutex_lock(&line->mutex);
if (line->buffers[0].size && line->base_timestamp < prev_time) {
clear_excess_audio_data(line,
prev_time - line->base_timestamp);
clear_excess_audio_data(line, prev_time);
line->base_timestamp = prev_time;
}
mix_audio_line(audio, line, bytes, prev_time);
line->base_timestamp = audio_time;
if (mix_audio_line(audio, line, bytes, prev_time))
line->base_timestamp = audio_time;
pthread_mutex_unlock(&line->mutex);
line = next;
}
/* output */
do_audio_output(audio, prev_time, frames);
return audio_time;
}
/* sample audio 40 times a second */
@@ -218,8 +283,8 @@ static void *audio_thread(void *param)
pthread_mutex_lock(&audio->line_mutex);
audio_time = os_gettime_ns() - buffer_time;
mix_and_output(audio, audio_time, prev_time);
prev_time = audio_time;
audio_time = mix_and_output(audio, audio_time, prev_time);
prev_time = audio_time;
pthread_mutex_unlock(&audio->line_mutex);
}
@@ -530,11 +595,19 @@ static void audio_line_place_data_pos(struct audio_line *line,
}
}
static inline void audio_line_place_data(struct audio_line *line,
void audio_line_place_data(struct audio_line *line,
const struct audio_data *data)
{
uint64_t time_offset = data->timestamp - line->base_timestamp;
size_t pos = time_to_bytes(line->audio, time_offset);
size_t pos = ts_diff_bytes(line->audio, data->timestamp,
line->base_timestamp);
#ifdef DEBUG_AUDIO
blog(LOG_DEBUG, "data->timestamp: %llu, line->base_timestamp: %llu, "
"pos: %lu, bytes: %lu, buf size: %lu",
data->timestamp, line->base_timestamp, pos,
data->frames * line->audio->block_size,
line->buffers[0].size);
#endif
audio_line_place_data_pos(line, data, pos);
}
@@ -547,8 +620,11 @@ void audio_line_output(audio_line_t line, const struct audio_data *data)
pthread_mutex_lock(&line->mutex);
if (!line->buffers[0].size) {
line->base_timestamp = data->timestamp;
audio_line_place_data_pos(line, data, 0);
/* XXX: not entirely sure if this is the wisest course of
* action in all circumstances */
line->base_timestamp = data->timestamp -
line->audio->info.buffer_ms * 1000000;
audio_line_place_data(line, data);
} else if (line->base_timestamp <= data->timestamp) {
audio_line_place_data(line, data);

4
libobs/media-io/audio-resampler-ffmpeg.c
View File

@@ -77,8 +77,8 @@ static inline uint64_t convert_speaker_layout(enum speaker_layout layout)
return 0;
}
audio_resampler_t audio_resampler_create(struct resample_info *dst,
struct resample_info *src)
audio_resampler_t audio_resampler_create(const struct resample_info *dst,
const struct resample_info *src)
{
struct audio_resampler *rs = bmalloc(sizeof(struct audio_resampler));
int errcode;

4
libobs/media-io/audio-resampler.h
View File

@@ -33,8 +33,8 @@ struct resample_info {
enum speaker_layout speakers;
};
EXPORT audio_resampler_t audio_resampler_create(struct resample_info *dst,
struct resample_info *src);
EXPORT audio_resampler_t audio_resampler_create(const struct resample_info *dst,
const struct resample_info *src);
EXPORT void audio_resampler_destroy(audio_resampler_t resampler);
EXPORT bool audio_resampler_resample(audio_resampler_t resampler,

66
libobs/media-io/format-conversion.c
View File

@@ -86,10 +86,10 @@ static inline void pack_chroma_2plane(uint8_t *u_plane, uint8_t *v_plane,
}
void compress_uyvx_to_i420(
const uint8_t *input, uint32_t in_row_bytes,
const uint8_t *input, uint32_t in_linesize,
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output[], const uint32_t out_row_bytes[])
uint8_t *output[], const uint32_t out_linesize[])
{
uint8_t *lum_plane = output[0];
uint8_t *u_plane = output[1];
@@ -100,19 +100,19 @@ void compress_uyvx_to_i420(
__m128i uv_mask = _mm_set1_epi16(0x00FF);
for (y = start_y; y < end_y; y += 2) {
uint32_t y_pos = y * in_row_bytes;
uint32_t chroma_y_pos = (y>>1) * out_row_bytes[1];
uint32_t lum_y_pos = y * out_row_bytes[0];
uint32_t y_pos = y * in_linesize;
uint32_t chroma_y_pos = (y>>1) * out_linesize[1];
uint32_t lum_y_pos = y * out_linesize[0];
uint32_t x;
for (x = 0; x < width; x += 4) {
const uint8_t *img = input + y_pos + x*4;
uint32_t lum_pos0 = lum_y_pos + x;
uint32_t lum_pos1 = lum_pos0 + out_row_bytes[0];
uint32_t lum_pos1 = lum_pos0 + out_linesize[0];
__m128i line1 = _mm_load_si128((const __m128i*)img);
__m128i line2 = _mm_load_si128(
(const __m128i*)(img + in_row_bytes));
(const __m128i*)(img + in_linesize));
pack_lum(lum_plane, lum_pos0, lum_pos1,
line1, line2, lum_mask);
@@ -124,10 +124,10 @@ void compress_uyvx_to_i420(
}
void compress_uyvx_to_nv12(
const uint8_t *input, uint32_t in_row_bytes,
const uint8_t *input, uint32_t in_linesize,
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output[], const uint32_t out_row_bytes[])
uint8_t *output[], const uint32_t out_linesize[])
{
uint8_t *lum_plane = output[0];
uint8_t *chroma_plane = output[1];
@@ -137,19 +137,19 @@ void compress_uyvx_to_nv12(
__m128i uv_mask = _mm_set1_epi16(0x00FF);
for (y = start_y; y < end_y; y += 2) {
uint32_t y_pos = y * in_row_bytes;
uint32_t chroma_y_pos = (y>>1) * out_row_bytes[1];
uint32_t lum_y_pos = y * out_row_bytes[0];
uint32_t y_pos = y * in_linesize;
uint32_t chroma_y_pos = (y>>1) * out_linesize[1];
uint32_t lum_y_pos = y * out_linesize[0];
uint32_t x;
for (x = 0; x < width; x += 4) {
const uint8_t *img = input + y_pos + x*4;
uint32_t lum_pos0 = lum_y_pos + x;
uint32_t lum_pos1 = lum_pos0 + out_row_bytes[0];
uint32_t lum_pos1 = lum_pos0 + out_linesize[0];
__m128i line1 = _mm_load_si128((const __m128i*)img);
__m128i line2 = _mm_load_si128(
(const __m128i*)(img + in_row_bytes));
(const __m128i*)(img + in_linesize));
pack_lum(lum_plane, lum_pos0, lum_pos1,
line1, line2, lum_mask);
@@ -160,10 +160,10 @@ void compress_uyvx_to_nv12(
}
void decompress_420(
const uint8_t *const input[], const uint32_t in_row_bytes[],
const uint8_t *const input[], const uint32_t in_linesize[],
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output, uint32_t out_row_bytes)
uint8_t *output, uint32_t out_linesize)
{
uint32_t start_y_d2 = start_y/2;
uint32_t width_d2 = width/2;
@@ -171,16 +171,16 @@ void decompress_420(
uint32_t y;
for (y = start_y_d2; y < height_d2; y++) {
const uint8_t *chroma0 = input[1] + y * in_row_bytes[1];
const uint8_t *chroma1 = input[2] + y * in_row_bytes[2];
const uint8_t *chroma0 = input[1] + y * in_linesize[1];
const uint8_t *chroma1 = input[2] + y * in_linesize[2];
register const uint8_t *lum0, *lum1;
register uint32_t *output0, *output1;
uint32_t x;
lum0 = input[0] + y * 2*width;
lum1 = lum0 + width;
output0 = (uint32_t*)(output + y * 2 * in_row_bytes[0]);
output1 = (uint32_t*)((uint8_t*)output0 + in_row_bytes[0]);
output0 = (uint32_t*)(output + y * 2 * in_linesize[0]);
output1 = (uint32_t*)((uint8_t*)output0 + in_linesize[0]);
for (x = 0; x < width_d2; x++) {
uint32_t out;
@@ -196,10 +196,10 @@ void decompress_420(
}
void decompress_nv12(
const uint8_t *const input[], const uint32_t in_row_bytes[],
const uint8_t *const input[], const uint32_t in_linesize[],
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output, uint32_t out_row_bytes)
uint8_t *output, uint32_t out_linesize)
{
uint32_t start_y_d2 = start_y/2;
uint32_t width_d2 = width/2;
@@ -212,11 +212,11 @@ void decompress_nv12(
register uint32_t *output0, *output1;
uint32_t x;
chroma = (const uint16_t*)(input[1] + y * in_row_bytes[1]);
lum0 = input[0] + y*2 * in_row_bytes[0];
lum1 = lum0 + in_row_bytes[0];
output0 = (uint32_t*)(output + y*2 * out_row_bytes);
output1 = (uint32_t*)((uint8_t*)output0 + out_row_bytes);
chroma = (const uint16_t*)(input[1] + y * in_linesize[1]);
lum0 = input[0] + y*2 * in_linesize[0];
lum1 = lum0 + in_linesize[0];
output0 = (uint32_t*)(output + y*2 * out_linesize);
output1 = (uint32_t*)((uint8_t*)output0 + out_linesize);
for (x = 0; x < width_d2; x++) {
uint32_t out = *(chroma++) << 8;
@@ -231,10 +231,10 @@ void decompress_nv12(
}
void decompress_422(
const uint8_t *input, uint32_t in_row_bytes,
const uint8_t *input, uint32_t in_linesize,
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output, uint32_t out_row_bytes,
uint8_t *output, uint32_t out_linesize,
bool leading_lum)
{
uint32_t width_d2 = width >> 1;
@@ -246,9 +246,9 @@ void decompress_422(
if (leading_lum) {
for (y = start_y; y < end_y; y++) {
input32 = (const uint32_t*)(input + y*in_row_bytes);
input32 = (const uint32_t*)(input + y*in_linesize);
input32_end = input32 + width_d2;
output32 = (uint32_t*)(output + y*out_row_bytes);
output32 = (uint32_t*)(output + y*out_linesize);
while(input32 < input32_end) {
register uint32_t dw = *input32;
@@ -264,9 +264,9 @@ void decompress_422(
}
} else {
for (y = start_y; y < end_y; y++) {
input32 = (const uint32_t*)(input + y*in_row_bytes);
input32 = (const uint32_t*)(input + y*in_linesize);
input32_end = input32 + width_d2;
output32 = (uint32_t*)(output + y*out_row_bytes);
output32 = (uint32_t*)(output + y*out_linesize);
while (input32 < input32_end) {
register uint32_t dw = *input32;

20
libobs/media-io/format-conversion.h
View File

@@ -28,34 +28,34 @@ extern "C" {
*/
EXPORT void compress_uyvx_to_i420(
const uint8_t *input, uint32_t in_row_bytes,
const uint8_t *input, uint32_t in_linesize,
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output[], const uint32_t out_row_bytes[]);
uint8_t *output[], const uint32_t out_linesize[]);
EXPORT void compress_uyvx_to_nv12(
const uint8_t *input, uint32_t in_row_bytes,
const uint8_t *input, uint32_t in_linesize,
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output[], const uint32_t out_row_bytes[]);
uint8_t *output[], const uint32_t out_linesize[]);
EXPORT void decompress_nv12(
const uint8_t *const input[], const uint32_t in_row_bytes[],
const uint8_t *const input[], const uint32_t in_linesize[],
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output, uint32_t out_row_bytes);
uint8_t *output, uint32_t out_linesize);
EXPORT void decompress_420(
const uint8_t *const input[], const uint32_t in_row_bytes[],
const uint8_t *const input[], const uint32_t in_linesize[],
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output, uint32_t out_row_bytes);
uint8_t *output, uint32_t out_linesize);
EXPORT void decompress_422(
const uint8_t *input, uint32_t in_row_bytes,
const uint8_t *input, uint32_t in_linesize,
uint32_t width, uint32_t height,
uint32_t start_y, uint32_t end_y,
uint8_t *output, uint32_t out_row_bytes,
uint8_t *output, uint32_t out_linesize,
bool leading_lum);
#ifdef __cplusplus

30
libobs/media-io/video-io.c
View File

@@ -55,14 +55,10 @@ struct video_output {
static inline void video_swapframes(struct video_output *video)
{
pthread_mutex_lock(&video->data_mutex);
if (video->new_frame) {
video->cur_frame = video->next_frame;
video->new_frame = false;
}
pthread_mutex_unlock(&video->data_mutex);
}
static inline void video_output_cur_frame(struct video_output *video)
@@ -75,28 +71,10 @@ static inline void video_output_cur_frame(struct video_output *video)
pthread_mutex_lock(&video->input_mutex);
/* TEST CODE */
/*static struct video_frame frame = {0};
if (!frame.data[0]) {
frame.data[0] = bmalloc(width * height);
frame.data[1] = bmalloc((width/2) * (height/2));
frame.data[2] = bmalloc((width/2) * (height/2));
frame.row_size[0] = width;
frame.row_size[1] = width/2;
frame.row_size[2] = width/2;
}
compress_uyvx_to_i420(
video->cur_frame.data[0], video->cur_frame.row_size[0],
width, height, 0, height,
(uint8_t**)frame.data, (uint32_t*)frame.row_size);*/
/* TODO: conversion */
for (size_t i = 0; i < video->inputs.num; i++) {
struct video_input *input = video->inputs.array+i;
input->callback(input->param, &video->cur_frame);//&frame);
input->callback(input->param, &video->cur_frame);
}
pthread_mutex_unlock(&video->input_mutex);
@@ -115,8 +93,13 @@ static void *video_thread(void *param)
/* wait another half a frame, swap and output frames */
os_sleepto_ns(cur_time += (video->frame_time/2));
pthread_mutex_lock(&video->data_mutex);
video_swapframes(video);
video_output_cur_frame(video);
pthread_mutex_unlock(&video->data_mutex);
}
return NULL;
@@ -217,7 +200,6 @@ void video_output_connect(video_t video,
input.conversion.format = video->info.format;
input.conversion.width = video->info.width;
input.conversion.height = video->info.height;
input.conversion.row_align = 1;
}
da_push_back(video->inputs, &input);

24
libobs/media-io/video-io.h
View File

@@ -43,8 +43,6 @@ enum video_format {
VIDEO_FORMAT_UYVY,
/* packed uncompressed formats */
VIDEO_FORMAT_YUVX,
VIDEO_FORMAT_UYVX,
VIDEO_FORMAT_RGBA,
VIDEO_FORMAT_BGRA,
VIDEO_FORMAT_BGRX,
@@ -52,7 +50,7 @@ enum video_format {
struct video_frame {
const uint8_t *data[MAX_VIDEO_PLANES];
uint32_t row_size[MAX_VIDEO_PLANES];
uint32_t linesize[MAX_VIDEO_PLANES];
uint64_t timestamp;
};
@@ -70,9 +68,27 @@ struct video_convert_info {
enum video_format format;
uint32_t width;
uint32_t height;
uint32_t row_align;
};
static inline bool format_is_yuv(enum video_format format)
{
switch (format) {
case VIDEO_FORMAT_I420:
case VIDEO_FORMAT_NV12:
case VIDEO_FORMAT_YVYU:
case VIDEO_FORMAT_YUY2:
case VIDEO_FORMAT_UYVY:
return true;
case VIDEO_FORMAT_NONE:
case VIDEO_FORMAT_RGBA:
case VIDEO_FORMAT_BGRA:
case VIDEO_FORMAT_BGRX:
return false;
}
return false;
}
#define VIDEO_OUTPUT_SUCCESS 0
#define VIDEO_OUTPUT_INVALIDPARAM -1
#define VIDEO_OUTPUT_FAIL -2