jp9000/obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

obs-filters: Add option to use RNNoise for noise reduction

Browse Source 
This commit adds support to using Xiph and Mozilla RNNoise library for
noise reduction.

RNNoise is a small library using an AI approach to noise reduction
using a pre-trained model like RTX Voice. But unlike RTX Voice, it is
very tiny, use CPU instead of GPU and only use little resources.
Obviously it is not as efficient but will effectively remove background
noise. It uses more CPU than the existing libspeex-based noise
reduction but it also sounds sounds way better.

RNNoise support is added to the noise reduction effect. It can be
enabled with a checkbox in the effect configuration. RNNoise has no
settings.
This commit is contained in:
Jérémy Farnaud
2020-07-11 05:02:06 +02:00
committed by jp9000
parent 51462e4f0f
commit 7e39109a86
6 changed files with 425 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
plugins/obs-filters/CMakeLists.txt
View File

@@ -1,22 +1,48 @@
project(obs-filters)
option(DISABLE_SPEEXDSP "Disable building of the SpeexDSP-based Noise Suppression filter" OFF)
option(DISABLE_RNNOISE "Disable building of the RNNoise-based Noise Suppression filter" OFF)
if(DISABLE_SPEEXDSP)
message(STATUS "SpeexDSP support disabled")
set(LIBSPEEXDSP_FOUND FALSE)
if(DISABLE_SPEEXDSP AND DISABLE_RNNOISE)
message(STATUS "SpeexDSP and RNNoise support disabled")
set(NOISEREDUCTION_ENABLED FALSE)
else()
find_package(Libspeexdsp QUIET)
if(NOT LIBSPEEXDSP_FOUND)
message(STATUS "SpeexDSP support not found")
if(DISABLE_SPEEXDSP)
message(STATUS "SpeexDSP support disabled")
set(LIBSPEEXDSP_FOUND FALSE)
else()
message(STATUS "SpeexDSP supported")
set(obs-filters_LIBSPEEXDSP_SOURCES
find_package(Libspeexdsp QUIET)
if(NOT LIBSPEEXDSP_FOUND)
message(STATUS "SpeexDSP support not found")
set(LIBSPEEXDSP_FOUND FALSE)
else()
message(STATUS "SpeexDSP supported")
add_definitions(-DLIBSPEEXDSP_ENABLED)
endif()
endif()
if(DISABLE_RNNOISE)
message(STATUS "RNNoise support disabled")
set(LIBRNNOISE_FOUND FALSE)
else()
find_package(Librnnoise QUIET)
if(NOT LIBRNNOISE_FOUND)
message(STATUS "RNNoise support not found")
set(LIBRNNOISE_FOUND FALSE)
else()
message(STATUS "RNNoise supported")
add_definitions(-DLIBRNNOISE_ENABLED)
endif()
endif()
if(LIBSPEEXDSP_FOUND OR LIBRNNOISE_FOUND)
message(STATUS "Noise Reduction effect enabled")
set(obs-filters_NOISEREDUCTION_SOURCES
noise-suppress-filter.c)
set(obs-filters_LIBSPEEXDSP_LIBRARIES
${LIBSPEEXDSP_LIBRARIES})
set(obs-filters_NOISEREDUCTION_LIBRARIES
${LIBSPEEXDSP_LIBRARIES} ${LIBRNNOISE_LIBRARIES})
set(NOISEREDUCTION_ENABLED TRUE)
endif()
endif()
@@ -25,7 +51,7 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/obs-filters-config.h.in"
set(obs-filters_config_HEADERS
"${CMAKE_BINARY_DIR}/plugins/obs-filters/config/obs-filters-config.h")
include_directories(${LIBSPEEXDSP_INCLUDE_DIRS}
include_directories(${LIBSPEEXDSP_INCLUDE_DIRS} ${LIBRNNOISE_INCLUDE_DIRS}
"${CMAKE_BINARY_DIR}/plugins/obs-filters/config")
if(MSVC)
@@ -64,11 +90,11 @@ endif()
add_library(obs-filters MODULE
${obs-filters_SOURCES}
${obs-filters_config_HEADERS}
${obs-filters_LIBSPEEXDSP_SOURCES})
${obs-filters_NOISEREDUCTION_SOURCES})
target_link_libraries(obs-filters
libobs
${obs-filters_PLATFORM_DEPS}
${obs-filters_LIBSPEEXDSP_LIBRARIES})
${obs-filters_NOISEREDUCTION_LIBRARIES})
set_target_properties(obs-filters PROPERTIES FOLDER "plugins")
install_obs_plugin_with_data(obs-filters data)

3
plugins/obs-filters/data/locale/en-US.ini
View File

@@ -69,6 +69,9 @@ ScaleFiltering.Bicubic="Bicubic"
ScaleFiltering.Lanczos="Lanczos"
ScaleFiltering.Area="Area"
NoiseSuppress.SuppressLevel="Suppression Level"
NoiseSuppress.Method="Method"
NoiseSuppress.Method.Speex="Speex (lower CPU usage)"
NoiseSuppress.Method.RNNoise="RNNoise (higher quality)"
Saturation="Saturation"
HueShift="Hue Shift"
Amount="Amount"

341
plugins/obs-filters/noise-suppress-filter.c
View File

@@ -3,7 +3,18 @@
#include <util/circlebuf.h>
#include <obs-module.h>
#ifdef LIBSPEEXDSP_ENABLED
#include <speex/speex_preprocess.h>
#endif
#ifdef LIBRNNOISE_ENABLED
#ifdef _MSC_VER
#define ssize_t intptr_t
#endif
#include <rnnoise.h>
#include <media-io/audio-resampler.h>
#endif
/* -------------------------------------------------------- */
@@ -23,12 +34,25 @@
/* -------------------------------------------------------- */
#define S_SUPPRESS_LEVEL "suppress_level"
#define S_METHOD "method"
#define S_METHOD_SPEEX "speex"
#define S_METHOD_RNN "rnnoise"
#define MT_ obs_module_text
#define TEXT_SUPPRESS_LEVEL MT_("NoiseSuppress.SuppressLevel")
#define TEXT_METHOD MT_("NoiseSuppress.Method")
#define TEXT_METHOD_SPEEX MT_("NoiseSuppress.Method.Speex")
#define TEXT_METHOD_RNN MT_("NoiseSuppress.Method.RNNoise")
#define MAX_PREPROC_CHANNELS 8
/* RNNoise constants, these can't be changed */
#define RNNOISE_SAMPLE_RATE 48000
#define RNNOISE_FRAME_SIZE 480
/* If the following constant changes, RNNoise breaks */
#define BUFFER_SIZE_MSEC 10
/* -------------------------------------------------------- */
struct noise_suppress_data {
@@ -36,6 +60,7 @@ struct noise_suppress_data {
int suppress_level;
uint64_t last_timestamp;
uint64_t latency;
size_t frames;
size_t channels;
@@ -44,12 +69,30 @@ struct noise_suppress_data {
struct circlebuf input_buffers[MAX_PREPROC_CHANNELS];
struct circlebuf output_buffers[MAX_PREPROC_CHANNELS];
/* Speex preprocessor state */
SpeexPreprocessState *states[MAX_PREPROC_CHANNELS];
bool use_rnnoise;
/* 16 bit PCM buffers */
#ifdef LIBSPEEXDSP_ENABLED
/* Speex preprocessor state */
SpeexPreprocessState *spx_states[MAX_PREPROC_CHANNELS];
#endif
#ifdef LIBRNNOISE_ENABLED
/* RNNoise state */
DenoiseState *rnn_states[MAX_PREPROC_CHANNELS];
/* Resampler */
audio_resampler_t *rnn_resampler;
audio_resampler_t *rnn_resampler_back;
#endif
/* PCM buffers */
float *copy_buffers[MAX_PREPROC_CHANNELS];
spx_int16_t *segment_buffers[MAX_PREPROC_CHANNELS];
#ifdef LIBSPEEXDSP_ENABLED
spx_int16_t *spx_segment_buffers[MAX_PREPROC_CHANNELS];
#endif
#ifdef LIBRNNOISE_ENABLED
float *rnn_segment_buffers[MAX_PREPROC_CHANNELS];
#endif
/* output data */
struct obs_audio_data output_audio;
@@ -77,12 +120,28 @@ static void noise_suppress_destroy(void *data)
struct noise_suppress_data *ng = data;
for (size_t i = 0; i < ng->channels; i++) {
speex_preprocess_state_destroy(ng->states[i]);
#ifdef LIBSPEEXDSP_ENABLED
speex_preprocess_state_destroy(ng->spx_states[i]);
#endif
#ifdef LIBRNNOISE_ENABLED
rnnoise_destroy(ng->rnn_states[i]);
#endif
circlebuf_free(&ng->input_buffers[i]);
circlebuf_free(&ng->output_buffers[i]);
}
bfree(ng->segment_buffers[0]);
#ifdef LIBSPEEXDSP_ENABLED
bfree(ng->spx_segment_buffers[0]);
#endif
#ifdef LIBRNNOISE_ENABLED
bfree(ng->rnn_segment_buffers[0]);
if (ng->rnn_resampler) {
audio_resampler_destroy(ng->rnn_resampler);
audio_resampler_destroy(ng->rnn_resampler_back);
}
#endif
bfree(ng->copy_buffers[0]);
circlebuf_free(&ng->info_buffer);
da_free(ng->output_data);
@@ -93,42 +152,111 @@ static inline void alloc_channel(struct noise_suppress_data *ng,
uint32_t sample_rate, size_t channel,
size_t frames)
{
ng->states[channel] =
#ifdef LIBSPEEXDSP_ENABLED
ng->spx_states[channel] =
speex_preprocess_state_init((int)frames, sample_rate);
#endif
#ifdef LIBRNNOISE_ENABLED
ng->rnn_states[channel] = rnnoise_create(NULL);
#endif
circlebuf_reserve(&ng->input_buffers[channel], frames * sizeof(float));
circlebuf_reserve(&ng->output_buffers[channel], frames * sizeof(float));
}
static inline enum speaker_layout convert_speaker_layout(uint8_t channels)
{
switch (channels) {
case 0:
return SPEAKERS_UNKNOWN;
case 1:
return SPEAKERS_MONO;
case 2:
return SPEAKERS_STEREO;
case 3:
return SPEAKERS_2POINT1;
case 4:
return SPEAKERS_4POINT0;
case 5:
return SPEAKERS_4POINT1;
case 6:
return SPEAKERS_5POINT1;
case 8:
return SPEAKERS_7POINT1;
default:
return SPEAKERS_UNKNOWN;
}
}
static void noise_suppress_update(void *data, obs_data_t *s)
{
struct noise_suppress_data *ng = data;
uint32_t sample_rate = audio_output_get_sample_rate(obs_get_audio());
size_t channels = audio_output_get_channels(obs_get_audio());
size_t frames = (size_t)sample_rate / 100;
size_t frames = (size_t)sample_rate / (1000 / BUFFER_SIZE_MSEC);
const char *method = obs_data_get_string(s, S_METHOD);
ng->suppress_level = (int)obs_data_get_int(s, S_SUPPRESS_LEVEL);
ng->latency = 1000000000LL / (1000 / BUFFER_SIZE_MSEC);
ng->use_rnnoise = strcmp(method, S_METHOD_RNN) == 0;
/* Process 10 millisecond segments to keep latency low */
/* Also RNNoise only supports buffers of this exact size. */
ng->frames = frames;
ng->channels = channels;
/* Ignore if already allocated */
if (ng->states[0])
#if defined(LIBSPEEXDSP_ENABLED)
if (ng->spx_states[0])
return;
#else
if (ng->rnn_states[0])
return;
#endif
/* One speex state for each channel (limit 2) */
/* One speex/rnnoise state for each channel (limit 2) */
ng->copy_buffers[0] = bmalloc(frames * channels * sizeof(float));
ng->segment_buffers[0] =
#ifdef LIBSPEEXDSP_ENABLED
ng->spx_segment_buffers[0] =
bmalloc(frames * channels * sizeof(spx_int16_t));
#endif
#ifdef LIBRNNOISE_ENABLED
ng->rnn_segment_buffers[0] =
bmalloc(RNNOISE_FRAME_SIZE * channels * sizeof(float));
#endif
for (size_t c = 1; c < channels; ++c) {
ng->copy_buffers[c] = ng->copy_buffers[c - 1] + frames;
ng->segment_buffers[c] = ng->segment_buffers[c - 1] + frames;
#ifdef LIBSPEEXDSP_ENABLED
ng->spx_segment_buffers[c] =
ng->spx_segment_buffers[c - 1] + frames;
#endif
#ifdef LIBRNNOISE_ENABLED
ng->rnn_segment_buffers[c] =
ng->rnn_segment_buffers[c - 1] + RNNOISE_FRAME_SIZE;
#endif
}
for (size_t i = 0; i < channels; i++)
alloc_channel(ng, sample_rate, i, frames);
#ifdef LIBRNNOISE_ENABLED
if (sample_rate == RNNOISE_SAMPLE_RATE) {
ng->rnn_resampler = NULL;
ng->rnn_resampler_back = NULL;
} else {
struct resample_info src, dst;
src.samples_per_sec = sample_rate;
src.format = AUDIO_FORMAT_FLOAT_PLANAR;
src.speakers = convert_speaker_layout((uint8_t)channels);
dst.samples_per_sec = RNNOISE_SAMPLE_RATE;
dst.format = AUDIO_FORMAT_FLOAT_PLANAR;
dst.speakers = convert_speaker_layout((uint8_t)channels);
ng->rnn_resampler = audio_resampler_create(&dst, &src);
ng->rnn_resampler_back = audio_resampler_create(&src, &dst);
}
#endif
}
static void *noise_suppress_create(obs_data_t *settings, obs_source_t *filter)
@@ -141,16 +269,12 @@ static void *noise_suppress_create(obs_data_t *settings, obs_source_t *filter)
return ng;
}
static inline void process(struct noise_suppress_data *ng)
static inline void process_speexdsp(struct noise_suppress_data *ng)
{
/* Pop from input circlebuf */
for (size_t i = 0; i < ng->channels; i++)
circlebuf_pop_front(&ng->input_buffers[i], ng->copy_buffers[i],
ng->frames * sizeof(float));
#ifdef LIBSPEEXDSP_ENABLED
/* Set args */
for (size_t i = 0; i < ng->channels; i++)
speex_preprocess_ctl(ng->states[i],
speex_preprocess_ctl(ng->spx_states[i],
SPEEX_PREPROCESS_SET_NOISE_SUPPRESS,
&ng->suppress_level);
@@ -162,19 +286,111 @@ static inline void process(struct noise_suppress_data *ng)
s = 1.0f;
else if (s < -1.0f)
s = -1.0f;
ng->segment_buffers[i][j] =
ng->spx_segment_buffers[i][j] =
(spx_int16_t)(s * c_32_to_16);
}
/* Execute */
for (size_t i = 0; i < ng->channels; i++)
speex_preprocess_run(ng->states[i], ng->segment_buffers[i]);
speex_preprocess_run(ng->spx_states[i],
ng->spx_segment_buffers[i]);
/* Convert back to 32bit */
for (size_t i = 0; i < ng->channels; i++)
for (size_t j = 0; j < ng->frames; j++)
ng->copy_buffers[i][j] =
(float)ng->segment_buffers[i][j] / c_16_to_32;
(float)ng->spx_segment_buffers[i][j] /
c_16_to_32;
#endif
}
static inline void process_rnnoise(struct noise_suppress_data *ng)
{
#ifdef LIBRNNOISE_ENABLED
/* Adjust signal level to what RNNoise expects, resample if necessary */
if (ng->rnn_resampler) {
float *output[MAX_PREPROC_CHANNELS];
uint32_t out_frames;
uint64_t ts_offset;
audio_resampler_resample(ng->rnn_resampler, (uint8_t **)output,
&out_frames, &ts_offset,
(const uint8_t **)ng->copy_buffers,
(uint32_t)ng->frames);
for (size_t i = 0; i < ng->channels; i++) {
for (ssize_t j = 0, k = (ssize_t)out_frames -
RNNOISE_FRAME_SIZE;
j < RNNOISE_FRAME_SIZE; ++j, ++k) {
if (k >= 0) {
ng->rnn_segment_buffers[i][j] =
output[i][k] * 32768.0f;
} else {
ng->rnn_segment_buffers[i][j] = 0;
}
}
}
} else {
for (size_t i = 0; i < ng->channels; i++) {
for (size_t j = 0; j < RNNOISE_FRAME_SIZE; ++j) {
ng->rnn_segment_buffers[i][j] =
ng->copy_buffers[i][j] * 32768.0f;
}
}
}
/* Execute */
for (size_t i = 0; i < ng->channels; i++) {
rnnoise_process_frame(ng->rnn_states[i],
ng->rnn_segment_buffers[i],
ng->rnn_segment_buffers[i]);
}
/* Revert signal level adjustment, resample back if necessary */
if (ng->rnn_resampler) {
float *output[MAX_PREPROC_CHANNELS];
uint32_t out_frames;
uint64_t ts_offset;
audio_resampler_resample(
ng->rnn_resampler_back, (uint8_t **)output, &out_frames,
&ts_offset, (const uint8_t **)ng->rnn_segment_buffers,
RNNOISE_FRAME_SIZE);
for (size_t i = 0; i < ng->channels; i++) {
for (ssize_t j = 0,
k = (ssize_t)out_frames - ng->frames;
j < (ssize_t)ng->frames; ++j, ++k) {
if (k >= 0) {
ng->copy_buffers[i][j] =
output[i][k] / 32768.0f;
} else {
ng->copy_buffers[i][j] = 0;
}
}
}
} else {
for (size_t i = 0; i < ng->channels; i++) {
for (size_t j = 0; j < RNNOISE_FRAME_SIZE; ++j) {
ng->copy_buffers[i][j] =
ng->rnn_segment_buffers[i][j] /
32768.0f;
}
}
}
#endif
}
static inline void process(struct noise_suppress_data *ng)
{
/* Pop from input circlebuf */
for (size_t i = 0; i < ng->channels; i++)
circlebuf_pop_front(&ng->input_buffers[i], ng->copy_buffers[i],
ng->frames * sizeof(float));
if (ng->use_rnnoise) {
process_rnnoise(ng);
} else {
process_speexdsp(ng);
}
/* Push to output circlebuf */
for (size_t i = 0; i < ng->channels; i++)
@@ -210,8 +426,13 @@ noise_suppress_filter_audio(void *data, struct obs_audio_data *audio)
size_t segment_size = ng->frames * sizeof(float);
size_t out_size;
if (!ng->states[0])
#ifdef LIBSPEEXDSP_ENABLED
if (!ng->spx_states[0])
return audio;
#else
if (!ng->rnn_states[0])
return audio;
#endif
/* -----------------------------------------------
* if timestamp has dramatically changed, consider it a new stream of
@@ -269,24 +490,66 @@ noise_suppress_filter_audio(void *data, struct obs_audio_data *audio)
}
ng->output_audio.frames = info.frames;
ng->output_audio.timestamp = info.timestamp;
ng->output_audio.timestamp = info.timestamp - ng->latency;
return &ng->output_audio;
}
static void noise_suppress_defaults(obs_data_t *s)
static bool noise_suppress_method_modified(obs_properties_t *props,
obs_property_t *property,
obs_data_t *settings)
{
obs_property_t *p_suppress_level =
obs_properties_get(props, S_SUPPRESS_LEVEL);
const char *method = obs_data_get_string(settings, S_METHOD);
bool enable_level = strcmp(method, S_METHOD_SPEEX) == 0;
obs_property_set_visible(p_suppress_level, enable_level);
UNUSED_PARAMETER(property);
return true;
}
static void noise_suppress_defaults_v1(obs_data_t *s)
{
obs_data_set_default_int(s, S_SUPPRESS_LEVEL, -30);
#if defined(LIBRNNOISE_ENABLED) && !defined(LIBSPEEXDSP_ENABLED)
obs_data_set_default_string(s, S_METHOD, S_METHOD_RNN);
#else
obs_data_set_default_string(s, S_METHOD, S_METHOD_SPEEX);
#endif
}
static void noise_suppress_defaults_v2(obs_data_t *s)
{
obs_data_set_default_int(s, S_SUPPRESS_LEVEL, -30);
#if defined(LIBRNNOISE_ENABLED)
obs_data_set_default_string(s, S_METHOD, S_METHOD_RNN);
#else
obs_data_set_default_string(s, S_METHOD, S_METHOD_SPEEX);
#endif
}
static obs_properties_t *noise_suppress_properties(void *data)
{
obs_properties_t *ppts = obs_properties_create();
obs_property_t *p = obs_properties_add_int_slider(ppts,
S_SUPPRESS_LEVEL,
TEXT_SUPPRESS_LEVEL,
SUP_MIN, SUP_MAX, 1);
obs_property_int_set_suffix(p, " dB");
#if defined(LIBRNNOISE_ENABLED) && defined(LIBSPEEXDSP_ENABLED)
obs_property_t *method = obs_properties_add_list(
ppts, S_METHOD, TEXT_METHOD, OBS_COMBO_TYPE_LIST,
OBS_COMBO_FORMAT_STRING);
obs_property_list_add_string(method, TEXT_METHOD_SPEEX, S_METHOD_SPEEX);
obs_property_list_add_string(method, TEXT_METHOD_RNN, S_METHOD_RNN);
obs_property_set_modified_callback(method,
noise_suppress_method_modified);
#endif
#ifdef LIBSPEEXDSP_ENABLED
obs_property_t *speex_slider = obs_properties_add_int_slider(
ppts, S_SUPPRESS_LEVEL, TEXT_SUPPRESS_LEVEL, SUP_MIN, SUP_MAX,
1);
obs_property_int_set_suffix(speex_slider, " dB");
#endif
UNUSED_PARAMETER(data);
return ppts;
@@ -295,12 +558,26 @@ static obs_properties_t *noise_suppress_properties(void *data)
struct obs_source_info noise_suppress_filter = {
.id = "noise_suppress_filter",
.type = OBS_SOURCE_TYPE_FILTER,
.output_flags = OBS_SOURCE_AUDIO | OBS_SOURCE_CAP_OBSOLETE,
.get_name = noise_suppress_name,
.create = noise_suppress_create,
.destroy = noise_suppress_destroy,
.update = noise_suppress_update,
.filter_audio = noise_suppress_filter_audio,
.get_defaults = noise_suppress_defaults_v1,
.get_properties = noise_suppress_properties,
};
struct obs_source_info noise_suppress_filter_v2 = {
.id = "noise_suppress_filter",
.version = 2,
.type = OBS_SOURCE_TYPE_FILTER,
.output_flags = OBS_SOURCE_AUDIO,
.get_name = noise_suppress_name,
.create = noise_suppress_create,
.destroy = noise_suppress_destroy,
.update = noise_suppress_update,
.filter_audio = noise_suppress_filter_audio,
.get_defaults = noise_suppress_defaults,
.get_defaults = noise_suppress_defaults_v2,
.get_properties = noise_suppress_properties,
};

2
plugins/obs-filters/obs-filters-config.h.in
View File

@@ -8,4 +8,4 @@
#define FALSE 0
#endif
#define SPEEXDSP_ENABLED @LIBSPEEXDSP_FOUND@
#define NOISEREDUCTION_ENABLED @NOISEREDUCTION_ENABLED@

6
plugins/obs-filters/obs-filters.c
View File

@@ -20,8 +20,9 @@ extern struct obs_source_info color_grade_filter;
extern struct obs_source_info sharpness_filter;
extern struct obs_source_info chroma_key_filter;
extern struct obs_source_info async_delay_filter;
#if SPEEXDSP_ENABLED
#if NOISEREDUCTION_ENABLED
extern struct obs_source_info noise_suppress_filter;
extern struct obs_source_info noise_suppress_filter_v2;
#endif
extern struct obs_source_info invert_polarity_filter;
extern struct obs_source_info noise_gate_filter;
@@ -44,8 +45,9 @@ bool obs_module_load(void)
obs_register_source(&sharpness_filter);
obs_register_source(&chroma_key_filter);
obs_register_source(&async_delay_filter);
#if SPEEXDSP_ENABLED
#if NOISEREDUCTION_ENABLED
obs_register_source(&noise_suppress_filter);
obs_register_source(&noise_suppress_filter_v2);
#endif
obs_register_source(&invert_polarity_filter);
obs_register_source(&noise_gate_filter);