jp9000
/
obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
obs-studio/plugins/obs-filters/noise-suppress-filter.c

1233 lines
33 KiB
C
Raw Permalink Blame History

#include <stdint.h>
#include <inttypes.h>
#include <util/circlebuf.h>
#include <util/threading.h>
#include <obs-module.h>
#ifdef LIBSPEEXDSP_ENABLED
#include <speex/speex_preprocess.h>
#endif
#ifdef LIBRNNOISE_ENABLED
#ifdef _MSC_VER
#define ssize_t intptr_t
#endif
#include <rnnoise.h>
#include <media-io/audio-resampler.h>
#endif
bool nvafx_loaded = false;
#ifdef LIBNVAFX_ENABLED
#include "nvafx-load.h"
#include <pthread.h>
#endif
/* -------------------------------------------------------- */
#define do_log(level, format, ...) \
blog(level, "[noise suppress: '%s'] " format, \
obs_source_get_name(ng->context), ##__VA_ARGS__)
#define warn(format, ...) do_log(LOG_WARNING, format, ##__VA_ARGS__)
#define info(format, ...) do_log(LOG_INFO, format, ##__VA_ARGS__)
#ifdef _DEBUG
#define debug(format, ...) do_log(LOG_DEBUG, format, ##__VA_ARGS__)
#else
#define debug(format, ...)
#endif
/* -------------------------------------------------------- */
#define S_SUPPRESS_LEVEL "suppress_level"
#define S_NVAFX_INTENSITY "intensity"
#define S_METHOD "method"
#define S_METHOD_SPEEX "speex"
#define S_METHOD_RNN "rnnoise"
#define S_METHOD_NVAFX_DENOISER "denoiser"
#define S_METHOD_NVAFX_DEREVERB "dereverb"
#define S_METHOD_NVAFX_DEREVERB_DENOISER "dereverb_denoiser"
#define MT_ obs_module_text
#define TEXT_SUPPRESS_LEVEL MT_("NoiseSuppress.SuppressLevel")
#define TEXT_NVAFX_INTENSITY MT_("NoiseSuppress.Intensity")
#define TEXT_METHOD MT_("NoiseSuppress.Method")
#define TEXT_METHOD_SPEEX MT_("NoiseSuppress.Method.Speex")
#define TEXT_METHOD_RNN MT_("NoiseSuppress.Method.RNNoise")
#define TEXT_METHOD_NVAFX_DENOISER MT_("NoiseSuppress.Method.Nvafx.Denoiser")
#define TEXT_METHOD_NVAFX_DEREVERB MT_("NoiseSuppress.Method.Nvafx.Dereverb")
#define TEXT_METHOD_NVAFX_DEREVERB_DENOISER \
MT_("NoiseSuppress.Method.Nvafx.DenoiserPlusDereverb")
#define MAX_PREPROC_CHANNELS 8
/* RNNoise constants, these can't be changed */
#define RNNOISE_SAMPLE_RATE 48000
#define RNNOISE_FRAME_SIZE 480
/* nvafx constants, these can't be changed */
#define NVAFX_SAMPLE_RATE 48000
#define NVAFX_FRAME_SIZE \
480 /* the sdk does not explicitly set this as a constant though it relies on it*/
/* If the following constant changes, RNNoise breaks */
#define BUFFER_SIZE_MSEC 10
/* -------------------------------------------------------- */
struct noise_suppress_data {
obs_source_t *context;
int suppress_level;
uint64_t last_timestamp;
uint64_t latency;
size_t frames;
size_t channels;
struct circlebuf info_buffer;
struct circlebuf input_buffers[MAX_PREPROC_CHANNELS];
struct circlebuf output_buffers[MAX_PREPROC_CHANNELS];
bool use_rnnoise;
bool use_nvafx;
bool nvafx_enabled;
bool has_mono_src;
volatile bool reinit_done;
#ifdef LIBSPEEXDSP_ENABLED
/* Speex preprocessor state */
SpeexPreprocessState *spx_states[MAX_PREPROC_CHANNELS];
#endif
#ifdef LIBRNNOISE_ENABLED
/* RNNoise state */
DenoiseState *rnn_states[MAX_PREPROC_CHANNELS];
/* Resampler */
audio_resampler_t *rnn_resampler;
audio_resampler_t *rnn_resampler_back;
#endif
#ifdef LIBNVAFX_ENABLED
/* NVAFX handle, one per audio channel */
NvAFX_Handle handle[MAX_PREPROC_CHANNELS];
uint32_t sample_rate;
float intensity_ratio;
unsigned int num_samples_per_frame, num_channels;
char *model;
bool nvafx_initialized;
const char *fx;
/* Resampler */
audio_resampler_t *nvafx_resampler;
audio_resampler_t *nvafx_resampler_back;
/* Initialization */
bool nvafx_loading;
pthread_t nvafx_thread;
pthread_mutex_t nvafx_mutex;
#endif
/* PCM buffers */
float *copy_buffers[MAX_PREPROC_CHANNELS];
#ifdef LIBSPEEXDSP_ENABLED
spx_int16_t *spx_segment_buffers[MAX_PREPROC_CHANNELS];
#endif
#ifdef LIBRNNOISE_ENABLED
float *rnn_segment_buffers[MAX_PREPROC_CHANNELS];
#endif
#ifdef LIBNVAFX_ENABLED
float *nvafx_segment_buffers[MAX_PREPROC_CHANNELS];
#endif
/* output data */
struct obs_audio_data output_audio;
DARRAY(float) output_data;
};
#ifdef LIBNVAFX_ENABLED
/* global mutex for nvafx load functions since they aren't thread-safe */
bool nvafx_initializer_mutex_initialized;
pthread_mutex_t nvafx_initializer_mutex;
#endif
/* -------------------------------------------------------- */
#define SUP_MIN -60
#define SUP_MAX 0
#ifdef LIBSPEEXDSP_ENABLED
static const float c_32_to_16 = (float)INT16_MAX;
static const float c_16_to_32 = ((float)INT16_MAX + 1.0f);
#endif
/* -------------------------------------------------------- */
static const char *noise_suppress_name(void *unused)
{
UNUSED_PARAMETER(unused);
return obs_module_text("NoiseSuppress");
}
static void noise_suppress_destroy(void *data)
{
struct noise_suppress_data *ng = data;
#ifdef LIBNVAFX_ENABLED
if (ng->nvafx_enabled)
pthread_mutex_lock(&ng->nvafx_mutex);
#endif
for (size_t i = 0; i < ng->channels; i++) {
#ifdef LIBSPEEXDSP_ENABLED
speex_preprocess_state_destroy(ng->spx_states[i]);
#endif
#ifdef LIBRNNOISE_ENABLED
rnnoise_destroy(ng->rnn_states[i]);
#endif
#ifdef LIBNVAFX_ENABLED
if (ng->handle[0]) {
if (NvAFX_DestroyEffect(ng->handle[i]) !=
NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR, "NvAFX_Release() failed");
}
}
#endif
circlebuf_free(&ng->input_buffers[i]);
circlebuf_free(&ng->output_buffers[i]);
}
#ifdef LIBSPEEXDSP_ENABLED
bfree(ng->spx_segment_buffers[0]);
#endif
#ifdef LIBRNNOISE_ENABLED
bfree(ng->rnn_segment_buffers[0]);
if (ng->rnn_resampler) {
audio_resampler_destroy(ng->rnn_resampler);
audio_resampler_destroy(ng->rnn_resampler_back);
}
#endif
#ifdef LIBNVAFX_ENABLED
bfree(ng->nvafx_segment_buffers[0]);
if (ng->nvafx_resampler) {
audio_resampler_destroy(ng->nvafx_resampler);
audio_resampler_destroy(ng->nvafx_resampler_back);
}
bfree(ng->model);
bfree((void *)ng->fx);
if (ng->nvafx_enabled) {
if (ng->use_nvafx)
pthread_join(ng->nvafx_thread, NULL);
pthread_mutex_unlock(&ng->nvafx_mutex);
pthread_mutex_destroy(&ng->nvafx_mutex);
}
#endif
bfree(ng->copy_buffers[0]);
circlebuf_free(&ng->info_buffer);
da_free(ng->output_data);
bfree(ng);
}
static bool nvafx_initialize_internal(void *data)
{
#ifdef LIBNVAFX_ENABLED
struct noise_suppress_data *ng = data;
NvAFX_Status err;
if (!ng->handle[0]) {
ng->sample_rate = NVAFX_SAMPLE_RATE;
for (size_t i = 0; i < ng->channels; i++) {
// Create FX
CUcontext old = {0};
CUcontext curr = {0};
if (cuCtxGetCurrent(&old) != CUDA_SUCCESS) {
goto failure;
}
// if initialization was with rnnoise or speex
if (strcmp(ng->fx, S_METHOD_NVAFX_DENOISER) != 0 &&
strcmp(ng->fx, S_METHOD_NVAFX_DEREVERB) != 0 &&
strcmp(ng->fx, S_METHOD_NVAFX_DEREVERB_DENOISER) !=
0) {
ng->fx = bstrdup(S_METHOD_NVAFX_DENOISER);
}
err = NvAFX_CreateEffect(ng->fx, &ng->handle[i]);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"%s FX creation failed, error %i",
ng->fx, err);
goto failure;
}
if (cuCtxGetCurrent(&curr) != CUDA_SUCCESS) {
goto failure;
}
if (curr != old) {
cuCtxPopCurrent(NULL);
}
// Set sample rate of FX
err = NvAFX_SetU32(ng->handle[i],
NVAFX_PARAM_INPUT_SAMPLE_RATE,
ng->sample_rate);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"NvAFX_SetU32(Sample Rate: %u) failed, error %i",
ng->sample_rate, err);
goto failure;
}
// Set intensity of FX
err = NvAFX_SetFloat(ng->handle[i],
NVAFX_PARAM_INTENSITY_RATIO,
ng->intensity_ratio);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"NvAFX_SetFloat(Intensity Ratio: %f) failed, error %i",
ng->intensity_ratio, err);
goto failure;
}
// Set AI models path
err = NvAFX_SetString(ng->handle[i],
NVAFX_PARAM_MODEL_PATH,
ng->model);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"NvAFX_SetString() failed, error %i",
err);
goto failure;
}
// Load FX
err = NvAFX_Load(ng->handle[i]);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"NvAFX_Load() failed with error %i",
err);
goto failure;
}
os_atomic_set_bool(&ng->reinit_done, true);
}
}
return true;
failure:
ng->use_nvafx = false;
return false;
#else
UNUSED_PARAMETER(data);
return false;
#endif
}
static void *nvafx_initialize(void *data)
{
#ifdef LIBNVAFX_ENABLED
struct noise_suppress_data *ng = data;
NvAFX_Status err;
if (!ng->use_nvafx || !nvafx_loaded) {
return NULL;
}
pthread_mutex_lock(&ng->nvafx_mutex);
pthread_mutex_lock(&nvafx_initializer_mutex);
if (!nvafx_initialize_internal(data)) {
goto failure;
}
if (ng->use_nvafx) {
err = NvAFX_GetU32(ng->handle[0],
NVAFX_PARAM_NUM_INPUT_CHANNELS,
&ng->num_channels);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"NvAFX_GetU32() failed to get the number of channels, error %i",
err);
goto failure;
}
if (ng->num_channels != 1) {
do_log(LOG_ERROR,
"The number of channels is not 1 in the sdk any more ==> update code");
goto failure;
}
NvAFX_Status err = NvAFX_GetU32(
ng->handle[0], NVAFX_PARAM_NUM_INPUT_SAMPLES_PER_FRAME,
&ng->num_samples_per_frame);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"NvAFX_GetU32() failed to get the number of samples per frame, error %i",
err);
goto failure;
}
if (ng->num_samples_per_frame != NVAFX_FRAME_SIZE) {
do_log(LOG_ERROR,
"The number of samples per frame has changed from 480 (= 10 ms) ==> update code");
goto failure;
}
}
ng->nvafx_initialized = true;
ng->nvafx_loading = false;
pthread_mutex_unlock(&nvafx_initializer_mutex);
pthread_mutex_unlock(&ng->nvafx_mutex);
return NULL;
failure:
ng->use_nvafx = false;
pthread_mutex_unlock(&nvafx_initializer_mutex);
pthread_mutex_unlock(&ng->nvafx_mutex);
return NULL;
#else
UNUSED_PARAMETER(data);
return NULL;
#endif
}
static inline void alloc_channel(struct noise_suppress_data *ng,
uint32_t sample_rate, size_t channel,
size_t frames)
{
#ifdef LIBSPEEXDSP_ENABLED
ng->spx_states[channel] =
speex_preprocess_state_init((int)frames, sample_rate);
#else
UNUSED_PARAMETER(sample_rate);
#endif
#ifdef LIBRNNOISE_ENABLED
ng->rnn_states[channel] = rnnoise_create(NULL);
#endif
circlebuf_reserve(&ng->input_buffers[channel], frames * sizeof(float));
circlebuf_reserve(&ng->output_buffers[channel], frames * sizeof(float));
}
static inline enum speaker_layout convert_speaker_layout(uint8_t channels)
{
switch (channels) {
case 0:
return SPEAKERS_UNKNOWN;
case 1:
return SPEAKERS_MONO;
case 2:
return SPEAKERS_STEREO;
case 3:
return SPEAKERS_2POINT1;
case 4:
return SPEAKERS_4POINT0;
case 5:
return SPEAKERS_4POINT1;
case 6:
return SPEAKERS_5POINT1;
case 8:
return SPEAKERS_7POINT1;
default:
return SPEAKERS_UNKNOWN;
}
}
static void noise_suppress_update(void *data, obs_data_t *s)
{
struct noise_suppress_data *ng = data;
uint32_t sample_rate = audio_output_get_sample_rate(obs_get_audio());
size_t channels = audio_output_get_channels(obs_get_audio());
size_t frames = (size_t)sample_rate / (1000 / BUFFER_SIZE_MSEC);
const char *method = obs_data_get_string(s, S_METHOD);
ng->suppress_level = (int)obs_data_get_int(s, S_SUPPRESS_LEVEL);
ng->latency = 1000000000LL / (1000 / BUFFER_SIZE_MSEC);
ng->use_rnnoise = strcmp(method, S_METHOD_RNN) == 0;
bool nvafx_requested =
strcmp(method, S_METHOD_NVAFX_DENOISER) == 0 ||
strcmp(method, S_METHOD_NVAFX_DEREVERB) == 0 ||
strcmp(method, S_METHOD_NVAFX_DEREVERB_DENOISER) == 0;
#ifdef LIBNVAFX_ENABLED
float intensity = (float)obs_data_get_double(s, S_NVAFX_INTENSITY);
if (ng->use_nvafx && ng->nvafx_initialized) {
if (intensity != ng->intensity_ratio &&
(strcmp(ng->fx, method) == 0)) {
NvAFX_Status err;
ng->intensity_ratio = intensity;
pthread_mutex_lock(&ng->nvafx_mutex);
for (size_t i = 0; i < ng->channels; i++) {
err = NvAFX_SetFloat(
ng->handle[i],
NVAFX_PARAM_INTENSITY_RATIO,
ng->intensity_ratio);
if (err != NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"NvAFX_SetFloat(Intensity Ratio: %f) failed, error %i",
ng->intensity_ratio, err);
ng->use_nvafx = false;
}
}
pthread_mutex_unlock(&ng->nvafx_mutex);
}
if ((strcmp(ng->fx, method) != 0)) {
bfree((void *)ng->fx);
ng->fx = bstrdup(method);
ng->intensity_ratio = intensity;
os_atomic_set_bool(&ng->reinit_done, false);
pthread_mutex_lock(&ng->nvafx_mutex);
for (int i = 0; i < (int)ng->channels; i++) {
/* Destroy previous FX */
if (NvAFX_DestroyEffect(ng->handle[i]) !=
NVAFX_STATUS_SUCCESS) {
do_log(LOG_ERROR,
"FX failed to be destroyed.");
ng->use_nvafx = false;
} else
ng->handle[i] = NULL;
}
if (ng->use_nvafx) {
nvafx_initialize_internal(data);
}
pthread_mutex_unlock(&ng->nvafx_mutex);
}
} else {
ng->fx = bstrdup(method);
}
#endif
ng->use_nvafx = ng->nvafx_enabled && nvafx_requested;
/* Process 10 millisecond segments to keep latency low. */
/* Also RNNoise only supports buffers of this exact size. */
/* At 48kHz, NVAFX processes 480 samples which corresponds to 10 ms.*/
ng->frames = frames;
ng->channels = channels;
#ifdef LIBNVAFX_ENABLED
#endif
/* Ignore if already allocated */
#if defined(LIBSPEEXDSP_ENABLED)
if (!ng->use_rnnoise && !ng->use_nvafx && ng->spx_states[0])
return;
#endif
#ifdef LIBNVAFX_ENABLED
if (ng->use_nvafx && (ng->nvafx_initialized || ng->nvafx_loading))
return;
#endif
#ifdef LIBRNNOISE_ENABLED
if (ng->use_rnnoise && ng->rnn_states[0])
return;
#endif
/* One speex/rnnoise state for each channel (limit 2) */
ng->copy_buffers[0] = bmalloc(frames * channels * sizeof(float));
#ifdef LIBSPEEXDSP_ENABLED
ng->spx_segment_buffers[0] =
bmalloc(frames * channels * sizeof(spx_int16_t));
#endif
#ifdef LIBRNNOISE_ENABLED
ng->rnn_segment_buffers[0] =
bmalloc(RNNOISE_FRAME_SIZE * channels * sizeof(float));
#endif
#ifdef LIBNVAFX_ENABLED
ng->nvafx_segment_buffers[0] =
bmalloc(NVAFX_FRAME_SIZE * channels * sizeof(float));
#endif
for (size_t c = 1; c < channels; ++c) {
ng->copy_buffers[c] = ng->copy_buffers[c - 1] + frames;
#ifdef LIBSPEEXDSP_ENABLED
ng->spx_segment_buffers[c] =
ng->spx_segment_buffers[c - 1] + frames;
#endif
#ifdef LIBRNNOISE_ENABLED
ng->rnn_segment_buffers[c] =
ng->rnn_segment_buffers[c - 1] + RNNOISE_FRAME_SIZE;
#endif
#ifdef LIBNVAFX_ENABLED
ng->nvafx_segment_buffers[c] =
ng->nvafx_segment_buffers[c - 1] + NVAFX_FRAME_SIZE;
#endif
}
#ifdef LIBNVAFX_ENABLED
if (!ng->nvafx_initialized && ng->use_nvafx && !ng->nvafx_loading) {
ng->intensity_ratio = intensity;
ng->nvafx_loading = true;
pthread_create(&ng->nvafx_thread, NULL, nvafx_initialize, ng);
}
#endif
for (size_t i = 0; i < channels; i++)
alloc_channel(ng, sample_rate, i, frames);
#ifdef LIBRNNOISE_ENABLED
if (sample_rate == RNNOISE_SAMPLE_RATE) {
ng->rnn_resampler = NULL;
ng->rnn_resampler_back = NULL;
} else {
struct resample_info src, dst;
src.samples_per_sec = sample_rate;
src.format = AUDIO_FORMAT_FLOAT_PLANAR;
src.speakers = convert_speaker_layout((uint8_t)channels);
dst.samples_per_sec = RNNOISE_SAMPLE_RATE;
dst.format = AUDIO_FORMAT_FLOAT_PLANAR;
dst.speakers = convert_speaker_layout((uint8_t)channels);
ng->rnn_resampler = audio_resampler_create(&dst, &src);
ng->rnn_resampler_back = audio_resampler_create(&src, &dst);
}
#endif
#ifdef LIBNVAFX_ENABLED
if (sample_rate == NVAFX_SAMPLE_RATE) {
ng->nvafx_resampler = NULL;
ng->nvafx_resampler_back = NULL;
} else {
struct resample_info src, dst;
src.samples_per_sec = sample_rate;
src.format = AUDIO_FORMAT_FLOAT_PLANAR;
src.speakers = convert_speaker_layout((uint8_t)channels);
dst.samples_per_sec = NVAFX_SAMPLE_RATE;
dst.format = AUDIO_FORMAT_FLOAT_PLANAR;
dst.speakers = convert_speaker_layout((uint8_t)channels);
ng->nvafx_resampler = audio_resampler_create(&dst, &src);
ng->nvafx_resampler_back = audio_resampler_create(&src, &dst);
}
#endif
}
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4706)
#endif
bool load_nvafx(void)
{
#ifdef LIBNVAFX_ENABLED
if (!load_lib()) {
blog(LOG_INFO,
"[noise suppress]: NVIDIA RTX denoiser disabled, redistributable not found");
return false;
}
nvafx_initializer_mutex_initialized =
pthread_mutex_init(&nvafx_initializer_mutex, NULL) == 0;
#define LOAD_SYM_FROM_LIB(sym, lib, dll) \
if (!(sym = (sym##_t)GetProcAddress(lib, #sym))) { \
DWORD err = GetLastError(); \
printf("[noise suppress]: Couldn't load " #sym " from " dll \
": %lu (0x%lx)", \
err, err); \
goto unload_everything; \
}
#define LOAD_SYM(sym) LOAD_SYM_FROM_LIB(sym, nv_audiofx, "NVAudioEffects.dll")
LOAD_SYM(NvAFX_GetEffectList);
LOAD_SYM(NvAFX_CreateEffect);
LOAD_SYM(NvAFX_CreateChainedEffect);
LOAD_SYM(NvAFX_DestroyEffect);
LOAD_SYM(NvAFX_SetU32);
LOAD_SYM(NvAFX_SetU32List);
LOAD_SYM(NvAFX_SetString);
LOAD_SYM(NvAFX_SetStringList);
LOAD_SYM(NvAFX_SetFloat);
LOAD_SYM(NvAFX_SetFloatList);
LOAD_SYM(NvAFX_GetU32);
LOAD_SYM(NvAFX_GetString);
LOAD_SYM(NvAFX_GetStringList);
LOAD_SYM(NvAFX_GetFloat);
LOAD_SYM(NvAFX_GetFloatList);
LOAD_SYM(NvAFX_Load);
LOAD_SYM(NvAFX_GetSupportedDevices);
LOAD_SYM(NvAFX_Run);
LOAD_SYM(NvAFX_Reset);
#undef LOAD_SYM
#define LOAD_SYM(sym) LOAD_SYM_FROM_LIB(sym, nv_cuda, "nvcuda.dll")
LOAD_SYM(cuCtxGetCurrent);
LOAD_SYM(cuCtxPopCurrent);
LOAD_SYM(cuInit);
#undef LOAD_SYM
NvAFX_Status err;
CUresult cudaerr;
NvAFX_Handle h = NULL;
cudaerr = cuInit(0);
if (cudaerr != CUDA_SUCCESS) {
goto cuda_errors;
}
CUcontext old = {0};
CUcontext curr = {0};
cudaerr = cuCtxGetCurrent(&old);
if (cudaerr != CUDA_SUCCESS) {
goto cuda_errors;
}
err = NvAFX_CreateEffect(NVAFX_EFFECT_DENOISER, &h);
cudaerr = cuCtxGetCurrent(&curr);
if (cudaerr != CUDA_SUCCESS) {
goto cuda_errors;
}
if (curr != old) {
cudaerr = cuCtxPopCurrent(NULL);
if (cudaerr != CUDA_SUCCESS)
goto cuda_errors;
}
if (err != NVAFX_STATUS_SUCCESS) {
if (err == NVAFX_STATUS_GPU_UNSUPPORTED) {
blog(LOG_INFO,
"[noise suppress]: NVIDIA RTX AUDIO FX disabled: unsupported GPU");
} else {
blog(LOG_ERROR,
"[noise suppress]: NVIDIA RTX AUDIO FX disabled, error %i",
err);
}
goto unload_everything;
}
err = NvAFX_DestroyEffect(h);
if (err != NVAFX_STATUS_SUCCESS) {
blog(LOG_ERROR,
"[noise suppress]: NVIDIA RTX AUDIO FX disabled, error %i",
err);
goto unload_everything;
}
nvafx_loaded = true;
blog(LOG_INFO, "[noise suppress]: NVIDIA RTX AUDIO FX enabled");
return true;
cuda_errors:
blog(LOG_ERROR,
"[noise suppress]: NVIDIA RTX AUDIO FX disabled, CUDA error %i",
cudaerr);
unload_everything:
release_lib();
#endif
return false;
}
#ifdef _MSC_VER
#pragma warning(pop)
#endif
#ifdef LIBNVAFX_ENABLED
void unload_nvafx(void)
{
release_lib();
if (nvafx_initializer_mutex_initialized) {
pthread_mutex_destroy(&nvafx_initializer_mutex);
nvafx_initializer_mutex_initialized = false;
}
}
#endif
static void *noise_suppress_create(obs_data_t *settings, obs_source_t *filter)
{
struct noise_suppress_data *ng =
bzalloc(sizeof(struct noise_suppress_data));
ng->context = filter;
#ifdef LIBNVAFX_ENABLED
char sdk_path[MAX_PATH];
if (!nvafx_get_sdk_path(sdk_path, sizeof(sdk_path))) {
ng->nvafx_enabled = false;
do_log(LOG_ERROR, "NVAFX redist is not installed.");
} else {
const char *file = "\\models\\denoiser_48k.trtpkg";
size_t size = strlen(sdk_path) + strlen(file) + 1;
char *buffer = (char *)bmalloc(size);
strcpy(buffer, sdk_path);
strcat(buffer, file);
ng->model = buffer;
ng->nvafx_enabled = true;
ng->nvafx_initialized = false;
ng->nvafx_loading = false;
ng->fx = NULL;
pthread_mutex_init(&ng->nvafx_mutex, NULL);
info("NVAFX SDK redist path was found here %s", sdk_path);
}
#endif
noise_suppress_update(ng, settings);
return ng;
}
static inline void process_speexdsp(struct noise_suppress_data *ng)
{
#ifdef LIBSPEEXDSP_ENABLED
/* Set args */
for (size_t i = 0; i < ng->channels; i++)
speex_preprocess_ctl(ng->spx_states[i],
SPEEX_PREPROCESS_SET_NOISE_SUPPRESS,
&ng->suppress_level);
/* Convert to 16bit */
for (size_t i = 0; i < ng->channels; i++)
for (size_t j = 0; j < ng->frames; j++) {
float s = ng->copy_buffers[i][j];
if (s > 1.0f)
s = 1.0f;
else if (s < -1.0f)
s = -1.0f;
ng->spx_segment_buffers[i][j] =
(spx_int16_t)(s * c_32_to_16);
}
/* Execute */
for (size_t i = 0; i < ng->channels; i++)
speex_preprocess_run(ng->spx_states[i],
ng->spx_segment_buffers[i]);
/* Convert back to 32bit */
for (size_t i = 0; i < ng->channels; i++)
for (size_t j = 0; j < ng->frames; j++)
ng->copy_buffers[i][j] =
(float)ng->spx_segment_buffers[i][j] /
c_16_to_32;
#else
UNUSED_PARAMETER(ng);
#endif
}
static inline void process_rnnoise(struct noise_suppress_data *ng)
{
#ifdef LIBRNNOISE_ENABLED
/* Adjust signal level to what RNNoise expects, resample if necessary */
if (ng->rnn_resampler) {
float *output[MAX_PREPROC_CHANNELS];
uint32_t out_frames;
uint64_t ts_offset;
audio_resampler_resample(ng->rnn_resampler, (uint8_t **)output,
&out_frames, &ts_offset,
(const uint8_t **)ng->copy_buffers,
(uint32_t)ng->frames);
for (size_t i = 0; i < ng->channels; i++) {
for (ssize_t j = 0, k = (ssize_t)out_frames -
RNNOISE_FRAME_SIZE;
j < RNNOISE_FRAME_SIZE; ++j, ++k) {
if (k >= 0) {
ng->rnn_segment_buffers[i][j] =
output[i][k] * 32768.0f;
} else {
ng->rnn_segment_buffers[i][j] = 0;
}
}
}
} else {
for (size_t i = 0; i < ng->channels; i++) {
for (size_t j = 0; j < RNNOISE_FRAME_SIZE; ++j) {
ng->rnn_segment_buffers[i][j] =
ng->copy_buffers[i][j] * 32768.0f;
}
}
}
/* Execute */
for (size_t i = 0; i < ng->channels; i++) {
rnnoise_process_frame(ng->rnn_states[i],
ng->rnn_segment_buffers[i],
ng->rnn_segment_buffers[i]);
}
/* Revert signal level adjustment, resample back if necessary */
if (ng->rnn_resampler) {
float *output[MAX_PREPROC_CHANNELS];
uint32_t out_frames;
uint64_t ts_offset;
audio_resampler_resample(
ng->rnn_resampler_back, (uint8_t **)output, &out_frames,
&ts_offset, (const uint8_t **)ng->rnn_segment_buffers,
RNNOISE_FRAME_SIZE);
for (size_t i = 0; i < ng->channels; i++) {
for (ssize_t j = 0,
k = (ssize_t)out_frames - ng->frames;
j < (ssize_t)ng->frames; ++j, ++k) {
if (k >= 0) {
ng->copy_buffers[i][j] =
output[i][k] / 32768.0f;
} else {
ng->copy_buffers[i][j] = 0;
}
}
}
} else {
for (size_t i = 0; i < ng->channels; i++) {
for (size_t j = 0; j < RNNOISE_FRAME_SIZE; ++j) {
ng->copy_buffers[i][j] =
ng->rnn_segment_buffers[i][j] /
32768.0f;
}
}
}
#else
UNUSED_PARAMETER(ng);
#endif
}
static inline void process_nvafx(struct noise_suppress_data *ng)
{
#ifdef LIBNVAFX_ENABLED
if (nvafx_loaded && ng->use_nvafx && ng->nvafx_initialized) {
/* Resample if necessary */
if (ng->nvafx_resampler) {
float *output[MAX_PREPROC_CHANNELS];
uint32_t out_frames;
uint64_t ts_offset;
audio_resampler_resample(
ng->nvafx_resampler, (uint8_t **)output,
&out_frames, &ts_offset,
(const uint8_t **)ng->copy_buffers,
(uint32_t)ng->frames);
for (size_t i = 0; i < ng->channels; i++) {
for (ssize_t j = 0, k = (ssize_t)out_frames -
NVAFX_FRAME_SIZE;
j < NVAFX_FRAME_SIZE; ++j, ++k) {
if (k >= 0) {
ng->nvafx_segment_buffers[i][j] =
output[i][k];
} else {
ng->nvafx_segment_buffers[i][j] =
0;
}
}
}
} else {
for (size_t i = 0; i < ng->channels; i++) {
for (size_t j = 0; j < NVAFX_FRAME_SIZE; ++j) {
ng->nvafx_segment_buffers[i][j] =
ng->copy_buffers[i][j];
}
}
}
/* Execute */
size_t runs = ng->has_mono_src ? 1 : ng->channels;
if (ng->reinit_done) {
for (size_t i = 0; i < runs; i++) {
NvAFX_Status err =
NvAFX_Run(ng->handle[i],
&ng->nvafx_segment_buffers[i],
&ng->nvafx_segment_buffers[i],
ng->num_samples_per_frame,
ng->num_channels);
if (err != NVAFX_STATUS_SUCCESS) {
if (err == NVAFX_STATUS_FAILED) {
do_log(LOG_DEBUG,
"NvAFX_Run() failed, error NVAFX_STATUS_FAILED.\n"
"This can occur when changing the FX and is not consequential.");
os_atomic_set_bool(
&ng->reinit_done,
false); // stop all processing; this will be reset at new init
} else {
do_log(LOG_ERROR,
"NvAFX_Run() failed, error %i.\n",
err);
}
}
}
}
if (ng->has_mono_src) {
memcpy(ng->nvafx_segment_buffers[1],
ng->nvafx_segment_buffers[0],
NVAFX_FRAME_SIZE * sizeof(float));
}
/* Revert signal level adjustment, resample back if necessary */
if (ng->nvafx_resampler) {
float *output[MAX_PREPROC_CHANNELS];
uint32_t out_frames;
uint64_t ts_offset;
audio_resampler_resample(
ng->nvafx_resampler_back, (uint8_t **)output,
&out_frames, &ts_offset,
(const uint8_t **)ng->nvafx_segment_buffers,
NVAFX_FRAME_SIZE);
for (size_t i = 0; i < ng->channels; i++) {
for (ssize_t j = 0, k = (ssize_t)out_frames -
ng->frames;
j < (ssize_t)ng->frames; ++j, ++k) {
if (k >= 0) {
ng->copy_buffers[i][j] =
output[i][k];
} else {
ng->copy_buffers[i][j] = 0;
}
}
}
} else {
for (size_t i = 0; i < ng->channels; i++) {
for (size_t j = 0; j < NVAFX_FRAME_SIZE; ++j) {
ng->copy_buffers[i][j] =
ng->nvafx_segment_buffers[i][j];
}
}
}
}
#else
UNUSED_PARAMETER(ng);
#endif
}
static inline void process(struct noise_suppress_data *ng)
{
/* Pop from input circlebuf */
for (size_t i = 0; i < ng->channels; i++)
circlebuf_pop_front(&ng->input_buffers[i], ng->copy_buffers[i],
ng->frames * sizeof(float));
if (ng->use_rnnoise) {
process_rnnoise(ng);
} else if (ng->use_nvafx) {
if (nvafx_loaded) {
process_nvafx(ng);
}
} else {
process_speexdsp(ng);
}
/* Push to output circlebuf */
for (size_t i = 0; i < ng->channels; i++)
circlebuf_push_back(&ng->output_buffers[i], ng->copy_buffers[i],
ng->frames * sizeof(float));
}
struct ng_audio_info {
uint32_t frames;
uint64_t timestamp;
};
static inline void clear_circlebuf(struct circlebuf *buf)
{
circlebuf_pop_front(buf, NULL, buf->size);
}
static void reset_data(struct noise_suppress_data *ng)
{
for (size_t i = 0; i < ng->channels; i++) {
clear_circlebuf(&ng->input_buffers[i]);
clear_circlebuf(&ng->output_buffers[i]);
}
clear_circlebuf(&ng->info_buffer);
}
static struct obs_audio_data *
noise_suppress_filter_audio(void *data, struct obs_audio_data *audio)
{
struct noise_suppress_data *ng = data;
struct ng_audio_info info;
size_t segment_size = ng->frames * sizeof(float);
size_t out_size;
obs_source_t *parent = obs_filter_get_parent(ng->context);
const char *name = obs_source_get_name(parent);
const char *id = obs_source_get_id(parent);
enum speaker_layout layout = obs_source_get_speaker_layout(parent);
ng->has_mono_src = layout == SPEAKERS_MONO && ng->channels == 2;
#ifdef LIBSPEEXDSP_ENABLED
if (!ng->spx_states[0])
return audio;
#endif
#ifdef LIBRNNOISE_ENABLED
if (!ng->rnn_states[0])
return audio;
#endif
/* -----------------------------------------------
* if timestamp has dramatically changed, consider it a new stream of
* audio data. clear all circular buffers to prevent old audio data
* from being processed as part of the new data. */
if (ng->last_timestamp) {
int64_t diff = llabs((int64_t)ng->last_timestamp -
(int64_t)audio->timestamp);
if (diff > 1000000000LL)
reset_data(ng);
}
ng->last_timestamp = audio->timestamp;
/* -----------------------------------------------
* push audio packet info (timestamp/frame count) to info circlebuf */
info.frames = audio->frames;
info.timestamp = audio->timestamp;
circlebuf_push_back(&ng->info_buffer, &info, sizeof(info));
/* -----------------------------------------------
* push back current audio data to input circlebuf */
for (size_t i = 0; i < ng->channels; i++)
circlebuf_push_back(&ng->input_buffers[i], audio->data[i],
audio->frames * sizeof(float));
/* -----------------------------------------------
* pop/process each 10ms segments, push back to output circlebuf */
while (ng->input_buffers[0].size >= segment_size)
process(ng);
/* -----------------------------------------------
* peek front of info circlebuf, check to see if we have enough to
* pop the expected packet size, if not, return null */
memset(&info, 0, sizeof(info));
circlebuf_peek_front(&ng->info_buffer, &info, sizeof(info));
out_size = info.frames * sizeof(float);
if (ng->output_buffers[0].size < out_size)
return NULL;
/* -----------------------------------------------
* if there's enough audio data buffered in the output circlebuf,
* pop and return a packet */
circlebuf_pop_front(&ng->info_buffer, NULL, sizeof(info));
da_resize(ng->output_data, out_size * ng->channels);
for (size_t i = 0; i < ng->channels; i++) {
ng->output_audio.data[i] =
(uint8_t *)&ng->output_data.array[i * out_size];
circlebuf_pop_front(&ng->output_buffers[i],
ng->output_audio.data[i], out_size);
}
ng->output_audio.frames = info.frames;
ng->output_audio.timestamp = info.timestamp - ng->latency;
return &ng->output_audio;
}
static bool noise_suppress_method_modified(obs_properties_t *props,
obs_property_t *property,
obs_data_t *settings)
{
obs_property_t *p_suppress_level =
obs_properties_get(props, S_SUPPRESS_LEVEL);
obs_property_t *p_navfx_intensity =
obs_properties_get(props, S_NVAFX_INTENSITY);
const char *method = obs_data_get_string(settings, S_METHOD);
bool enable_level = strcmp(method, S_METHOD_SPEEX) == 0;
bool enable_intensity =
strcmp(method, S_METHOD_NVAFX_DENOISER) == 0 ||
strcmp(method, S_METHOD_NVAFX_DEREVERB) == 0 ||
strcmp(method, S_METHOD_NVAFX_DEREVERB_DENOISER) == 0;
obs_property_set_visible(p_suppress_level, enable_level);
obs_property_set_visible(p_navfx_intensity, enable_intensity);
UNUSED_PARAMETER(property);
return true;
}
static void noise_suppress_defaults_v1(obs_data_t *s)
{
obs_data_set_default_int(s, S_SUPPRESS_LEVEL, -30);
#if defined(LIBRNNOISE_ENABLED) && !defined(LIBSPEEXDSP_ENABLED)
obs_data_set_default_string(s, S_METHOD, S_METHOD_RNN);
#else
obs_data_set_default_string(s, S_METHOD, S_METHOD_SPEEX);
#endif
#if defined(LIBNVAFX_ENABLED)
obs_data_set_default_double(s, S_NVAFX_INTENSITY, 1.0);
#endif
}
static void noise_suppress_defaults_v2(obs_data_t *s)
{
obs_data_set_default_int(s, S_SUPPRESS_LEVEL, -30);
#if defined(LIBRNNOISE_ENABLED)
obs_data_set_default_string(s, S_METHOD, S_METHOD_RNN);
#else
obs_data_set_default_string(s, S_METHOD, S_METHOD_SPEEX);
#endif
#if defined(LIBNVAFX_ENABLED)
obs_data_set_default_double(s, S_NVAFX_INTENSITY, 1.0);
#endif
}
static obs_properties_t *noise_suppress_properties(void *data)
{
obs_properties_t *ppts = obs_properties_create();
#ifdef LIBNVAFX_ENABLED
struct noise_suppress_data *ng = (struct noise_suppress_data *)data;
#else
UNUSED_PARAMETER(data);
#endif
#if defined(LIBRNNOISE_ENABLED) && defined(LIBSPEEXDSP_ENABLED)
obs_property_t *method = obs_properties_add_list(
ppts, S_METHOD, TEXT_METHOD, OBS_COMBO_TYPE_LIST,
OBS_COMBO_FORMAT_STRING);
obs_property_list_add_string(method, TEXT_METHOD_SPEEX, S_METHOD_SPEEX);
obs_property_list_add_string(method, TEXT_METHOD_RNN, S_METHOD_RNN);
#ifdef LIBNVAFX_ENABLED
if (ng->nvafx_enabled) {
obs_property_list_add_string(method, TEXT_METHOD_NVAFX_DENOISER,
S_METHOD_NVAFX_DENOISER);
obs_property_list_add_string(method, TEXT_METHOD_NVAFX_DEREVERB,
S_METHOD_NVAFX_DEREVERB);
obs_property_list_add_string(
method, TEXT_METHOD_NVAFX_DEREVERB_DENOISER,
S_METHOD_NVAFX_DEREVERB_DENOISER);
}
#endif
obs_property_set_modified_callback(method,
noise_suppress_method_modified);
#endif
#ifdef LIBSPEEXDSP_ENABLED
obs_property_t *speex_slider = obs_properties_add_int_slider(
ppts, S_SUPPRESS_LEVEL, TEXT_SUPPRESS_LEVEL, SUP_MIN, SUP_MAX,
1);
obs_property_int_set_suffix(speex_slider, " dB");
#endif
#ifdef LIBNVAFX_ENABLED
obs_properties_add_float_slider(ppts, S_NVAFX_INTENSITY,
TEXT_NVAFX_INTENSITY, 0.0f, 1.0f,
0.01f);
#if defined(LIBRNNOISE_ENABLED) && defined(LIBSPEEXDSP_ENABLED)
if (!nvafx_loaded) {
obs_property_list_item_disable(method, 2, true);
obs_property_list_item_disable(method, 3, true);
obs_property_list_item_disable(method, 4, true);
}
#endif
#endif
return ppts;
}
struct obs_source_info noise_suppress_filter = {
.id = "noise_suppress_filter",
.type = OBS_SOURCE_TYPE_FILTER,
.output_flags = OBS_SOURCE_AUDIO | OBS_SOURCE_CAP_OBSOLETE,
.get_name = noise_suppress_name,
.create = noise_suppress_create,
.destroy = noise_suppress_destroy,
.update = noise_suppress_update,
.filter_audio = noise_suppress_filter_audio,
.get_defaults = noise_suppress_defaults_v1,
.get_properties = noise_suppress_properties,
};
struct obs_source_info noise_suppress_filter_v2 = {
.id = "noise_suppress_filter",
.version = 2,
.type = OBS_SOURCE_TYPE_FILTER,
.output_flags = OBS_SOURCE_AUDIO,
.get_name = noise_suppress_name,
.create = noise_suppress_create,
.destroy = noise_suppress_destroy,
.update = noise_suppress_update,
.filter_audio = noise_suppress_filter_audio,
.get_defaults = noise_suppress_defaults_v2,
.get_properties = noise_suppress_properties,
};
Reference in New Issue View Git Blame Copy Permalink