jp9000
/
obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

obs-filters: Fix NVIDIA greenscreen issues 

Fix non-async sources, make sure math is linear-correct, preserve
incoming alpha, and allow the filter to work in previews.
master
jpark37 2022-08-07 02:32:39 -07:00 committed by Jim
parent 1272209710
commit 9c77796299
2 changed files with 147 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
plugins/obs-filters/data/rtx_greenscreen.effect
View File

@ -11,24 +11,27 @@ sampler_state texSampler {
};
struct VertData {
float2 uv : TEXCOORD0;
float4 pos : POSITION;
};
struct FragData {
float2 uv : TEXCOORD0;
};
VertData VSDefault(VertData v_in)
{
VertData v_out;
v_out.pos = mul(float4(v_in.pos.xyz, 1.0), ViewProj);
v_out.uv = v_in.uv;
v_out.pos = mul(float4(v_in.pos.xyz, 1.), ViewProj);
return v_out;
}
float4 PSMask(VertData v2_in) : TARGET
float4 PSMask(FragData f_in) : TARGET
{
float4 pix;
pix.rgb = image.Sample(texSampler, v2_in.uv).rgb;
pix.a = smoothstep(threshold - 0.1,threshold,mask.Sample(texSampler, v2_in.uv).a);
return pix;
float4 rgba = image.Sample(texSampler, f_in.uv);
rgba *= smoothstep(threshold - 0.1,threshold,mask.Sample(texSampler, f_in.uv).a);
return rgba;
}
technique Draw
@ -36,6 +39,6 @@ technique Draw
pass
{
vertex_shader = VSDefault(v_in);
pixel_shader = PSMask(v2_in);
pixel_shader = PSMask(f_in);
}
}

249
plugins/obs-filters/nvidia-greenscreen-filter.c
View File

@ -42,7 +42,7 @@ struct nv_greenscreen_data {
volatile bool processing_stop;
bool processed_frame;
bool target_valid;
volatile bool got_new_frame;
bool got_new_frame;
signal_handler_t *handler;
/* RTX SDK vars */
@ -58,14 +58,14 @@ struct nv_greenscreen_data {
/* alpha mask effect */
gs_effect_t *effect;
gs_texrender_t *render;
gs_texture_t *render_unorm;
gs_texture_t *alpha_texture;
uint32_t width; // width of texture
uint32_t height; // height of texture
gs_eparam_t *mask_param;
gs_texture_t *src_texture;
gs_eparam_t *src_param;
gs_eparam_t *threshold_param;
double threshold;
float threshold;
};
static const char *nv_greenscreen_filter_name(void *unused)
@ -86,8 +86,7 @@ static void nv_greenscreen_filter_update(void *data, obs_data_t *settings)
if (NVCV_SUCCESS != vfxErr)
error("Error loading AI Greenscreen FX %i", vfxErr);
}
filter->threshold =
(double)obs_data_get_double(settings, S_THRESHOLDFX);
filter->threshold = (float)obs_data_get_double(settings, S_THRESHOLDFX);
}
static void nv_greenscreen_filter_actual_destroy(void *data)
@ -104,6 +103,7 @@ static void nv_greenscreen_filter_actual_destroy(void *data)
obs_enter_graphics();
gs_texture_destroy(filter->alpha_texture);
gs_texrender_destroy(filter->render);
gs_texture_destroy(filter->render_unorm);
obs_leave_graphics();
NvCVImage_Destroy(filter->src_img);
NvCVImage_Destroy(filter->BGR_src_img);
@ -195,7 +195,6 @@ static void init_images_greenscreen(struct nv_greenscreen_data *filter)
uint32_t height = filter->height;
/* 1. create alpha texture */
obs_enter_graphics();
if (filter->alpha_texture) {
gs_texture_destroy(filter->alpha_texture);
}
@ -208,11 +207,11 @@ static void init_images_greenscreen(struct nv_greenscreen_data *filter)
struct ID3D11Texture2D *d11texture =
(struct ID3D11Texture2D *)gs_texture_get_obj(
filter->alpha_texture);
obs_leave_graphics();
/* 2. Create NvCVImage which will hold final alpha texture. */
if (NvCVImage_Create(width, height, NVCV_A, NVCV_U8, NVCV_CHUNKY,
NVCV_GPU, 1, &filter->dst_img) != NVCV_SUCCESS) {
if (!filter->dst_img &&
(NvCVImage_Create(width, height, NVCV_A, NVCV_U8, NVCV_CHUNKY,
NVCV_GPU, 1, &filter->dst_img) != NVCV_SUCCESS)) {
goto fail;
}
@ -225,46 +224,80 @@ static void init_images_greenscreen(struct nv_greenscreen_data *filter)
}
/* 3. create texrender */
obs_enter_graphics();
if (filter->render)
gs_texrender_destroy(filter->render);
filter->render = gs_texrender_create(GS_BGRA_UNORM, GS_ZS_NONE);
obs_leave_graphics();
filter->render = gs_texrender_create(GS_BGRA, GS_ZS_NONE);
if (!filter->render) {
error("Failed to create a texture renderer", vfxErr);
error("Failed to create render texrenderer", vfxErr);
goto fail;
}
if (filter->render_unorm)
gs_texture_destroy(filter->render_unorm);
filter->render_unorm =
gs_texture_create(width, height, GS_BGRA_UNORM, 1, NULL, 0);
if (!filter->render_unorm) {
error("Failed to create render_unorm texture", vfxErr);
goto fail;
}
/* 4. Create and allocate BGR NvCVImage (fx src). */
if (NvCVImage_Create(width, height, NVCV_BGR, NVCV_U8, NVCV_CHUNKY,
NVCV_GPU, 1,
&filter->BGR_src_img) != NVCV_SUCCESS) {
goto fail;
}
if (NvCVImage_Alloc(filter->BGR_src_img, width, height, NVCV_BGR,
NVCV_U8, NVCV_CHUNKY, NVCV_GPU,
1) != NVCV_SUCCESS) {
goto fail;
if (filter->BGR_src_img) {
if (NvCVImage_Realloc(filter->BGR_src_img, width, height,
NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU,
1) != NVCV_SUCCESS) {
goto fail;
}
} else {
if (NvCVImage_Create(width, height, NVCV_BGR, NVCV_U8,
NVCV_CHUNKY, NVCV_GPU, 1,
&filter->BGR_src_img) != NVCV_SUCCESS) {
goto fail;
}
if (NvCVImage_Alloc(filter->BGR_src_img, width, height,
NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU,
1) != NVCV_SUCCESS) {
goto fail;
}
}
/* 5. Create and allocate Alpha NvCVimage (fx dst). */
if (NvCVImage_Create(width, height, NVCV_A, NVCV_U8, NVCV_CHUNKY,
NVCV_GPU, 1, &filter->A_dst_img) != NVCV_SUCCESS) {
goto fail;
}
if (NvCVImage_Alloc(filter->A_dst_img, width, height, NVCV_A, NVCV_U8,
NVCV_CHUNKY, NVCV_GPU, 1) != NVCV_SUCCESS) {
goto fail;
if (filter->A_dst_img) {
if (NvCVImage_Realloc(filter->A_dst_img, width, height, NVCV_A,
NVCV_U8, NVCV_CHUNKY, NVCV_GPU,
1) != NVCV_SUCCESS) {
goto fail;
}
} else {
if (NvCVImage_Create(width, height, NVCV_A, NVCV_U8,
NVCV_CHUNKY, NVCV_GPU, 1,
&filter->A_dst_img) != NVCV_SUCCESS) {
goto fail;
}
if (NvCVImage_Alloc(filter->A_dst_img, width, height, NVCV_A,
NVCV_U8, NVCV_CHUNKY, NVCV_GPU,
1) != NVCV_SUCCESS) {
goto fail;
}
}
/* 6. Create stage NvCVImage which will be used as buffer for transfer */
if (NvCVImage_Create(width, height, NVCV_RGBA, NVCV_U8, NVCV_PLANAR,
NVCV_GPU, 1, &filter->stage) != NVCV_SUCCESS) {
goto fail;
}
if (NvCVImage_Alloc(filter->stage, width, height, NVCV_RGBA, NVCV_U8,
NVCV_PLANAR, NVCV_GPU, 1) != NVCV_SUCCESS) {
goto fail;
if (filter->stage) {
if (NvCVImage_Realloc(filter->stage, width, height, NVCV_RGBA,
NVCV_U8, NVCV_PLANAR, NVCV_GPU,
1) != NVCV_SUCCESS) {
goto fail;
}
} else {
if (NvCVImage_Create(width, height, NVCV_RGBA, NVCV_U8,
NVCV_PLANAR, NVCV_GPU, 1,
&filter->stage) != NVCV_SUCCESS) {
goto fail;
}
if (NvCVImage_Alloc(filter->stage, width, height, NVCV_RGBA,
NVCV_U8, NVCV_PLANAR, NVCV_GPU,
1) != NVCV_SUCCESS) {
goto fail;
}
}
/* 7. Set input & output images for nv FX. */
@ -287,11 +320,9 @@ fail:
static bool process_texture_greenscreen(struct nv_greenscreen_data *filter)
{
gs_texrender_t *render = filter->render;
NvCV_Status vfxErr;
/* 1. Map src img holding texture. */
vfxErr = NvCVImage_MapResource(filter->src_img, filter->stream);
NvCV_Status vfxErr =
NvCVImage_MapResource(filter->src_img, filter->stream);
if (vfxErr != NVCV_SUCCESS) {
const char *errString = NvCV_GetErrorStringFromCode(vfxErr);
error("Error mapping resource for source texture; error %i : %s",
@ -474,7 +505,7 @@ static struct obs_source_frame *
nv_greenscreen_filter_video(void *data, struct obs_source_frame *frame)
{
struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data;
os_atomic_set_bool(&filter->got_new_frame, true);
filter->got_new_frame = true;
return frame;
}
@ -491,13 +522,11 @@ static void nv_greenscreen_filter_tick(void *data, float t)
return;
}
obs_source_t *target = obs_filter_get_target(filter->context);
uint32_t cx;
uint32_t cy;
filter->target_valid = true;
cx = obs_source_get_base_width(target);
cy = obs_source_get_base_height(target);
const uint32_t cx = obs_source_get_base_width(target);
const uint32_t cy = obs_source_get_base_height(target);
// initially the sizes are 0
if (!cx && !cy) {
@ -517,8 +546,10 @@ static void nv_greenscreen_filter_tick(void *data, float t)
filter->height = cy;
}
if (!filter->images_allocated) {
obs_enter_graphics();
init_images_greenscreen(filter);
filter->initial_render = 0;
obs_leave_graphics();
filter->initial_render = false;
}
filter->processed_frame = false;
@ -527,63 +558,56 @@ static void nv_greenscreen_filter_tick(void *data, float t)
static void draw_greenscreen(struct nv_greenscreen_data *filter)
{
/* Render alpha mask */
if (!obs_source_process_filter_begin(filter->context, GS_BGRA_UNORM,
OBS_ALLOW_DIRECT_RENDERING)) {
return;
}
gs_effect_set_texture(filter->mask_param, filter->alpha_texture);
gs_effect_set_texture(filter->src_param, filter->src_texture);
gs_effect_set_float(filter->threshold_param, (float)filter->threshold);
while (gs_effect_loop(filter->effect, "Draw")) {
gs_draw_sprite(NULL, 0, filter->width, filter->height);
}
obs_source_process_filter_end(filter->context, filter->effect, 0, 0);
}
if (obs_source_process_filter_begin(filter->context, GS_RGBA,
OBS_ALLOW_DIRECT_RENDERING)) {
gs_effect_set_texture(filter->mask_param,
filter->alpha_texture);
gs_effect_set_texture_srgb(
filter->src_param,
gs_texrender_get_texture(filter->render));
gs_effect_set_float(filter->threshold_param, filter->threshold);
static void draw_greenscreen_srgb(struct nv_greenscreen_data *filter)
{
/* Render alpha mask */
if (!obs_source_process_filter_begin(filter->context, GS_BGRA_UNORM,
OBS_ALLOW_DIRECT_RENDERING)) {
return;
gs_blend_state_push();
gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
obs_source_process_filter_end(filter->context, filter->effect,
0, 0);
gs_blend_state_pop();
}
const bool previous = gs_framebuffer_srgb_enabled();
gs_enable_framebuffer_srgb(true);
gs_effect_set_texture_srgb(filter->mask_param, filter->alpha_texture);
gs_effect_set_texture_srgb(filter->src_param, filter->src_texture);
gs_effect_set_float(filter->threshold_param, (float)filter->threshold);
while (gs_effect_loop(filter->effect, "Draw")) {
gs_draw_sprite(NULL, 0, filter->width, filter->height);
}
gs_enable_framebuffer_srgb(previous);
obs_source_process_filter_end(filter->context, filter->effect, 0, 0);
}
static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect)
{
NvCV_Status vfxErr;
bool ret;
struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data;
if (filter->processing_stop)
return;
obs_source_t *target = obs_filter_get_target(filter->context);
obs_source_t *parent = obs_filter_get_parent(filter->context);
gs_texrender_t *render;
/* Skip if processing of a frame hasn't yet started */
if (!filter->target_valid || !target || !parent ||
filter->processed_frame) {
if (filter->processing_stop) {
obs_source_skip_video_filter(filter->context);
return;
}
obs_source_t *target = obs_filter_get_target(filter->context);
obs_source_t *parent = obs_filter_get_parent(filter->context);
/* Skip if processing of a frame hasn't yet started */
if (!filter->target_valid || !target || !parent) {
obs_source_skip_video_filter(filter->context);
return;
}
/* Render processed image from earlier in the frame */
if (filter->processed_frame) {
draw_greenscreen(filter);
return;
}
if (parent && !filter->handler) {
filter->handler = obs_source_get_signal_handler(parent);
signal_handler_connect(filter->handler, "update_properties",
nv_greenscreen_filter_reset, filter);
}
/* 1. Render to retrieve texture. */
render = filter->render;
gs_texrender_t *const render = filter->render;
if (!render) {
obs_source_skip_video_filter(filter->context);
return;
@ -593,7 +617,6 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect)
bool custom_draw = (target_flags & OBS_SOURCE_CUSTOM_DRAW) != 0;
bool async = (target_flags & OBS_SOURCE_ASYNC) != 0;
bool srgb_draw = (parent_flags & OBS_SOURCE_SRGB) != 0;
gs_texrender_reset(render);
gs_blend_state_push();
@ -612,33 +635,38 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect)
obs_source_video_render(target);
gs_texrender_end(render);
gs_copy_texture(filter->render_unorm,
gs_texrender_get_texture(filter->render));
}
gs_blend_state_pop();
/* 2. Initialize src_texture (only at startup or reset) */
if (!filter->initial_render) {
obs_enter_graphics();
filter->src_texture = gs_texrender_get_texture(filter->render);
struct ID3D11Texture2D *d11texture2 =
(struct ID3D11Texture2D *)gs_texture_get_obj(
filter->src_texture);
obs_leave_graphics();
filter->render_unorm);
if (!d11texture2) {
error("Couldn't retrieve d3d11texture2d.");
return;
}
vfxErr = NvCVImage_Create(filter->width, filter->height,
NVCV_BGRA, NVCV_U8, NVCV_CHUNKY,
NVCV_GPU, 1, &filter->src_img);
if (vfxErr != NVCV_SUCCESS) {
const char *errString =
NvCV_GetErrorStringFromCode(vfxErr);
error("Error creating src img; error %i: %s", vfxErr,
errString);
os_atomic_set_bool(&filter->processing_stop, true);
return;
if (!filter->src_img) {
vfxErr = NvCVImage_Create(filter->width, filter->height,
NVCV_BGRA, NVCV_U8,
NVCV_CHUNKY, NVCV_GPU, 1,
&filter->src_img);
if (vfxErr != NVCV_SUCCESS) {
const char *errString =
NvCV_GetErrorStringFromCode(vfxErr);
error("Error creating src img; error %i: %s",
vfxErr, errString);
os_atomic_set_bool(&filter->processing_stop,
true);
return;
}
}
vfxErr = NvCVImage_InitFromD3D11Texture(filter->src_img,
d11texture2);
if (vfxErr != NVCV_SUCCESS) {
@ -649,27 +677,24 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect)
os_atomic_set_bool(&filter->processing_stop, true);
return;
}
filter->initial_render = true;
}
/* 3. Process FX (outputs a mask) & draw. */
if (filter->initial_render && filter->images_allocated) {
ret = true;
if (filter->got_new_frame) {
ret = process_texture_greenscreen(filter);
os_atomic_set_bool(&filter->got_new_frame, false);
bool draw = true;
if (!async || filter->got_new_frame) {
draw = process_texture_greenscreen(filter);
filter->got_new_frame = false;
}
if (ret) {
if (!srgb_draw)
draw_greenscreen(filter);
else
draw_greenscreen_srgb(filter);
if (draw) {
draw_greenscreen(filter);
filter->processed_frame = true;
}
} else {
obs_source_skip_video_filter(filter->context);
return;
}
UNUSED_PARAMETER(effect);
}
@ -789,7 +814,7 @@ void unload_nvvfx(void)
struct obs_source_info nvidia_greenscreen_filter_info = {
.id = "nv_greenscreen_filter",
.type = OBS_SOURCE_TYPE_FILTER,
.output_flags = OBS_SOURCE_VIDEO,
.output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_SRGB,
.get_name = nv_greenscreen_filter_name,
.create = nv_greenscreen_filter_create,
.destroy = nv_greenscreen_filter_destroy,