From 7d811499e0b9f2e81857604eabea30e66ab40778 Mon Sep 17 00:00:00 2001 From: James Park Date: Wed, 6 Mar 2019 20:53:15 -0800 Subject: [PATCH] Add "Area" scale filter This new scale filter computes pixels by weighing the coverage area of source pixels over the target pixel. This algorithm works well for both upsampling and downsampling, but was mainly designed to upscale high-quality low-resolution sources like RGB/HDMI retro consoles. I've heard of people using odd workarounds like scaling up to very high resolutions before scaling back down to preserve pixel shartpness. This algorithm directly addresses this use-case in a much more direct fashion. The Area scale filter does a better job of preserving the thickness of thin features than the Point filter. The Area scale filter does not look at source pixels that lie outside of the target pixel, leading to a much sharper image than Bilinear, Bicubic, and Lanczos filters. This filter should interpolate pixels in linear space, but OBS is not equipped to do that at the moment. libobs: Add GPU effect, and wire up scene serialization. obs-filters: Add Area as an option for scale_filter. UI: Add Area as an option for both scene items, and canvas downscaling. --- UI/data/locale/en-US.ini | 2 + UI/window-basic-main.cpp | 3 + UI/window-basic-settings.cpp | 5 + libobs/data/area.effect | 119 ++++++++++++++++++++++ libobs/obs-internal.h | 1 + libobs/obs-scene.c | 11 ++ libobs/obs.c | 11 ++ libobs/obs.h | 4 +- plugins/obs-filters/data/locale/en-US.ini | 1 + plugins/obs-filters/scale-filter.c | 13 ++- 10 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 libobs/data/area.effect diff --git a/UI/data/locale/en-US.ini b/UI/data/locale/en-US.ini index d1b3fdecf..a7ac36c4a 100644 --- a/UI/data/locale/en-US.ini +++ b/UI/data/locale/en-US.ini @@ -337,6 +337,7 @@ ScaleFiltering.Point="Point" ScaleFiltering.Bilinear="Bilinear" ScaleFiltering.Bicubic="Bicubic" ScaleFiltering.Lanczos="Lanczos" +ScaleFiltering.Area="Area" # deinterlacing Deinterlacing="Deinterlacing" @@ -739,6 +740,7 @@ Basic.Settings.Video.DisableAero="Disable Aero" Basic.Settings.Video.DownscaleFilter.Bilinear="Bilinear (Fastest, but blurry if scaling)" Basic.Settings.Video.DownscaleFilter.Bicubic="Bicubic (Sharpened scaling, 16 samples)" Basic.Settings.Video.DownscaleFilter.Lanczos="Lanczos (Sharpened scaling, 32 samples)" +Basic.Settings.Video.DownscaleFilter.Area="Area" # basic mode 'audio' settings Basic.Settings.Audio="Audio" diff --git a/UI/window-basic-main.cpp b/UI/window-basic-main.cpp index 1ab536f66..0adf0833d 100644 --- a/UI/window-basic-main.cpp +++ b/UI/window-basic-main.cpp @@ -3408,6 +3408,8 @@ static inline enum obs_scale_type GetScaleType(ConfigFile &basicConfig) return OBS_SCALE_BILINEAR; else if (astrcmpi(scaleTypeStr, "lanczos") == 0) return OBS_SCALE_LANCZOS; + else if (astrcmpi(scaleTypeStr, "area") == 0) + return OBS_SCALE_AREA; else return OBS_SCALE_BICUBIC; } @@ -4175,6 +4177,7 @@ QMenu *OBSBasic::AddScaleFilteringMenu(QMenu *menu, obs_sceneitem_t *item) ADD_MODE("ScaleFiltering.Bilinear", OBS_SCALE_BILINEAR); ADD_MODE("ScaleFiltering.Bicubic", OBS_SCALE_BICUBIC); ADD_MODE("ScaleFiltering.Lanczos", OBS_SCALE_LANCZOS); + ADD_MODE("ScaleFiltering.Area", OBS_SCALE_AREA); #undef ADD_MODE return menu; diff --git a/UI/window-basic-settings.cpp b/UI/window-basic-settings.cpp index 7293ef441..c8e6b5625 100644 --- a/UI/window-basic-settings.cpp +++ b/UI/window-basic-settings.cpp @@ -1308,6 +1308,9 @@ void OBSBasicSettings::LoadDownscaleFilters() ui->downscaleFilter->addItem( QTStr("Basic.Settings.Video.DownscaleFilter.Lanczos"), QT_UTF8("lanczos")); + ui->downscaleFilter->addItem( + QTStr("Basic.Settings.Video.DownscaleFilter.Area"), + QT_UTF8("area")); const char *scaleType = config_get_string(main->Config(), "Video", "ScaleType"); @@ -1316,6 +1319,8 @@ void OBSBasicSettings::LoadDownscaleFilters() ui->downscaleFilter->setCurrentIndex(0); else if (astrcmpi(scaleType, "lanczos") == 0) ui->downscaleFilter->setCurrentIndex(2); + else if (astrcmpi(scaleType, "area") == 0) + ui->downscaleFilter->setCurrentIndex(3); else ui->downscaleFilter->setCurrentIndex(1); } diff --git a/libobs/data/area.effect b/libobs/data/area.effect new file mode 100644 index 000000000..50a197828 --- /dev/null +++ b/libobs/data/area.effect @@ -0,0 +1,119 @@ +uniform float4x4 ViewProj; +uniform float4x4 color_matrix; +uniform float3 color_range_min = {0.0, 0.0, 0.0}; +uniform float3 color_range_max = {1.0, 1.0, 1.0}; +uniform float2 base_dimension_i; +uniform texture2d image; + +sampler_state def_sampler { + Filter = Linear; + AddressU = Clamp; + AddressV = Clamp; +}; + +struct VertInOut { + float4 pos : POSITION; + float2 uv : TEXCOORD0; +}; + +VertInOut VSDefault(VertInOut vert_in) +{ + VertInOut vert_out; + vert_out.pos = mul(float4(vert_in.pos.xyz, 1.0), ViewProj); + vert_out.uv = vert_in.uv; + return vert_out; +} + +float4 PSDrawAreaRGBA(VertInOut vert_in) : TARGET +{ + float4 totalcolor = float4(0.0, 0.0, 0.0, 0.0); + + const float2 uv = vert_in.uv; + const float2 uvdelta = float2(ddx(uv.x), ddy(uv.y)); + + const float2 uvhalfdelta = 0.5 * uvdelta; + const float2 uvmin = uv - uvhalfdelta; + const float2 uvmax = uv + uvhalfdelta; + + const int2 loadindexmin = int2(uvmin / base_dimension_i); + const int2 loadindexmax = int2(uvmax / base_dimension_i); + + const float2 targetpos = uv / uvdelta; + const float2 targetposleft = targetpos - 0.5; + const float2 targetposright = targetpos + 0.5; + for (int loadindexy = loadindexmin.y; loadindexy <= loadindexmax.y; ++loadindexy) + { + for (int loadindexx = loadindexmin.x; loadindexx <= loadindexmax.x; ++loadindexx) + { + const float2 loadindex = float2(loadindexx, loadindexy); + const float2 potentialtargetmin = loadindex / uvdelta * base_dimension_i; + const float2 potentialtargetmax = (loadindex + 1.0) / uvdelta * base_dimension_i; + const float2 targetmin = max(potentialtargetmin, targetposleft); + const float2 targetmax = min(potentialtargetmax, targetposright); + const float area = (targetmax.x - targetmin.x) * (targetmax.y - targetmin.y); + const float4 sample = image.SampleLevel(def_sampler, (loadindex + 0.5) * base_dimension_i, 0.0); + totalcolor += area * float4(sample.rgb * sample.a, sample.a); + } + } + + return float4(totalcolor.rgb / totalcolor.a, totalcolor.a); +} + +float3 ConvertFromYuv(float3 yuv) +{ + yuv = clamp(yuv, color_range_min, color_range_max); + return saturate(mul(float4(yuv, 1.0), color_matrix)).rgb; +} + +float4 PSDrawAreaMatrix(VertInOut vert_in) : TARGET +{ + float3 totalcolor = float3(0.0, 0.0, 0.0); + + const float2 uv = vert_in.uv; + const float2 uvdelta = float2(ddx(uv.x), ddy(uv.y)); + + const float2 uvhalfdelta = 0.5 * uvdelta; + const float2 uvmin = uv - uvhalfdelta; + const float2 uvmax = uv + uvhalfdelta; + + const int2 loadindexmin = int2(uvmin / base_dimension_i); + const int2 loadindexmax = int2(uvmax / base_dimension_i); + + const float2 targetpos = uv / uvdelta; + const float2 targetposleft = targetpos - 0.5; + const float2 targetposright = targetpos + 0.5; + for (int loadindexy = loadindexmin.y; loadindexy <= loadindexmax.y; ++loadindexy) + { + for (int loadindexx = loadindexmin.x; loadindexx <= loadindexmax.x; ++loadindexx) + { + const float2 loadindex = float2(loadindexx, loadindexy); + const float2 potentialtargetmin = loadindex / uvdelta * base_dimension_i; + const float2 potentialtargetmax = (loadindex + 1.0) / uvdelta * base_dimension_i; + const float2 targetmin = max(potentialtargetmin, targetposleft); + const float2 targetmax = min(potentialtargetmax, targetposright); + const float area = (targetmax.x - targetmin.x) * (targetmax.y - targetmin.y); + const float3 yuv = image.SampleLevel(def_sampler, (loadindex + 0.5) * base_dimension_i, 0.0).xyz; + totalcolor += area * ConvertFromYuv(yuv); + } + } + + return float4(totalcolor, 1.0); +} + +technique Draw +{ + pass + { + vertex_shader = VSDefault(vert_in); + pixel_shader = PSDrawAreaRGBA(vert_in); + } +} + +technique DrawMatrix +{ + pass + { + vertex_shader = VSDefault(vert_in); + pixel_shader = PSDrawAreaMatrix(vert_in); + } +} diff --git a/libobs/obs-internal.h b/libobs/obs-internal.h index cc7083f27..576427e4b 100644 --- a/libobs/obs-internal.h +++ b/libobs/obs-internal.h @@ -259,6 +259,7 @@ struct obs_core_video { gs_effect_t *conversion_effect; gs_effect_t *bicubic_effect; gs_effect_t *lanczos_effect; + gs_effect_t *area_effect; gs_effect_t *bilinear_lowres_effect; gs_effect_t *premultiplied_alpha_effect; gs_samplerstate_t *point_sampler; diff --git a/libobs/obs-scene.c b/libobs/obs-scene.c index a3ba2d806..d6bae7558 100644 --- a/libobs/obs-scene.c +++ b/libobs/obs-scene.c @@ -486,6 +486,13 @@ static void render_item_texture(struct obs_scene_item *item) effect = obs->video.bicubic_effect; } else if (type == OBS_SCALE_LANCZOS) { effect = obs->video.lanczos_effect; + } else if (type == OBS_SCALE_AREA) { + effect = obs->video.area_effect; + + gs_eparam_t *image = gs_effect_get_param_by_name( + effect, "image"); + gs_effect_set_next_sampler(image, + obs->video.point_sampler); } scale_param = gs_effect_get_param_by_name(effect, @@ -748,6 +755,8 @@ static void scene_load_item(struct obs_scene *scene, obs_data_t *item_data) item->scale_filter = OBS_SCALE_BICUBIC; else if (astrcmpi(scale_filter_str, "lanczos") == 0) item->scale_filter = OBS_SCALE_LANCZOS; + else if (astrcmpi(scale_filter_str, "area") == 0) + item->scale_filter = OBS_SCALE_AREA; } if (item->item_render && !item_texture_enabled(item)) { @@ -857,6 +866,8 @@ static void scene_save_item(obs_data_array_t *array, scale_filter = "bicubic"; else if (item->scale_filter == OBS_SCALE_LANCZOS) scale_filter = "lanczos"; + else if (item->scale_filter == OBS_SCALE_AREA) + scale_filter = "area"; else scale_filter = "disable"; diff --git a/libobs/obs.c b/libobs/obs.c index 7a8ff28a4..c0f503055 100644 --- a/libobs/obs.c +++ b/libobs/obs.c @@ -329,6 +329,11 @@ static int obs_init_graphics(struct obs_video_info *ovi) NULL); bfree(filename); + filename = obs_find_data_file("area.effect"); + video->area_effect = gs_effect_create_from_file(filename, + NULL); + bfree(filename); + filename = obs_find_data_file("bilinear_lowres_scale.effect"); video->bilinear_lowres_effect = gs_effect_create_from_file(filename, NULL); @@ -533,6 +538,7 @@ static void obs_free_graphics(void) gs_effect_destroy(video->bicubic_effect); gs_effect_destroy(video->repeat_effect); gs_effect_destroy(video->lanczos_effect); + gs_effect_destroy(video->area_effect); gs_effect_destroy(video->bilinear_lowres_effect); video->default_effect = NULL; @@ -1115,6 +1121,9 @@ int obs_reset_video(struct obs_video_info *ovi) case OBS_SCALE_LANCZOS: scale_type_name = "Lanczos"; break; + case OBS_SCALE_AREA: + scale_type_name = "Area"; + break; } bool yuv = format_is_yuv(ovi->output_format); @@ -1582,6 +1591,8 @@ gs_effect_t *obs_get_base_effect(enum obs_base_effect effect) return obs->video.bicubic_effect; case OBS_EFFECT_LANCZOS: return obs->video.lanczos_effect; + case OBS_EFFECT_AREA: + return obs->video.area_effect; case OBS_EFFECT_BILINEAR_LOWRES: return obs->video.bilinear_lowres_effect; case OBS_EFFECT_PREMULTIPLIED_ALPHA: diff --git a/libobs/obs.h b/libobs/obs.h index 759e253f8..8d80cd518 100644 --- a/libobs/obs.h +++ b/libobs/obs.h @@ -116,7 +116,8 @@ enum obs_scale_type { OBS_SCALE_POINT, OBS_SCALE_BICUBIC, OBS_SCALE_BILINEAR, - OBS_SCALE_LANCZOS + OBS_SCALE_LANCZOS, + OBS_SCALE_AREA, }; /** @@ -599,6 +600,7 @@ enum obs_base_effect { OBS_EFFECT_SOLID, /**< RGB/YUV (solid color only) */ OBS_EFFECT_BICUBIC, /**< Bicubic downscale */ OBS_EFFECT_LANCZOS, /**< Lanczos downscale */ + OBS_EFFECT_AREA, /**< Area rescale */ OBS_EFFECT_BILINEAR_LOWRES, /**< Bilinear low resolution downscale */ OBS_EFFECT_PREMULTIPLIED_ALPHA,/**< Premultiplied alpha */ OBS_EFFECT_REPEAT, /**< RGB/YUV (repeating) */ diff --git a/plugins/obs-filters/data/locale/en-US.ini b/plugins/obs-filters/data/locale/en-US.ini index b806f9ddc..1d598eea5 100644 --- a/plugins/obs-filters/data/locale/en-US.ini +++ b/plugins/obs-filters/data/locale/en-US.ini @@ -65,6 +65,7 @@ ScaleFiltering.Point="Point" ScaleFiltering.Bilinear="Bilinear" ScaleFiltering.Bicubic="Bicubic" ScaleFiltering.Lanczos="Lanczos" +ScaleFiltering.Area="Area" NoiseSuppress.SuppressLevel="Suppression Level (dB)" Saturation="Saturation" HueShift="Hue Shift" diff --git a/plugins/obs-filters/scale-filter.c b/plugins/obs-filters/scale-filter.c index b0d66d4fc..c4c402239 100644 --- a/plugins/obs-filters/scale-filter.c +++ b/plugins/obs-filters/scale-filter.c @@ -17,6 +17,7 @@ #define T_SAMPLING_BILINEAR obs_module_text("ScaleFiltering.Bilinear") #define T_SAMPLING_BICUBIC obs_module_text("ScaleFiltering.Bicubic") #define T_SAMPLING_LANCZOS obs_module_text("ScaleFiltering.Lanczos") +#define T_SAMPLING_AREA obs_module_text("ScaleFiltering.Area") #define T_UNDISTORT obs_module_text("UndistortCenter") #define T_BASE obs_module_text("Base.Canvas") @@ -24,6 +25,7 @@ #define S_SAMPLING_BILINEAR "bilinear" #define S_SAMPLING_BICUBIC "bicubic" #define S_SAMPLING_LANCZOS "lanczos" +#define S_SAMPLING_AREA "area" struct scale_filter_data { obs_source_t *context; @@ -95,6 +97,9 @@ static void scale_filter_update(void *data, obs_data_t *settings) } else if (astrcmpi(sampling, S_SAMPLING_LANCZOS) == 0) { filter->sampling = OBS_SCALE_LANCZOS; + } else if (astrcmpi(sampling, S_SAMPLING_AREA) == 0) { + filter->sampling = OBS_SCALE_AREA; + } else { /* S_SAMPLING_BICUBIC */ filter->sampling = OBS_SCALE_BICUBIC; } @@ -218,6 +223,7 @@ static void scale_filter_tick(void *data, float seconds) case OBS_SCALE_BILINEAR: type = OBS_EFFECT_DEFAULT; break; case OBS_SCALE_BICUBIC: type = OBS_EFFECT_BICUBIC; break; case OBS_SCALE_LANCZOS: type = OBS_EFFECT_LANCZOS; break; + case OBS_SCALE_AREA: type = OBS_EFFECT_AREA; break; } } @@ -309,15 +315,15 @@ static bool sampling_modified(obs_properties_t *props, obs_property_t *p, bool has_undistort; if (astrcmpi(sampling, S_SAMPLING_POINT) == 0) { has_undistort = false; - } else if (astrcmpi(sampling, S_SAMPLING_BILINEAR) == 0) { has_undistort = false; - } else if (astrcmpi(sampling, S_SAMPLING_LANCZOS) == 0) { has_undistort = true; - + } + else if (astrcmpi(sampling, S_SAMPLING_AREA) == 0) { + has_undistort = false; } else { /* S_SAMPLING_BICUBIC */ has_undistort = true; @@ -360,6 +366,7 @@ static obs_properties_t *scale_filter_properties(void *data) obs_property_list_add_string(p, T_SAMPLING_BILINEAR, S_SAMPLING_BILINEAR); obs_property_list_add_string(p, T_SAMPLING_BICUBIC, S_SAMPLING_BICUBIC); obs_property_list_add_string(p, T_SAMPLING_LANCZOS, S_SAMPLING_LANCZOS); + obs_property_list_add_string(p, T_SAMPLING_AREA, S_SAMPLING_AREA); /* ----------------- */