From 37f663a7897fcb792d46056e71e94a154916f667 Mon Sep 17 00:00:00 2001 From: James Park Date: Mon, 17 Jun 2019 22:25:18 -0700 Subject: [PATCH 1/2] libobs: obs-ffmpeg: win-dshow: Planar 4:2:2 video This format has been seen when using FFmpeg MJPEG decompression. --- libobs/data/format_conversion.effect | 28 +++++++++++++++++++++++++ libobs/media-io/video-frame.c | 18 ++++++++++++++++ libobs/media-io/video-io.h | 3 +++ libobs/media-io/video-scaler-ffmpeg.c | 2 ++ libobs/obs-source.c | 25 ++++++++++++++++++++++ plugins/obs-ffmpeg/obs-ffmpeg-formats.h | 4 ++++ plugins/win-dshow/ffmpeg-decode.c | 3 +++ 7 files changed, 83 insertions(+) diff --git a/libobs/data/format_conversion.effect b/libobs/data/format_conversion.effect index c3aa18f52..f1d284144 100644 --- a/libobs/data/format_conversion.effect +++ b/libobs/data/format_conversion.effect @@ -363,6 +363,25 @@ float4 PSPlanar420_Reverse(FragTex frag_in) : TARGET return saturate(mul(float4(yuv, 1.0), color_matrix)); } +float4 PSPlanar422_Reverse(FragTex frag_in) : TARGET +{ + int x = int(frag_in.uv.x * width + PRECISION_OFFSET); + int y = int(frag_in.uv.y * height + PRECISION_OFFSET); + + int lum_offset = y * int_width + x; + int chroma_offset = y * (int_width / 2) + x / 2; + int chroma1 = int_u_plane_offset + chroma_offset; + int chroma2 = int_v_plane_offset + chroma_offset; + + float3 yuv = float3( + GetIntOffsetColor(lum_offset), + GetIntOffsetColor(chroma1), + GetIntOffsetColor(chroma2) + ); + yuv = clamp(yuv, color_range_min, color_range_max); + return saturate(mul(float4(yuv, 1.0), color_matrix)); +} + float4 PSPlanar444_Reverse(FragTex frag_in) : TARGET { int x = int(frag_in.uv.x * width + PRECISION_OFFSET); @@ -535,6 +554,15 @@ technique I420_Reverse } } +technique I422_Reverse +{ + pass + { + vertex_shader = VSPosTex(id); + pixel_shader = PSPlanar422_Reverse(frag_in); + } +} + technique I444_Reverse { pass diff --git a/libobs/media-io/video-frame.c b/libobs/media-io/video-frame.c index ce1bb498e..15cd68529 100644 --- a/libobs/media-io/video-frame.c +++ b/libobs/media-io/video-frame.c @@ -108,6 +108,23 @@ void video_frame_init(struct video_frame *frame, enum video_format format, frame->data[0] = bmalloc(size); frame->linesize[0] = width * 3; break; + + case VIDEO_FORMAT_I422: + size = width * height; + ALIGN_SIZE(size, alignment); + offsets[0] = size; + size += (width / 2) * height; + ALIGN_SIZE(size, alignment); + offsets[1] = size; + size += (width / 2) * height; + ALIGN_SIZE(size, alignment); + frame->data[0] = bmalloc(size); + frame->data[1] = (uint8_t *)frame->data[0] + offsets[0]; + frame->data[2] = (uint8_t *)frame->data[0] + offsets[1]; + frame->linesize[0] = width; + frame->linesize[1] = width / 2; + frame->linesize[2] = width / 2; + break; } } @@ -141,6 +158,7 @@ void video_frame_copy(struct video_frame *dst, const struct video_frame *src, break; case VIDEO_FORMAT_I444: + case VIDEO_FORMAT_I422: memcpy(dst->data[0], src->data[0], src->linesize[0] * cy); memcpy(dst->data[1], src->data[1], src->linesize[1] * cy); memcpy(dst->data[2], src->data[2], src->linesize[2] * cy); diff --git a/libobs/media-io/video-io.h b/libobs/media-io/video-io.h index a5d4312de..d64f05cfd 100644 --- a/libobs/media-io/video-io.h +++ b/libobs/media-io/video-io.h @@ -53,6 +53,9 @@ enum video_format { /* more packed uncompressed formats */ VIDEO_FORMAT_BGR3, + + /* planar 4:2:2 */ + VIDEO_FORMAT_I422, }; enum video_colorspace { diff --git a/libobs/media-io/video-scaler-ffmpeg.c b/libobs/media-io/video-scaler-ffmpeg.c index bec81ab91..e84772236 100644 --- a/libobs/media-io/video-scaler-ffmpeg.c +++ b/libobs/media-io/video-scaler-ffmpeg.c @@ -53,6 +53,8 @@ get_ffmpeg_video_format(enum video_format format) return AV_PIX_FMT_YUV444P; case VIDEO_FORMAT_BGR3: return AV_PIX_FMT_BGR24; + case VIDEO_FORMAT_I422: + return AV_PIX_FMT_YUV422P; } return AV_PIX_FMT_NONE; diff --git a/libobs/obs-source.c b/libobs/obs-source.c index 802393823..a9f05c575 100644 --- a/libobs/obs-source.c +++ b/libobs/obs-source.c @@ -1334,6 +1334,7 @@ enum convert_type { CONVERT_NONE, CONVERT_NV12, CONVERT_420, + CONVERT_422, CONVERT_422_U, CONVERT_422_Y, CONVERT_444, @@ -1352,6 +1353,8 @@ static inline enum convert_type get_convert_type(enum video_format format, return CONVERT_NV12; case VIDEO_FORMAT_I444: return CONVERT_444; + case VIDEO_FORMAT_I422: + return CONVERT_422; case VIDEO_FORMAT_YVYU: case VIDEO_FORMAT_YUY2: @@ -1409,6 +1412,20 @@ static inline bool set_planar420_sizes(struct obs_source *source, return true; } +static inline bool set_planar422_sizes(struct obs_source *source, + const struct obs_source_frame *frame) +{ + uint32_t size = frame->width * frame->height; + size *= 2; + + source->async_convert_width = frame->width; + source->async_convert_height = size / frame->width; + source->async_texture_format = GS_R8; + source->async_plane_offset[0] = (int)(frame->data[1] - frame->data[0]); + source->async_plane_offset[1] = (int)(frame->data[2] - frame->data[0]); + return true; +} + static inline bool set_nv12_sizes(struct obs_source *source, const struct obs_source_frame *frame) { @@ -1460,6 +1477,9 @@ static inline bool init_gpu_conversion(struct obs_source *source, case CONVERT_420: return set_planar420_sizes(source, frame); + case CONVERT_422: + return set_planar422_sizes(source, frame); + case CONVERT_NV12: return set_nv12_sizes(source, frame); @@ -1557,6 +1577,7 @@ static void upload_raw_frame(gs_texture_t *tex, break; case CONVERT_420: + case CONVERT_422: case CONVERT_NV12: case CONVERT_444: gs_texture_set_image(tex, frame->data[0], frame->width, false); @@ -1596,6 +1617,9 @@ static const char *select_conversion_technique(enum video_format format, case VIDEO_FORMAT_BGR3: return full_range ? "BGR3_Full" : "BGR3_Limited"; + case VIDEO_FORMAT_I422: + return "I422_Reverse"; + case VIDEO_FORMAT_BGRA: case VIDEO_FORMAT_BGRX: case VIDEO_FORMAT_RGBA: @@ -2316,6 +2340,7 @@ static void copy_frame_data(struct obs_source_frame *dst, break; case VIDEO_FORMAT_I444: + case VIDEO_FORMAT_I422: copy_frame_data_plane(dst, src, 0, dst->height); copy_frame_data_plane(dst, src, 1, dst->height); copy_frame_data_plane(dst, src, 2, dst->height); diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-formats.h b/plugins/obs-ffmpeg/obs-ffmpeg-formats.h index 98cf6534b..a01edf9ec 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-formats.h +++ b/plugins/obs-ffmpeg/obs-ffmpeg-formats.h @@ -35,6 +35,8 @@ obs_to_ffmpeg_video_format(enum video_format format) return AV_PIX_FMT_GRAY8; case VIDEO_FORMAT_BGR3: return AV_PIX_FMT_BGR24; + case VIDEO_FORMAT_I422: + return AV_PIX_FMT_YUV422P; } return AV_PIX_FMT_NONE; @@ -62,6 +64,8 @@ ffmpeg_to_obs_video_format(enum AVPixelFormat format) return VIDEO_FORMAT_Y800; case AV_PIX_FMT_BGR24: return VIDEO_FORMAT_BGR3; + case AV_PIX_FMT_YUV422P: + return VIDEO_FORMAT_I422; case AV_PIX_FMT_NONE: default: return VIDEO_FORMAT_NONE; diff --git a/plugins/win-dshow/ffmpeg-decode.c b/plugins/win-dshow/ffmpeg-decode.c index b5830fcf7..66419abdc 100644 --- a/plugins/win-dshow/ffmpeg-decode.c +++ b/plugins/win-dshow/ffmpeg-decode.c @@ -74,6 +74,9 @@ static inline enum video_format convert_pixel_format(int f) return VIDEO_FORMAT_YUY2; case AV_PIX_FMT_UYVY422: return VIDEO_FORMAT_UYVY; + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUVJ422P: + return VIDEO_FORMAT_I422; case AV_PIX_FMT_RGBA: return VIDEO_FORMAT_RGBA; case AV_PIX_FMT_BGRA: From 0e7eec8f2d8d918d3dac08213f2ddc7bc2602d84 Mon Sep 17 00:00:00 2001 From: James Park Date: Mon, 17 Jun 2019 22:25:56 -0700 Subject: [PATCH 2/2] win-dshow: Use FFmpeg for MJPEG decompression Measured lower CPU usage and latency than default DirectShow filter. Tested with Logitech BRIO camera at 4K, 30 FPS. --- plugins/win-dshow/win-dshow.cpp | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/plugins/win-dshow/win-dshow.cpp b/plugins/win-dshow/win-dshow.cpp index 48ddc71fc..e4c25a273 100644 --- a/plugins/win-dshow/win-dshow.cpp +++ b/plugins/win-dshow/win-dshow.cpp @@ -418,8 +418,6 @@ static inline video_format ConvertVideoFormat(VideoFormat format) return VIDEO_FORMAT_UYVY; case VideoFormat::HDYC: return VIDEO_FORMAT_UYVY; - case VideoFormat::MJPEG: - return VIDEO_FORMAT_YUY2; default: return VIDEO_FORMAT_NONE; } @@ -502,6 +500,11 @@ void DShowInput::OnVideoData(const VideoConfig &config, unsigned char *data, return; } + if (videoConfig.format == VideoFormat::MJPEG) { + OnEncodedVideoData(AV_CODEC_ID_MJPEG, data, size, startTime); + return; + } + const int cx = config.cx; const int cy = config.cy; @@ -905,8 +908,7 @@ bool DShowInput::UpdateVideoConfig(obs_data_t *settings) placeholders::_3, placeholders::_4, placeholders::_5); - if (videoConfig.internalFormat != VideoFormat::MJPEG) - videoConfig.format = videoConfig.internalFormat; + videoConfig.format = videoConfig.internalFormat; if (!device.SetVideoConfig(&videoConfig)) { blog(LOG_WARNING, "%s: device.SetVideoConfig failed", @@ -914,19 +916,6 @@ bool DShowInput::UpdateVideoConfig(obs_data_t *settings) return false; } - if (videoConfig.internalFormat == VideoFormat::MJPEG) { - videoConfig.format = VideoFormat::XRGB; - videoConfig.useDefaultConfig = false; - - if (!device.SetVideoConfig(&videoConfig)) { - blog(LOG_WARNING, - "%s: device.SetVideoConfig (XRGB) " - "failed", - obs_source_get_name(source)); - return false; - } - } - DStr formatName = GetVideoFormatName(videoConfig.internalFormat); double fps = 0.0;