From 55f35d1d892fb15414405415c834fd1e5c0ae675 Mon Sep 17 00:00:00 2001 From: jpark37 Date: Sun, 31 Jul 2022 17:08:07 -0700 Subject: [PATCH 1/3] libobs: Consolidate H.264 priority scheme The spec for nal_ref_idc doesn't indicate that it should be used for priority like x264 seems to be using it for. NVENC seems to pass different values, so let's not rely on it. The new scheme might be naive, but we can iterate on it, and apply evenly to all H.264 encoders. --- libobs/obs-avc.c | 45 +++++++++++++++++++++------------------------ libobs/obs-avc.h | 9 +-------- libobs/obs-nal.h | 7 +++++++ 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/libobs/obs-avc.c b/libobs/obs-avc.c index cec0ee1d9..8b725fb53 100644 --- a/libobs/obs-avc.c +++ b/libobs/obs-avc.c @@ -51,19 +51,11 @@ const uint8_t *obs_avc_find_startcode(const uint8_t *p, const uint8_t *end) return obs_nal_find_startcode(p, end); } -static inline int get_drop_priority(int priority) -{ - return priority; -} - static void serialize_avc_data(struct serializer *s, const uint8_t *data, size_t size, bool *is_keyframe, int *priority) { - const uint8_t *nal_start, *nal_end; - const uint8_t *end = data + size; - int type; - - nal_start = obs_avc_find_startcode(data, end); + const uint8_t *const end = data + size; + const uint8_t *nal_start = obs_nal_find_startcode(data, end); while (true) { while (nal_start < end && !*(nal_start++)) ; @@ -71,18 +63,23 @@ static void serialize_avc_data(struct serializer *s, const uint8_t *data, if (nal_start == end) break; - type = nal_start[0] & 0x1F; + const int type = nal_start[0] & 0x1F; - if (type == OBS_NAL_SLICE_IDR || type == OBS_NAL_SLICE) { - if (is_keyframe) - *is_keyframe = (type == OBS_NAL_SLICE_IDR); - if (priority) - *priority = nal_start[0] >> 5; + switch (type) { + case OBS_NAL_SLICE: + if (*priority < OBS_NAL_PRIORITY_HIGH) + *priority = OBS_NAL_PRIORITY_HIGH; + break; + case OBS_NAL_SLICE_IDR: + *is_keyframe = true; + *priority = OBS_NAL_PRIORITY_HIGHEST; } - nal_end = obs_avc_find_startcode(nal_start, end); - s_wb32(s, (uint32_t)(nal_end - nal_start)); - s_write(s, nal_start, nal_end - nal_start); + const uint8_t *const nal_end = + obs_nal_find_startcode(nal_start, end); + const size_t size = nal_end - nal_start; + s_wb32(s, (uint32_t)size); + s_write(s, nal_start, size); nal_start = nal_end; } } @@ -103,7 +100,7 @@ void obs_parse_avc_packet(struct encoder_packet *avc_packet, avc_packet->data = output.bytes.array + sizeof(ref); avc_packet->size = output.bytes.num - sizeof(ref); - avc_packet->drop_priority = get_drop_priority(avc_packet->priority); + avc_packet->drop_priority = avc_packet->priority; } static inline bool has_start_code(const uint8_t *data) @@ -121,7 +118,7 @@ static void get_sps_pps(const uint8_t *data, size_t size, const uint8_t **sps, const uint8_t *end = data + size; int type; - nal_start = obs_avc_find_startcode(data, end); + nal_start = obs_nal_find_startcode(data, end); while (true) { while (nal_start < end && !*(nal_start++)) ; @@ -129,7 +126,7 @@ static void get_sps_pps(const uint8_t *data, size_t size, const uint8_t **sps, if (nal_start == end) break; - nal_end = obs_avc_find_startcode(nal_start, end); + nal_end = obs_nal_find_startcode(nal_start, end); type = nal_start[0] & 0x1F; if (type == OBS_NAL_SPS) { @@ -195,7 +192,7 @@ void obs_extract_avc_headers(const uint8_t *packet, size_t size, da_init(header); da_init(sei); - nal_start = obs_avc_find_startcode(packet, end); + nal_start = obs_nal_find_startcode(packet, end); nal_end = NULL; while (nal_end != end) { nal_codestart = nal_start; @@ -208,7 +205,7 @@ void obs_extract_avc_headers(const uint8_t *packet, size_t size, const uint8_t type = nal_start[0] & 0x1F; - nal_end = obs_avc_find_startcode(nal_start, end); + nal_end = obs_nal_find_startcode(nal_start, end); if (!nal_end) nal_end = end; diff --git a/libobs/obs-avc.h b/libobs/obs-avc.h index 8d042380a..e35e0e3f6 100644 --- a/libobs/obs-avc.h +++ b/libobs/obs-avc.h @@ -17,7 +17,7 @@ #pragma once -#include "util/c99defs.h" +#include "obs-nal.h" #ifdef __cplusplus extern "C" { @@ -39,13 +39,6 @@ enum { OBS_NAL_FILLER = 12, }; -enum { - OBS_NAL_PRIORITY_DISPOSABLE = 0, - OBS_NAL_PRIORITY_LOW = 1, - OBS_NAL_PRIORITY_HIGH = 2, - OBS_NAL_PRIORITY_HIGHEST = 3, -}; - /* Helpers for parsing AVC NAL units. */ EXPORT bool obs_avc_keyframe(const uint8_t *data, size_t size); diff --git a/libobs/obs-nal.h b/libobs/obs-nal.h index fece75d58..e9488c056 100644 --- a/libobs/obs-nal.h +++ b/libobs/obs-nal.h @@ -23,6 +23,13 @@ extern "C" { #endif +enum { + OBS_NAL_PRIORITY_DISPOSABLE = 0, + OBS_NAL_PRIORITY_LOW = 1, + OBS_NAL_PRIORITY_HIGH = 2, + OBS_NAL_PRIORITY_HIGHEST = 3, +}; + EXPORT const uint8_t *obs_nal_find_startcode(const uint8_t *p, const uint8_t *end); From 5361d936ce5a5e8a4fb03e4e8553b0da75cd90ff Mon Sep 17 00:00:00 2001 From: jpark37 Date: Sun, 31 Jul 2022 17:09:16 -0700 Subject: [PATCH 2/3] libobs: Implement obs_parse_hevc_packet --- libobs/obs-hevc.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++- libobs/obs-hevc.h | 4 ++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/libobs/obs-hevc.c b/libobs/obs-hevc.c index c7b5cb815..d7bd9ecb7 100644 --- a/libobs/obs-hevc.c +++ b/libobs/obs-hevc.c @@ -17,8 +17,9 @@ #include "obs-hevc.h" +#include "obs.h" #include "obs-nal.h" -#include "util/darray.h" +#include "util/array-serializer.h" enum { OBS_HEVC_NAL_TRAIL_N = 0, @@ -89,6 +90,63 @@ bool obs_hevc_keyframe(const uint8_t *data, size_t size) return false; } +static void serialize_hevc_data(struct serializer *s, const uint8_t *data, + size_t size, bool *is_keyframe, int *priority) +{ + const uint8_t *const end = data + size; + const uint8_t *nal_start = obs_nal_find_startcode(data, end); + while (true) { + while (nal_start < end && !*(nal_start++)) + ; + + if (nal_start == end) + break; + + // HEVC contains NAL unit specifier at [6..1] bits of + // the byte next to the startcode 0x000001 + const int type = (nal_start[0] & 0x7F) >> 1; + + // Mark IDR slices as key-frames and set them to highest + // priority if needed. Assume other slices are non-key + // frames and set their priority as high + if (type >= OBS_HEVC_NAL_BLA_W_LP && + type <= OBS_HEVC_NAL_RSV_IRAP_VCL23) { + *is_keyframe = 1; + *priority = OBS_NAL_PRIORITY_HIGHEST; + } else if (type >= OBS_HEVC_NAL_TRAIL_N && + type <= OBS_HEVC_NAL_RASL_R) { + if (*priority < OBS_NAL_PRIORITY_HIGH) + *priority = OBS_NAL_PRIORITY_HIGH; + } + + const uint8_t *const nal_end = + obs_nal_find_startcode(nal_start, end); + const size_t size = nal_end - nal_start; + s_wb32(s, (uint32_t)size); + s_write(s, nal_start, size); + nal_start = nal_end; + } +} + +void obs_parse_hevc_packet(struct encoder_packet *hevc_packet, + const struct encoder_packet *src) +{ + struct array_output_data output; + struct serializer s; + long ref = 1; + + array_output_serializer_init(&s, &output); + *hevc_packet = *src; + + serialize(&s, &ref, sizeof(ref)); + serialize_hevc_data(&s, src->data, src->size, &hevc_packet->keyframe, + &hevc_packet->priority); + + hevc_packet->data = output.bytes.array + sizeof(ref); + hevc_packet->size = output.bytes.num - sizeof(ref); + hevc_packet->drop_priority = hevc_packet->priority; +} + void obs_extract_hevc_headers(const uint8_t *packet, size_t size, uint8_t **new_packet_data, size_t *new_packet_size, uint8_t **header_data, diff --git a/libobs/obs-hevc.h b/libobs/obs-hevc.h index e68cffbfe..43adcfd59 100644 --- a/libobs/obs-hevc.h +++ b/libobs/obs-hevc.h @@ -23,7 +23,11 @@ extern "C" { #endif +struct encoder_packet; + EXPORT bool obs_hevc_keyframe(const uint8_t *data, size_t size); +EXPORT void obs_parse_hevc_packet(struct encoder_packet *hevc_packet, + const struct encoder_packet *src); EXPORT void obs_extract_hevc_headers(const uint8_t *packet, size_t size, uint8_t **new_packet_data, size_t *new_packet_size, From 2a1b047d33fb81affa765994fb93bdb8cd16b7e0 Mon Sep 17 00:00:00 2001 From: jpark37 Date: Sun, 31 Jul 2022 17:10:06 -0700 Subject: [PATCH 3/3] obs-ffmpeg: Implement priority for HEVC over HLS --- plugins/obs-ffmpeg/obs-ffmpeg-hls-mux.c | 21 ++++++++++++++++++--- plugins/obs-ffmpeg/obs-ffmpeg-mux.h | 1 - 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-hls-mux.c b/plugins/obs-ffmpeg/obs-ffmpeg-hls-mux.c index 4614afa27..c3dc98e50 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-hls-mux.c +++ b/plugins/obs-ffmpeg/obs-ffmpeg-hls-mux.c @@ -1,4 +1,8 @@ #include "obs-ffmpeg-mux.h" +#include +#ifdef ENABLE_HEVC +#include +#endif #define do_log(level, format, ...) \ blog(level, "[ffmpeg hls muxer: '%s'] " format, \ @@ -292,9 +296,20 @@ void ffmpeg_hls_mux_data(void *data, struct encoder_packet *packet) } if (packet->type == OBS_ENCODER_VIDEO) { - obs_parse_avc_packet(&tmp_packet, packet); - packet->drop_priority = tmp_packet.priority; - obs_encoder_packet_release(&tmp_packet); + const char *const codec = + obs_encoder_get_codec(packet->encoder); + if (strcmp(codec, "h264") == 0) { + obs_parse_avc_packet(&tmp_packet, packet); + packet->drop_priority = tmp_packet.priority; + obs_encoder_packet_release(&tmp_packet); + } +#ifdef ENABLE_HEVC + else if (strcmp(codec, "hevc") == 0) { + obs_parse_hevc_packet(&tmp_packet, packet); + packet->drop_priority = tmp_packet.priority; + obs_encoder_packet_release(&tmp_packet); + } +#endif } obs_encoder_packet_ref(&new_packet, packet); diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-mux.h b/plugins/obs-ffmpeg/obs-ffmpeg-mux.h index daf76b8dd..453a133ce 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-mux.h +++ b/plugins/obs-ffmpeg/obs-ffmpeg-mux.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include