decklink: Add ability to ingest/embed cea 708 captions

(This commit also modifies libobs, UI)

UI/frontend-plugins/CMakeLists.txt

@@ -1,2 +1,3 @@
 add_subdirectory(decklink-output-ui)
 add_subdirectory(frontend-tools)
+add_subdirectory(decklink-captions)

UI/frontend-plugins/decklink-captions/CMakeLists.txt

@@ -0,0 +1,43 @@
+project(decklink-captions)
+
+if(APPLE)
+    find_library(COCOA Cocoa)
+    include_directories(${COCOA})
+endif()
+
+if(UNIX AND NOT APPLE)
+    find_package(X11 REQUIRED)
+    link_libraries(${X11_LIBRARIES})
+    include_directories(${X11_INCLUDE_DIR})
+endif()
+
+set(decklink-captions_HEADERS
+        decklink-captions.h
+        )
+set(decklink-captions_SOURCES
+        decklink-captions.cpp
+        )
+set(decklink-captions_UI
+        forms/captions.ui
+        )
+
+if(APPLE)
+    set(decklink-captions_PLATFORM_LIBS
+            ${COCOA})
+endif()
+
+qt5_wrap_ui(decklink-captions_UI_HEADERS
+        ${decklink-captions_UI})
+
+add_library(decklink-captions MODULE
+        ${decklink-captions_HEADERS}
+        ${decklink-captions_SOURCES}
+        ${decklink-captions_UI_HEADERS}
+        )
+target_link_libraries(decklink-captions
+        ${frontend-tools_PLATFORM_LIBS}
+        obs-frontend-api
+        Qt5::Widgets
+        libobs)
+
+install_obs_plugin_with_data(decklink-captions data)

UI/frontend-plugins/decklink-captions/decklink-captions.cpp

@@ -0,0 +1,157 @@
+#include <obs-frontend-api.h>
+#include <QMainWindow>
+#include <QAction>
+#include <obs.hpp>
+#include "decklink-captions.h"
+
+OBS_DECLARE_MODULE()
+OBS_MODULE_USE_DEFAULT_LOCALE("decklink-captons", "en-US")
+
+struct obs_captions {
+	std::string source_name;
+	OBSWeakSource source;
+
+	void start();
+	void stop();
+
+	obs_captions();
+	inline ~obs_captions() { stop(); }
+};
+
+obs_captions::obs_captions() {}
+
+static obs_captions *captions = nullptr;
+
+DecklinkCaptionsUI::DecklinkCaptionsUI(QWidget *parent)
+	: QDialog(parent), ui(new Ui_CaptionsDialog)
+{
+	ui->setupUi(this);
+
+	setSizeGripEnabled(true);
+
+	setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint);
+
+	auto cb = [this](obs_source_t *source) {
+		uint32_t caps = obs_source_get_output_flags(source);
+		QString name = obs_source_get_name(source);
+
+		if (caps & OBS_SOURCE_CEA_708)
+			ui->source->addItem(name);
+
+		OBSWeakSource weak = OBSGetWeakRef(source);
+		if (weak == captions->source)
+			ui->source->setCurrentText(name);
+		return true;
+	};
+
+	using cb_t = decltype(cb);
+
+	ui->source->blockSignals(true);
+	ui->source->addItem(QStringLiteral(""));
+	ui->source->setCurrentIndex(0);
+	obs_enum_sources(
+		[](void *data, obs_source_t *source) {
+			return (*static_cast<cb_t *>(data))(source);
+		},
+		&cb);
+	ui->source->blockSignals(false);
+}
+
+void DecklinkCaptionsUI::on_source_currentIndexChanged(int)
+{
+	captions->stop();
+
+	captions->source_name = ui->source->currentText().toUtf8().constData();
+	captions->source = GetWeakSourceByName(captions->source_name.c_str());
+
+	captions->start();
+}
+
+static void caption_callback(void *param, obs_source_t *source,
+			     const struct obs_source_cea_708 *captions)
+{
+	obs_output *output = obs_frontend_get_streaming_output();
+	if (output) {
+		if (obs_frontend_streaming_active() &&
+		    obs_output_active(output)) {
+			obs_output_caption(output, captions);
+		}
+		obs_output_release(output);
+	}
+}
+
+void obs_captions::start()
+{
+	OBSSource s = OBSGetStrongRef(source);
+	if (!s) {
+		//warn("Source invalid");
+		return;
+	}
+	obs_source_add_caption_callback(s, caption_callback, nullptr);
+}
+
+void obs_captions::stop()
+{
+	OBSSource s = OBSGetStrongRef(source);
+	if (s)
+		obs_source_remove_caption_callback(s, caption_callback,
+						   nullptr);
+}
+
+static void save_decklink_caption_data(obs_data_t *save_data, bool saving,
+				       void *)
+{
+	if (saving) {
+		obs_data_t *obj = obs_data_create();
+
+		obs_data_set_string(obj, "source",
+				    captions->source_name.c_str());
+
+		obs_data_set_obj(save_data, "decklink_captions", obj);
+		obs_data_release(obj);
+	} else {
+		captions->stop();
+
+		obs_data_t *obj =
+			obs_data_get_obj(save_data, "decklink_captions");
+		if (!obj)
+			obj = obs_data_create();
+
+		captions->source_name = obs_data_get_string(obj, "source");
+		captions->source =
+			GetWeakSourceByName(captions->source_name.c_str());
+		obs_data_release(obj);
+
+		captions->start();
+	}
+}
+
+void addOutputUI(void)
+{
+	QAction *action = (QAction *)obs_frontend_add_tools_menu_qaction(
+		obs_module_text("Decklink Captions"));
+
+	captions = new obs_captions;
+
+	auto cb = []() {
+		obs_frontend_push_ui_translation(obs_module_get_string);
+
+		QWidget *window = (QWidget *)obs_frontend_get_main_window();
+
+		DecklinkCaptionsUI dialog(window);
+		dialog.exec();
+
+		obs_frontend_pop_ui_translation();
+	};
+
+	obs_frontend_add_save_callback(save_decklink_caption_data, nullptr);
+
+	action->connect(action, &QAction::triggered, cb);
+}
+
+bool obs_module_load(void)
+{
+	addOutputUI();
+
+	return true;
+}

UI/frontend-plugins/decklink-captions/decklink-captions.h

@@ -0,0 +1,30 @@
+#include <QDialog>
+#include <obs-module.h>
+#include <util/platform.h>
+#include <obs.hpp>
+#include <memory>
+#include "ui_captions.h"
+
+class DecklinkCaptionsUI : public QDialog {
+	Q_OBJECT
+private:
+public:
+	std::unique_ptr<Ui_CaptionsDialog> ui;
+	DecklinkCaptionsUI(QWidget *parent);
+
+public slots:
+	void on_source_currentIndexChanged(int idx);
+};
+
+static inline OBSWeakSource GetWeakSourceByName(const char *name)
+{
+	OBSWeakSource weak;
+	obs_source_t *source = obs_get_source_by_name(name);
+	if (source) {
+		weak = obs_source_get_weak_source(source);
+		obs_weak_source_release(weak);
+		obs_source_release(source);
+	}
+
+	return weak;
+}

UI/frontend-plugins/decklink-captions/forms/captions.ui

@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>CaptionsDialog</class>
+ <widget class="QDialog" name="CaptionsDialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>519</width>
+    <height>104</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Captions</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <layout class="QFormLayout" name="formLayout">
+     <property name="labelAlignment">
+      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+     </property>
+     <item row="0" column="0">
+      <widget class="QLabel" name="label">
+       <property name="text">
+        <string>Captions.Source</string>
+       </property>
+       <property name="buddy">
+        <cstring>source</cstring>
+       </property>
+      </widget>
+     </item>
+     <item row="0" column="1">
+      <widget class="QComboBox" name="source">
+       <property name="insertPolicy">
+        <enum>QComboBox::InsertAlphabetically</enum>
+       </property>
+       <property name="sizeAdjustPolicy">
+        <enum>QComboBox::AdjustToContents</enum>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>0</width>
+       <height>0</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+   <item>
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <spacer name="horizontalSpacer">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>40</width>
+         <height>20</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QPushButton" name="accept">
+       <property name="text">
+        <string>OK</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <spacer name="horizontalSpacer_2">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>40</width>
+         <height>20</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>accept</sender>
+   <signal>clicked()</signal>
+   <receiver>CaptionsDialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>268</x>
+     <y>331</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>229</x>
+     <y>-11</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

libobs/CMakeLists.txt

@@ -365,7 +365,8 @@ set(libobs_util_SOURCES
 	util/crc32.c
 	util/text-lookup.c
 	util/cf-parser.c
-	util/profiler.c)
+	util/profiler.c
+	util/bitstream.c)
 set(libobs_util_HEADERS
 	util/curl/curl-helper.h
 	util/sse-intrin.h
@@ -392,7 +393,8 @@ set(libobs_util_HEADERS
 	util/lexer.h
 	util/platform.h
 	util/profiler.h
-	util/profiler.hpp)
+	util/profiler.hpp
+	util/bitstream.h)
 
 set(libobs_libobs_SOURCES
 	${libobs_PLATFORM_SOURCES}

libobs/obs-internal.h

@@ -36,6 +36,8 @@
 
 #include "obs.h"
 
+#include <caption/caption.h>
+
 #define NUM_TEXTURES 2
 #define NUM_CHANNELS 3
 #define MICROSECOND_DEN 1000000
@@ -587,6 +589,11 @@ struct audio_cb_info {
 	void *param;
 };
 
+struct caption_cb_info {
+	obs_source_caption_t callback;
+	void *param;
+};
+
 struct obs_source {
 	struct obs_context_data context;
 	struct obs_source_info info;
@@ -690,6 +697,9 @@ struct obs_source {
 	uint32_t async_convert_width[MAX_AV_PLANES];
 	uint32_t async_convert_height[MAX_AV_PLANES];
 
+	pthread_mutex_t caption_cb_mutex;
+	DARRAY(struct caption_cb_info) caption_cb_list;
+
 	/* async video deinterlacing */
 	uint64_t deinterlace_offset;
 	uint64_t deinterlace_frame_ts;
@@ -977,6 +987,8 @@ struct obs_output {
 	struct caption_text *caption_head;
 	struct caption_text *caption_tail;
 
+	struct circlebuf caption_data;
+
 	bool valid;
 
 	uint64_t active_delay_ns;

libobs/obs-output.c

@@ -227,6 +227,7 @@ void obs_output_destroy(obs_output_t *output)
 		os_event_destroy(output->reconnect_stop_event);
 		obs_context_data_free(&output->context);
 		circlebuf_free(&output->delay_data);
+		circlebuf_free(&output->caption_data);
 		if (output->owns_info_id)
 			bfree((void *)output->info.id);
 		if (output->last_error_message)
@@ -267,6 +268,10 @@ bool obs_output_actual_start(obs_output_t *output)
 		os_atomic_dec_long(&output->delay_restart_refs);
 
 	output->caption_timestamp = 0;
+
+	circlebuf_free(&output->caption_data);
+	circlebuf_init(&output->caption_data);
+
 	return success;
 }
 
@@ -1207,7 +1212,6 @@ static const uint8_t nal_start[4] = {0, 0, 0, 1};
 static bool add_caption(struct obs_output *output, struct encoder_packet *out)
 {
 	struct encoder_packet backup = *out;
-	caption_frame_t cf;
 	sei_t sei;
 	uint8_t *data;
 	size_t size;
@@ -1224,11 +1228,63 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out)
 	da_push_back_array(out_data, &ref, sizeof(ref));
 	da_push_back_array(out_data, out->data, out->size);
 
+	if (output->caption_data.size > 0) {
+
+		cea708_t cea708;
+		cea708_init(&cea708, 0); // set up a new popon frame
+		void *caption_buf = bzalloc(3 * sizeof(uint8_t));
+
+		while (output->caption_data.size > 0) {
+			circlebuf_pop_front(&output->caption_data, caption_buf,
+					    3 * sizeof(uint8_t));
+
+			if ((((uint8_t *)caption_buf)[0] & 0x3) != 0) {
+				// only send cea 608
+				continue;
+			}
+
+			uint16_t captionData = ((uint8_t *)caption_buf)[1];
+			captionData = captionData << 8;
+			captionData += ((uint8_t *)caption_buf)[2];
+
+			// padding
+			if (captionData == 0x8080) {
+				continue;
+			}
+
+			if (captionData == 0) {
+				continue;
+			}
+
+			if (!eia608_parity_varify(captionData)) {
+				continue;
+			}
+
+			cea708_add_cc_data(&cea708, 1,
+					   ((uint8_t *)caption_buf)[0] & 0x3,
+					   captionData);
+		}
+
+		bfree(caption_buf);
+
+		sei_message_t *msg =
+			sei_message_new(sei_type_user_data_registered_itu_t_t35,
+					0, CEA608_MAX_SIZE);
+		msg->size = cea708_render(&cea708, sei_message_data(msg),
+					  sei_message_size(msg));
+		sei_message_append(&sei, msg);
+	} else if (output->caption_head) {
+		caption_frame_t cf;
 		caption_frame_init(&cf);
 		caption_frame_from_text(&cf, &output->caption_head->text[0]);
 
 		sei_from_caption_frame(&sei, &cf);
 
+		struct obs_caption_frame *next = output->caption_head->next;
+		bfree(output->caption_head);
+		output->caption_head = next;
+	}
+
 	data = malloc(sei_render_size(&sei));
 	size = sei_render(&sei, data);
 	/* TODO SEI should come after AUD/SPS/PPS, but before any VCL */
@@ -1244,13 +1300,12 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out)
 
 	sei_free(&sei);
 
-	struct caption_text *next = output->caption_head->next;
-	bfree(output->caption_head);
-	output->caption_head = next;
 	return true;
 }
 #endif
 
+double last_caption_timestamp = 0;
+
 static inline void send_interleaved(struct obs_output *output)
 {
 	struct encoder_packet out = output->interleaved_packets.array[0];
@@ -1286,6 +1341,13 @@ static inline void send_interleaved(struct obs_output *output)
 			}
 		}
 
+		if (output->caption_data.size > 0) {
+			if (last_caption_timestamp < frame_timestamp) {
+				last_caption_timestamp = frame_timestamp;
+				add_caption(output, &out);
+			}
+		}
+
 		pthread_mutex_unlock(&output->caption_mutex);
 #endif
 	}
@@ -2471,6 +2533,18 @@ const char *obs_output_get_id(const obs_output_t *output)
 							     : NULL;
 }
 
+void obs_output_caption(obs_output_t *output,
+			const struct obs_source_cea_708 *captions)
+{
+	pthread_mutex_lock(&output->caption_mutex);
+	for (int i = 0; i < captions->packets; i++) {
+		circlebuf_push_back(&output->caption_data,
+				    captions->data + (i * 3),
+				    3 * sizeof(uint8_t));
+	}
+	pthread_mutex_unlock(&output->caption_mutex);
+}
+
 #if BUILD_CAPTIONS
 static struct caption_text *caption_text_new(const char *text, size_t bytes,
 					     struct caption_text *tail,

libobs/obs-source.c

@@ -184,6 +184,7 @@ static bool obs_source_init(struct obs_source *source)
 	pthread_mutex_init_value(&source->audio_mutex);
 	pthread_mutex_init_value(&source->audio_buf_mutex);
 	pthread_mutex_init_value(&source->audio_cb_mutex);
+	pthread_mutex_init_value(&source->caption_cb_mutex);
 
 	if (pthread_mutexattr_init(&attr) != 0)
 		return false;
@@ -201,6 +202,8 @@ static bool obs_source_init(struct obs_source *source)
 		return false;
 	if (pthread_mutex_init(&source->async_mutex, NULL) != 0)
 		return false;
+	if (pthread_mutex_init(&source->caption_cb_mutex, NULL) != 0)
+		return false;
 
 	if (is_audio_source(source) || is_composite_source(source))
 		allocate_audio_output_buffer(source);
@@ -683,6 +686,7 @@ void obs_source_destroy(struct obs_source *source)
 
 	da_free(source->audio_actions);
 	da_free(source->audio_cb_list);
+	da_free(source->caption_cb_list);
 	da_free(source->async_cache);
 	da_free(source->async_frames);
 	da_free(source->filters);
@@ -691,6 +695,7 @@ void obs_source_destroy(struct obs_source *source)
 	pthread_mutex_destroy(&source->audio_buf_mutex);
 	pthread_mutex_destroy(&source->audio_cb_mutex);
 	pthread_mutex_destroy(&source->audio_mutex);
+	pthread_mutex_destroy(&source->caption_cb_mutex);
 	pthread_mutex_destroy(&source->async_mutex);
 	obs_data_release(source->private_settings);
 	obs_context_data_free(&source->context);
@@ -2898,6 +2903,51 @@ void obs_source_set_async_rotation(obs_source_t *source, long rotation)
 		source->async_rotation = rotation;
 }
 
+void obs_source_output_cea708(obs_source_t *source,
+			      const struct obs_source_cea_708 *captions)
+{
+	if (!captions) {
+		return;
+	}
+
+	pthread_mutex_lock(&source->caption_cb_mutex);
+
+	for (size_t i = source->caption_cb_list.num; i > 0; i--) {
+		struct caption_cb_info info =
+			source->caption_cb_list.array[i - 1];
+		info.callback(info.param, source, captions);
+	}
+
+	pthread_mutex_unlock(&source->caption_cb_mutex);
+}
+
+void obs_source_add_caption_callback(obs_source_t *source,
+				     obs_source_caption_t callback, void *param)
+{
+	struct caption_cb_info info = {callback, param};
+
+	if (!obs_source_valid(source, "obs_source_add_caption_callback"))
+		return;
+
+	pthread_mutex_lock(&source->caption_cb_mutex);
+	da_push_back(source->caption_cb_list, &info);
+	pthread_mutex_unlock(&source->caption_cb_mutex);
+}
+
+void obs_source_remove_caption_callback(obs_source_t *source,
+					obs_source_caption_t callback,
+					void *param)
+{
+	struct caption_cb_info info = {callback, param};
+
+	if (!obs_source_valid(source, "obs_source_remove_caption_callback"))
+		return;
+
+	pthread_mutex_lock(&source->caption_cb_mutex);
+	da_erase_item(source->caption_cb_list, &info);
+	pthread_mutex_unlock(&source->caption_cb_mutex);
+}
+
 static inline bool preload_frame_changed(obs_source_t *source,
 					 const struct obs_source_frame *in)
 {

libobs/obs-source.h

@@ -186,6 +186,11 @@ enum obs_media_state {
  */
 #define OBS_SOURCE_CONTROLLABLE_MEDIA (1 << 13)
 
+/**
+ * Source type provides cea708 data
+ */
+#define OBS_SOURCE_CEA_708 (1 << 14)
+
 /** @} */
 
 typedef void (*obs_source_enum_proc_t)(obs_source_t *parent,

libobs/obs.h

@@ -212,6 +212,12 @@ struct obs_source_audio {
 	uint64_t timestamp;
 };
 
+struct obs_source_cea_708 {
+	const uint8_t *data;
+	uint32_t packets;
+	uint64_t timestamp;
+};
+
 /**
  * Source asynchronous video output structure.  Used with
  * obs_source_output_video to output asynchronous video.  Video is buffered as
@@ -1117,6 +1123,16 @@ EXPORT void obs_source_add_audio_capture_callback(
 EXPORT void obs_source_remove_audio_capture_callback(
 	obs_source_t *source, obs_source_audio_capture_t callback, void *param);
 
+typedef void (*obs_source_caption_t)(void *param, obs_source_t *source,
+				     const struct obs_source_cea_708 *captions);
+
+EXPORT void obs_source_add_caption_callback(obs_source_t *source,
+					    obs_source_caption_t callback,
+					    void *param);
+EXPORT void obs_source_remove_caption_callback(obs_source_t *source,
+					       obs_source_caption_t callback,
+					       void *param);
+
 enum obs_deinterlace_mode {
 	OBS_DEINTERLACE_MODE_DISABLE,
 	OBS_DEINTERLACE_MODE_DISCARD,
@@ -1208,6 +1224,9 @@ EXPORT void obs_source_output_video2(obs_source_t *source,
 
 EXPORT void obs_source_set_async_rotation(obs_source_t *source, long rotation);
 
+EXPORT void obs_source_output_cea708(obs_source_t *source,
+				     const struct obs_source_cea_708 *captions);
+
 /**
  * Preloads asynchronous video data to allow instantaneous playback
  *
@@ -1884,12 +1903,16 @@ EXPORT uint32_t obs_output_get_height(const obs_output_t *output);
 
 EXPORT const char *obs_output_get_id(const obs_output_t *output);
 
+EXPORT void obs_output_caption(obs_output_t *output,
+			       const struct obs_source_cea_708 *captions);
+
 #if BUILD_CAPTIONS
 EXPORT void obs_output_output_caption_text1(obs_output_t *output,
 					    const char *text);
 EXPORT void obs_output_output_caption_text2(obs_output_t *output,
 					    const char *text,
 					    double display_duration);
+
 #endif
 
 EXPORT float obs_output_get_congestion(obs_output_t *output);

libobs/util/bitstream.c

@@ -0,0 +1,52 @@
+#include "bitstream.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+void bitstream_reader_init(struct bitstream_reader *r, uint8_t *data,
+			   size_t len)
+{
+	memset(r, 0, sizeof(struct bitstream_reader));
+	r->buf = data;
+	r->subPos = 0x80;
+	r->len = len;
+}
+
+uint8_t bitstream_reader_read_bit(struct bitstream_reader *r)
+{
+	if (r->pos >= r->len)
+		return 0;
+
+	uint8_t bit = (*(r->buf + r->pos) & r->subPos) == r->subPos ? 1 : 0;
+
+	r->subPos >>= 0x1;
+	if (r->subPos == 0) {
+		r->subPos = 0x80;
+		r->pos++;
+	}
+
+	return bit;
+}
+
+uint8_t bitstream_reader_read_bits(struct bitstream_reader *r, int bits)
+{
+	uint8_t res = 0;
+
+	for (int i = 1; i <= bits; i++) {
+		res <<= 1;
+		res |= bitstream_reader_read_bit(r);
+	}
+
+	return res;
+}
+
+uint8_t bitstream_reader_r8(struct bitstream_reader *r)
+{
+	return bitstream_reader_read_bits(r, 8);
+}
+
+uint16_t bitstream_reader_r16(struct bitstream_reader *r)
+{
+	uint8_t b = bitstream_reader_read_bits(r, 8);
+	return ((uint16_t)b << 8) | bitstream_reader_read_bits(r, 8);
+}

libobs/util/bitstream.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "c99defs.h"
+
+/*
+ *   General programmable serialization functions.  (A shared interface to
+ * various reading/writing to/from different inputs/outputs)
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct bitstream_reader {
+	uint8_t pos;
+	uint8_t subPos;
+	uint8_t *buf;
+	size_t len;
+};
+
+EXPORT void bitstream_reader_init(struct bitstream_reader *r, uint8_t *data,
+				  size_t len);
+EXPORT uint8_t bitstream_reader_read_bits(struct bitstream_reader *r, int bits);
+EXPORT uint8_t bitstream_reader_r8(struct bitstream_reader *r);
+EXPORT uint16_t bitstream_reader_r16(struct bitstream_reader *r);
+
+#ifdef __cplusplus
+}
+#endif

plugins/decklink/OBSVideoFrame.cpp

@@ -0,0 +1,72 @@
+#include "OBSVideoFrame.h"
+
+OBSVideoFrame::OBSVideoFrame(long width, long height)
+{
+	this->width = width;
+	this->height = height;
+	this->rowBytes = width * 2;
+	this->data = new unsigned char[width * height * 2 + 1];
+}
+
+HRESULT OBSVideoFrame::SetFlags(BMDFrameFlags newFlags)
+{
+	flags = newFlags;
+	return S_OK;
+}
+
+HRESULT OBSVideoFrame::SetTimecode(BMDTimecodeFormat format,
+				   IDeckLinkTimecode *timecode)
+{
+	return 0;
+}
+
+HRESULT
+OBSVideoFrame::SetTimecodeFromComponents(BMDTimecodeFormat format,
+					 uint8_t hours, uint8_t minutes,
+					 uint8_t seconds, uint8_t frames,
+					 BMDTimecodeFlags flags)
+{
+	return 0;
+}
+
+HRESULT OBSVideoFrame::SetAncillaryData(IDeckLinkVideoFrameAncillary *ancillary)
+{
+	return 0;
+}
+
+HRESULT OBSVideoFrame::SetTimecodeUserBits(BMDTimecodeFormat format,
+					   BMDTimecodeUserBits userBits)
+{
+	return 0;
+}
+
+long OBSVideoFrame::GetWidth()
+{
+	return width;
+}
+
+long OBSVideoFrame::GetHeight()
+{
+	return height;
+}
+
+long OBSVideoFrame::GetRowBytes()
+{
+	return rowBytes;
+}
+
+BMDPixelFormat OBSVideoFrame::GetPixelFormat()
+{
+	return pixelFormat;
+}
+
+BMDFrameFlags OBSVideoFrame::GetFlags()
+{
+	return flags;
+}
+
+HRESULT OBSVideoFrame::GetBytes(void **buffer)
+{
+	*buffer = this->data;
+	return S_OK;
+}

plugins/decklink/OBSVideoFrame.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include "platform.hpp"
+
+class OBSVideoFrame : public IDeckLinkMutableVideoFrame {
+private:
+	BMDFrameFlags flags;
+	BMDPixelFormat pixelFormat = bmdFormat8BitYUV;
+
+	long width;
+	long height;
+	long rowBytes;
+
+	unsigned char *data;
+
+public:
+	OBSVideoFrame(long width, long height);
+
+	HRESULT STDMETHODCALLTYPE SetFlags(BMDFrameFlags newFlags) override;
+
+	HRESULT STDMETHODCALLTYPE SetTimecode(
+		BMDTimecodeFormat format, IDeckLinkTimecode *timecode) override;
+
+	HRESULT STDMETHODCALLTYPE SetTimecodeFromComponents(
+		BMDTimecodeFormat format, uint8_t hours, uint8_t minutes,
+		uint8_t seconds, uint8_t frames,
+		BMDTimecodeFlags flags) override;
+
+	HRESULT
+	STDMETHODCALLTYPE
+	SetAncillaryData(IDeckLinkVideoFrameAncillary *ancillary) override;
+
+	HRESULT STDMETHODCALLTYPE
+	SetTimecodeUserBits(BMDTimecodeFormat format,
+			    BMDTimecodeUserBits userBits) override;
+
+	long STDMETHODCALLTYPE GetWidth() override;
+
+	long STDMETHODCALLTYPE GetHeight() override;
+
+	long STDMETHODCALLTYPE GetRowBytes() override;
+
+	BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() override;
+
+	BMDFrameFlags STDMETHODCALLTYPE GetFlags() override;
+
+	HRESULT STDMETHODCALLTYPE GetBytes(void **buffer) override;
+
+	//Dummy implementations of remaining virtual methods
+	virtual HRESULT STDMETHODCALLTYPE
+	GetTimecode(/* in */ BMDTimecodeFormat format,
+		    /* out */ IDeckLinkTimecode **timecode)
+	{
+		return E_NOINTERFACE;
+	};
+	virtual HRESULT STDMETHODCALLTYPE
+	GetAncillaryData(/* out */ IDeckLinkVideoFrameAncillary **ancillary)
+	{
+		return E_NOINTERFACE;
+	};
+
+	// IUnknown interface (dummy implementation)
+	virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid,
+							 LPVOID *ppv)
+	{
+		return E_NOINTERFACE;
+	}
+	virtual ULONG STDMETHODCALLTYPE AddRef() { return 1; }
+	virtual ULONG STDMETHODCALLTYPE Release() { return 1; }
+};

plugins/decklink/decklink-device-instance.cpp

@@ -9,8 +9,14 @@
 #include <util/util_uint64.h>
 
 #include <sstream>
+#include <iomanip>
 #include <algorithm>
 
+#include "OBSVideoFrame.h"
+
+#include <caption/caption.h>
+#include <util/bitstream.h>
+
 static inline enum video_format ConvertPixelFormat(BMDPixelFormat format)
 {
 	switch (format) {
@@ -62,14 +68,23 @@ static inline audio_repack_mode_t ConvertRepackFormat(speaker_layout format,
 
 DeckLinkDeviceInstance::DeckLinkDeviceInstance(DecklinkBase *decklink_,
 					       DeckLinkDevice *device_)
-	: currentFrame(), currentPacket(), decklink(decklink_), device(device_)
+	: currentFrame(),
+	  currentPacket(),
+	  currentCaptions(),
+	  decklink(decklink_),
+	  device(device_)
 {
 	currentPacket.samples_per_sec = 48000;
 	currentPacket.speakers = SPEAKERS_STEREO;
 	currentPacket.format = AUDIO_FORMAT_16BIT;
 }
 
-DeckLinkDeviceInstance::~DeckLinkDeviceInstance() {}
+DeckLinkDeviceInstance::~DeckLinkDeviceInstance()
+{
+	if (convertFrame) {
+		delete convertFrame;
+	}
+}
 
 void DeckLinkDeviceInstance::HandleAudioPacket(
 	IDeckLinkAudioInputPacket *audioPacket, const uint64_t timestamp)
@@ -127,16 +142,47 @@ void DeckLinkDeviceInstance::HandleVideoFrame(
 	if (videoFrame == nullptr)
 		return;
 
+	IDeckLinkVideoFrameAncillaryPackets *packets;
+
+	if (videoFrame->QueryInterface(IID_IDeckLinkVideoFrameAncillaryPackets,
+				       (void **)&packets) == S_OK) {
+		IDeckLinkAncillaryPacketIterator *iterator;
+		packets->GetPacketIterator(&iterator);
+
+		IDeckLinkAncillaryPacket *packet;
+		iterator->Next(&packet);
+
+		if (packet) {
+			auto did = packet->GetDID();
+			auto sdid = packet->GetSDID();
+
+			// Caption data
+			if (did == 0x61 & sdid == 0x01) {
+				this->HandleCaptionPacket(packet, timestamp);
+			}
+
+			packet->Release();
+		}
+
+		iterator->Release();
+		packets->Release();
+	}
+
+	IDeckLinkVideoConversion *frameConverter =
+		CreateVideoConversionInstance();
+
+	frameConverter->ConvertFrame(videoFrame, convertFrame);
+
 	void *bytes;
-	if (videoFrame->GetBytes(&bytes) != S_OK) {
+	if (convertFrame->GetBytes(&bytes) != S_OK) {
 		LOG(LOG_WARNING, "Failed to get video frame data");
 		return;
 	}
 
 	currentFrame.data[0] = (uint8_t *)bytes;
-	currentFrame.linesize[0] = (uint32_t)videoFrame->GetRowBytes();
+	currentFrame.linesize[0] = (uint32_t)convertFrame->GetRowBytes();
-	currentFrame.width = (uint32_t)videoFrame->GetWidth();
+	currentFrame.width = (uint32_t)convertFrame->GetWidth();
-	currentFrame.height = (uint32_t)videoFrame->GetHeight();
+	currentFrame.height = (uint32_t)convertFrame->GetHeight();
 	currentFrame.timestamp = timestamp;
 
 	obs_source_output_video2(
@@ -144,6 +190,86 @@ void DeckLinkDeviceInstance::HandleVideoFrame(
 		&currentFrame);
 }
 
+void DeckLinkDeviceInstance::HandleCaptionPacket(
+	IDeckLinkAncillaryPacket *packet, const uint64_t timestamp)
+{
+	auto line = packet->GetLineNumber();
+
+	const void *data;
+	uint32_t size;
+	packet->GetBytes(bmdAncillaryPacketFormatUInt8, &data, &size);
+
+	auto anc = (uint8_t *)data;
+	struct bitstream_reader reader;
+	bitstream_reader_init(&reader, anc, size);
+
+	auto header1 = bitstream_reader_r8(&reader);
+	auto header2 = bitstream_reader_r8(&reader);
+
+	uint8_t length = bitstream_reader_r8(&reader);
+	uint8_t frameRate = bitstream_reader_read_bits(&reader, 4);
+	//reserved
+	bitstream_reader_read_bits(&reader, 4);
+
+	auto cdp_timecode_added = bitstream_reader_read_bits(&reader, 1);
+	auto cdp_data_block_added = bitstream_reader_read_bits(&reader, 1);
+	auto cdp_service_info_added = bitstream_reader_read_bits(&reader, 1);
+	auto cdp_service_info_start = bitstream_reader_read_bits(&reader, 1);
+	auto cdp_service_info_changed = bitstream_reader_read_bits(&reader, 1);
+	auto cdp_service_info_end = bitstream_reader_read_bits(&reader, 1);
+	auto cdp_contains_captions = bitstream_reader_read_bits(&reader, 1);
+	//reserved
+	bitstream_reader_read_bits(&reader, 1);
+
+	auto cdp_counter = bitstream_reader_r8(&reader);
+	auto cdp_counter2 = bitstream_reader_r8(&reader);
+
+	if (cdp_timecode_added) {
+		auto timecodeSectionID = bitstream_reader_r8(&reader);
+		//reserved
+		bitstream_reader_read_bits(&reader, 2);
+		bitstream_reader_read_bits(&reader, 2);
+		bitstream_reader_read_bits(&reader, 4);
+		// reserved
+		bitstream_reader_read_bits(&reader, 1);
+		bitstream_reader_read_bits(&reader, 3);
+		bitstream_reader_read_bits(&reader, 4);
+		bitstream_reader_read_bits(&reader, 1);
+		bitstream_reader_read_bits(&reader, 3);
+		bitstream_reader_read_bits(&reader, 4);
+		bitstream_reader_read_bits(&reader, 1);
+		bitstream_reader_read_bits(&reader, 1);
+		bitstream_reader_read_bits(&reader, 3);
+		bitstream_reader_read_bits(&reader, 4);
+	}
+
+	if (cdp_contains_captions) {
+		auto cdp_data_section = bitstream_reader_r8(&reader);
+
+		auto process_em_data_flag =
+			bitstream_reader_read_bits(&reader, 1);
+		auto process_cc_data_flag =
+			bitstream_reader_read_bits(&reader, 1);
+		auto additional_data_flag =
+			bitstream_reader_read_bits(&reader, 1);
+
+		auto cc_count = bitstream_reader_read_bits(&reader, 5);
+
+		auto *outData =
+			(uint8_t *)bzalloc(sizeof(uint8_t) * cc_count * 3);
+		memcpy(outData, anc + reader.pos, cc_count * 3);
+
+		currentCaptions.data = outData;
+		currentCaptions.timestamp = timestamp;
+		currentCaptions.packets = cc_count;
+
+		obs_source_output_cea708(
+			static_cast<DeckLinkInput *>(decklink)->GetSource(),
+			&currentCaptions);
+		bfree(outData);
+	}
+}
+
 void DeckLinkDeviceInstance::FinalizeStream()
 {
 	input->SetCallback(nullptr);
@@ -189,6 +315,11 @@ void DeckLinkDeviceInstance::SetupVideoFormat(DeckLinkDeviceMode *mode_)
 				    currentFrame.color_range_min,
 				    currentFrame.color_range_max);
 
+	if (convertFrame) {
+		delete convertFrame;
+	}
+	convertFrame = new OBSVideoFrame(mode_->GetWidth(), mode_->GetHeight());
+
 #ifdef LOG_SETUP_VIDEO_FORMAT
 	LOG(LOG_INFO, "Setup video format: %s, %s, %s",
 	    pixelFormat == bmdFormat8BitYUV ? "YUV" : "RGB",
@@ -250,7 +381,7 @@ bool DeckLinkDeviceInstance::StartCapture(DeckLinkDeviceMode *mode_,
 	bool isauto = mode_->GetName() == "Auto";
 	if (isauto) {
 		displayMode = bmdModeNTSC;
-		pixelFormat = bmdFormat8BitYUV;
+		pixelFormat = bmdFormat10BitYUV;
 		flags = bmdVideoInputEnableFormatDetection;
 	} else {
 		displayMode = mode_->GetDisplayMode();
@@ -503,7 +634,7 @@ HRESULT STDMETHODCALLTYPE DeckLinkDeviceInstance::VideoInputFormatChanged(
 
 		default:
 		case bmdDetectedVideoInputYCbCr422:
-			pixelFormat = bmdFormat8BitYUV;
+			pixelFormat = bmdFormat10BitYUV;
 			break;
 		}
 	}

plugins/decklink/decklink-device-instance.hpp

@@ -6,6 +6,7 @@
 #include <obs-module.h>
 #include "decklink-device.hpp"
 #include "../../libobs/media-io/video-scaler.h"
+#include "OBSVideoFrame.h"
 
 class AudioRepacker;
 class DecklinkBase;
@@ -14,6 +15,7 @@ class DeckLinkDeviceInstance : public IDeckLinkInputCallback {
 protected:
 	struct obs_source_frame2 currentFrame;
 	struct obs_source_audio currentPacket;
+	struct obs_source_cea_708 currentCaptions;
 	DecklinkBase *decklink = nullptr;
 	DeckLinkDevice *device = nullptr;
 	DeckLinkDeviceMode *mode = nullptr;
@@ -34,6 +36,7 @@ protected:
 	speaker_layout channelFormat = SPEAKERS_STEREO;
 	bool swap;
 
+	OBSVideoFrame *convertFrame = nullptr;
 	IDeckLinkMutableVideoFrame *decklinkOutputFrame = nullptr;
 
 	void FinalizeStream();
@@ -104,4 +107,6 @@ public:
 
 	void DisplayVideoFrame(video_data *frame);
 	void WriteAudio(audio_data *frames);
+	void HandleCaptionPacket(IDeckLinkAncillaryPacket *packet,
+				 const uint64_t timestamp);
 };

plugins/decklink/decklink-source.cpp

@@ -331,9 +331,9 @@ struct obs_source_info create_decklink_source_info()
 	struct obs_source_info decklink_source_info = {};
 	decklink_source_info.id = "decklink-input";
 	decklink_source_info.type = OBS_SOURCE_TYPE_INPUT;
-	decklink_source_info.output_flags = OBS_SOURCE_ASYNC_VIDEO |
+	decklink_source_info.output_flags =
-					    OBS_SOURCE_AUDIO |
+		OBS_SOURCE_ASYNC_VIDEO | OBS_SOURCE_AUDIO |
-					    OBS_SOURCE_DO_NOT_DUPLICATE;
+		OBS_SOURCE_DO_NOT_DUPLICATE | OBS_SOURCE_CEA_708;
 	decklink_source_info.create = decklink_create;
 	decklink_source_info.destroy = decklink_destroy;
 	decklink_source_info.get_defaults = decklink_get_defaults;

plugins/decklink/linux/CMakeLists.txt

@@ -5,6 +5,8 @@ if(DISABLE_DECKLINK)
 	return()
 endif()
 
+include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption)
+
 set(linux-decklink-sdk_HEADERS
 	decklink-sdk/DeckLinkAPI.h
 	decklink-sdk/DeckLinkAPIConfiguration.h
@@ -34,6 +36,7 @@ set(linux-decklink_HEADERS
 	../audio-repack.h
 	../audio-repack.hpp
 	../util.hpp
+	../OBSVideoFrame.h
 	)
 
 set(linux-decklink_SOURCES
@@ -51,6 +54,7 @@ set(linux-decklink_SOURCES
 	../audio-repack.c
 	platform.cpp
 	../util.cpp
+	../OBSVideoFrame.h
 	)
 
 add_library(linux-decklink MODULE
@@ -62,7 +66,7 @@ add_library(linux-decklink MODULE
 
 target_link_libraries(linux-decklink
 	libobs
-	)
+	caption)
 set_target_properties(linux-decklink PROPERTIES FOLDER "plugins")
 
 install_obs_plugin_with_data(linux-decklink ../data)

plugins/decklink/mac/CMakeLists.txt

@@ -9,6 +9,8 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}")
 
 find_library(COREFOUNDATION CoreFoundation)
 
+include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption)
+
 set(mac-decklink-sdk_HEADERS
 	decklink-sdk/DeckLinkAPI.h
 	decklink-sdk/DeckLinkAPIConfiguration.h
@@ -37,6 +39,7 @@ set(mac-decklink_HEADERS
 	../audio-repack.h
 	../audio-repack.hpp
 	../util.hpp
+	../OBSVideoFrame.h
 	)
 
 set(mac-decklink_SOURCES
@@ -54,6 +57,7 @@ set(mac-decklink_SOURCES
 	../audio-repack.c
 	platform.cpp
 	../util.cpp
+	../OBSVideoFrame.cpp
 	)
 
 list(APPEND decklink_HEADERS ${decklink_UI_HEADERS})
@@ -73,7 +77,9 @@ add_library(mac-decklink MODULE
 
 target_link_libraries(mac-decklink
 	libobs
-	${COREFOUNDATION})
+	obs-frontend-api
+	${COREFOUNDATION}
+	caption)
 set_target_properties(mac-decklink PROPERTIES FOLDER "plugins")
 
 install_obs_plugin_with_data(mac-decklink ../data)

plugins/decklink/platform.hpp

@@ -7,6 +7,7 @@ typedef BOOL decklink_bool_t;
 typedef BSTR decklink_string_t;
 IDeckLinkDiscovery *CreateDeckLinkDiscoveryInstance(void);
 IDeckLinkIterator *CreateDeckLinkIteratorInstance(void);
+IDeckLinkVideoConversion *CreateVideoConversionInstance(void);
 #define IUnknownUUID IID_IUnknown
 typedef REFIID CFUUIDBytes;
 #define CFUUIDGetUUIDBytes(x) x

plugins/decklink/win/CMakeLists.txt

@@ -7,6 +7,8 @@ endif()
 
 include(IDLFileHelper)
 
+include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption)
+
 set(win-decklink-sdk_IDLS
 	decklink-sdk/DeckLinkAPI.idl
 	)
@@ -29,6 +31,7 @@ set(win-decklink_HEADERS
 	../audio-repack.h
 	../audio-repack.hpp
 	../util.hpp
+	../OBSVideoFrame.h
 	)
 
 set(MODULE_DESCRIPTION "OBS DeckLink Windows module")
@@ -48,7 +51,8 @@ set(win-decklink_SOURCES
 	../audio-repack.c
 	platform.cpp
 	../util.cpp
-	win-decklink.rc)
+	win-decklink.rc
+	../OBSVideoFrame.cpp)
 
 add_idl_files(win-decklink-sdk_GENERATED_FILES
 	${win-decklink-sdk_IDLS}
@@ -56,6 +60,7 @@ add_idl_files(win-decklink-sdk_GENERATED_FILES
 
 include_directories(
 	${CMAKE_CURRENT_BINARY_DIR}
+	"${CMAKE_SOURCE_DIR}/UI/obs-frontend-api"
 )
 
 add_library(win-decklink MODULE
@@ -66,7 +71,9 @@ add_library(win-decklink MODULE
 	)
 
 target_link_libraries(win-decklink
-	libobs)
+	libobs
+	obs-frontend-api
+	caption)
 set_target_properties(win-decklink PROPERTIES FOLDER "plugins")
 
 install_obs_plugin_with_data(win-decklink ../data)

plugins/decklink/win/platform.cpp

@@ -20,6 +20,16 @@ IDeckLinkIterator *CreateDeckLinkIteratorInstance(void)
 	return result == S_OK ? iterator : nullptr;
 }
 
+IDeckLinkVideoConversion *CreateVideoConversionInstance(void)
+{
+	IDeckLinkVideoConversion *conversion;
+	const HRESULT result = CoCreateInstance(CLSID_CDeckLinkVideoConversion,
+						nullptr, CLSCTX_ALL,
+						IID_IDeckLinkVideoConversion,
+						(void **)&conversion);
+	return result == S_OK ? conversion : nullptr;
+}
+
 bool DeckLinkStringToStdString(decklink_string_t input, std::string &output)
 {
 	if (input == nullptr)

test/cmocka/CMakeLists.txt

@@ -37,3 +37,10 @@ target_link_libraries(test_darray ${CMOCKA_LIBRARIES} libobs)
 
 add_test(test_darray ${CMAKE_CURRENT_BINARY_DIR}/test_darray)
 fixLink(test_darray)
+
+# bitstream test
+add_executable(test_bitstream test_bitstream.c)
+target_link_libraries(test_bitstream ${CMOCKA_LIBRARIES} libobs)
+
+add_test(test_bitstream ${CMAKE_CURRENT_BINARY_DIR}/test_bitstream)
+fixLink(test_bitstream)

test/cmocka/test_bitstream.c

@@ -0,0 +1,34 @@
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <util/bitstream.h>
+
+static void bitstream_test(void **state)
+{
+	struct bitstream_reader reader;
+	uint8_t data[6] = {0x34, 0xff, 0xe1, 0x23, 0x91, 0x45};
+
+	// set len to one less than the array to show that we stop reading at that len
+	bitstream_reader_init(&reader, data, 5);
+
+	assert_int_equal(bitstream_reader_read_bits(&reader, 8), 0x34);
+	assert_int_equal(bitstream_reader_read_bits(&reader, 1), 1);
+	assert_int_equal(bitstream_reader_read_bits(&reader, 3), 7);
+	assert_int_equal(bitstream_reader_read_bits(&reader, 4), 0xF);
+	assert_int_equal(bitstream_reader_r8(&reader), 0xe1);
+	assert_int_equal(bitstream_reader_r16(&reader), 0x2391);
+
+	// test reached end
+	assert_int_equal(bitstream_reader_r8(&reader), 0);
+}
+
+int main()
+{
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test(bitstream_test),
+	};
+
+	return cmocka_run_group_tests(tests, NULL, NULL);
+}