From 2ab549e1c1df23bb5405b9634e3e47a857973c49 Mon Sep 17 00:00:00 2001
From: pkv <pkv@obsproject.com>
Date: Fri, 17 Jul 2020 04:33:22 +0200
Subject: [PATCH] coreaudio-encoder: Fix encoding of 4.0 speaker layout

For four channels, coreaudio encoder defaults to quad
(FL FR, BL BR).
So 4.0 needs to be explicitly set for the encoder.
---
 plugins/coreaudio-encoder/encoder.cpp       | 22 ++++++++++++++++++++-
 plugins/coreaudio-encoder/windows-imports.h | 18 +++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/plugins/coreaudio-encoder/encoder.cpp b/plugins/coreaudio-encoder/encoder.cpp
index 34fbd6477..4bbcddf77 100644
--- a/plugins/coreaudio-encoder/encoder.cpp
+++ b/plugins/coreaudio-encoder/encoder.cpp
@@ -597,14 +597,34 @@ static void *aac_create(obs_data_t *settings, obs_encoder_t *encoder)
 
 	/*
 	 * Fix channel map differences between CoreAudio AAC, FFmpeg, Wav
-	 * New channel mappings below assume 2.1, 4.1, 5.1, 7.1 resp.
+	 * New channel mappings below assume 2.1, 4.0, 4.1, 5.1, 7.1 resp.
 	 */
+
 	if (ca->channels == 3) {
 		SInt32 channelMap3[3] = {2, 0, 1};
 		AudioConverterSetProperty(ca->converter,
 					  kAudioConverterChannelMap,
 					  sizeof(channelMap3), channelMap3);
 
+	} else if (ca->channels == 4) {
+		/*
+		 * For four channels coreaudio encoder has default channel "quad"
+		 * instead of 4.0. So explicitly set channel layout to
+		 * kAudioChannelLayoutTag_MPEG_4_0_B = (116L << 16) | 4.
+		 */
+		AudioChannelLayout inAcl = {0};
+		inAcl.mChannelLayoutTag = (116L << 16) | 4;
+		AudioConverterSetProperty(ca->converter,
+					  kAudioConverterInputChannelLayout,
+					  sizeof(inAcl), &inAcl);
+		AudioConverterSetProperty(ca->converter,
+					  kAudioConverterOutputChannelLayout,
+					  sizeof(inAcl), &inAcl);
+		SInt32 channelMap4[4] = {2, 0, 1, 3};
+		AudioConverterSetProperty(ca->converter,
+					  kAudioConverterChannelMap,
+					  sizeof(channelMap4), channelMap4);
+
 	} else if (ca->channels == 5) {
 		SInt32 channelMap5[5] = {2, 0, 1, 3, 4};
 		AudioConverterSetProperty(ca->converter,
diff --git a/plugins/coreaudio-encoder/windows-imports.h b/plugins/coreaudio-encoder/windows-imports.h
index 73f814ccc..a78eb5043 100644
--- a/plugins/coreaudio-encoder/windows-imports.h
+++ b/plugins/coreaudio-encoder/windows-imports.h
@@ -61,6 +61,24 @@ struct AudioStreamPacketDescription {
 };
 typedef struct AudioStreamPacketDescription AudioStreamPacketDescription;
 
+typedef UInt32 AudioChannelLabel;
+typedef UInt32 AudioChannelLayoutTag;
+
+struct AudioChannelDescription {
+	AudioChannelLabel mChannelLabel;
+	UInt32 mChannelFlags;
+	float mCoordinates[3];
+};
+typedef struct AudioChannelDescription AudioChannelDescription;
+
+struct AudioChannelLayout {
+	AudioChannelLayoutTag mChannelLayoutTag;
+	UInt32 mChannelBitmap;
+	UInt32 mNumberChannelDescriptions;
+	AudioChannelDescription mChannelDescriptions[kVariableLengthArray];
+};
+typedef struct AudioChannelLayout AudioChannelLayout;
+
 typedef OSStatus (*AudioConverterComplexInputDataProc)(
 	AudioConverterRef inAudioConverter, UInt32 *ioNumberDataPackets,
 	AudioBufferList *ioData,