diff --git a/libobs/obs-encoder.c b/libobs/obs-encoder.c
index 1679b0ed6..75389b48b 100644
--- a/libobs/obs-encoder.c
+++ b/libobs/obs-encoder.c
@@ -424,6 +424,8 @@ void obs_encoder_shutdown(obs_encoder_t *encoder)
 		encoder->info.destroy(encoder->context.data);
 		encoder->context.data    = NULL;
 		encoder->paired_encoder  = NULL;
+		encoder->first_received  = false;
+		encoder->offset_usec     = 0;
 		encoder->start_ts        = 0;
 	}
 	pthread_mutex_unlock(&encoder->init_mutex);
@@ -798,9 +800,15 @@ static inline void do_encode(struct obs_encoder *encoder,
 	}
 
 	if (received) {
+		if (!encoder->first_received) {
+			encoder->offset_usec = packet_dts_usec(&pkt);
+			encoder->first_received = true;
+		}
+
 		/* we use system time here to ensure sync with other encoders,
 		 * you do not want to use relative timestamps here */
-		pkt.dts_usec = encoder->start_ts / 1000 + packet_dts_usec(&pkt);
+		pkt.dts_usec = encoder->start_ts / 1000 +
+			packet_dts_usec(&pkt) - encoder->offset_usec;
 
 		pthread_mutex_lock(&encoder->callbacks_mutex);
 
diff --git a/libobs/obs-internal.h b/libobs/obs-internal.h
index 01d095f04..2b0be22f0 100644
--- a/libobs/obs-internal.h
+++ b/libobs/obs-internal.h
@@ -743,7 +743,9 @@ struct obs_encoder {
 	 * wait_for_video makes it wait until it's ready to sync up with
 	 * video */
 	bool                            wait_for_video;
+	bool                            first_received;
 	struct obs_encoder              *paired_encoder;
+	int64_t                         offset_usec;
 	uint64_t                        start_ts;
 
 	pthread_mutex_t                 outputs_mutex;