From b1956113f5601b0cc6ac525d3918a0dfa8d240af Mon Sep 17 00:00:00 2001
From: Sven Gothel <sgothel@jausoft.com>
Date: Sun, 15 Oct 2023 07:06:53 +0200
Subject: Bug 1472: Enhance GLMediaPlayer AV Sync: Utilize SCR aware audio PTS
 used as master-clock, enabling proper AV sync w/ untouched audio

We can finally utilize the added pass through audio PTS, see commits
- GlueGen 52725b4c6525487f93407f529dc0a758b387a4fc
- JOAL 12029f1ec1d8afa576e1ac61655f318cc37c1d16

This enables us to use the audio PTS as the master-clock and adjust video to the untouched audio.

In case no audio is selected/playing or audio is muted,
we sync merely on the system-clock (SCR) w/o audio.

AV granularity is 22ms, however, since the ALAudioSink PTS may be a little late,
it renders even a slightly better sync in case of too early audio (d_apts < 0).

Since video frames are sync'ed to audio, the resync procedure may result
in a hysteresis swinging into sync. This might be notable at start
and when resumed audio or after seek.

We leave the audio frames untouched to reduce processing burden
and allow non-disrupted listening.

Passed AV sync tests
- Five-minute-sync-test.mp4
- Audio-Video-Sync-Test-Calibration-23.98fps-24fps.mp4
- Audio-Video-Sync-Test-2.mkv
---
 .../jogamp/opengl/util/av/NullGLMediaPlayer.java   | 34 +++++++++++++---------
 1 file changed, 20 insertions(+), 14 deletions(-)

(limited to 'src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java')

diff --git a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java
index be440adaf..d616020e7 100644
--- a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java
+++ b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java
@@ -34,11 +34,13 @@ import java.nio.ByteBuffer;
 import com.jogamp.opengl.GL;
 import com.jogamp.opengl.GLException;
 import com.jogamp.opengl.GLProfile;
-
+import com.jogamp.common.av.PTS;
 import com.jogamp.common.nio.Buffers;
+import com.jogamp.common.os.Clock;
 import com.jogamp.common.os.Platform;
 import com.jogamp.common.util.IOUtil;
 import com.jogamp.opengl.util.av.GLMediaPlayer;
+import com.jogamp.opengl.util.av.GLMediaPlayer.State;
 import com.jogamp.opengl.util.texture.Texture;
 import com.jogamp.opengl.util.texture.TextureData;
 import com.jogamp.opengl.util.texture.TextureIO;
@@ -50,8 +52,7 @@ import com.jogamp.opengl.util.texture.TextureSequence;
  */
 public class NullGLMediaPlayer extends GLMediaPlayerImpl {
     private TextureData texData = null;
-    private int pos_ms = 0;
-    private long pos_start = 0;
+    private final PTS video_pts = new PTS( () -> { return State.Playing == getState() ? getPlaySpeed() : 0f; } );
 
     public NullGLMediaPlayer() {
         super();
@@ -65,7 +66,7 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
 
     @Override
     protected final boolean resumeImpl() {
-        pos_start = Platform.currentTimeMillis();
+        video_pts.setSCR(Clock.currentMillis());
         return true;
     }
 
@@ -80,24 +81,29 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
 
     @Override
     protected final int seekImpl(final int msec) {
-        pos_ms = msec;
+        video_pts.setPTS(msec);
         validatePos();
-        return pos_ms;
+        return msec;
     }
 
     @Override
     protected final int getNextTextureImpl(final GL gl, final TextureFrame nextFrame) {
-        final int pts = getAudioPTSImpl();
+        final int pts = getLastBufferedAudioPTS();
         nextFrame.setPTS( pts );
         return pts;
     }
 
     @Override
-    protected final int getAudioPTSImpl() {
-        pos_ms = (int) ( Platform.currentTimeMillis() - pos_start );
+    protected PTS getAudioPTSImpl() { return video_pts; }
+    @Override
+    protected final PTS getUpdatedAudioPTS() {
         validatePos();
-        return pos_ms;
+        return video_pts;
     }
+    @Override
+    protected int getAudioQueuedDuration() { return 0; }
+    @Override
+    protected int getLastBufferedAudioPTS() { return video_pts.getLast(); }
 
     @Override
     protected final void destroyImpl() {
@@ -176,11 +182,11 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
 
     private void validatePos() {
         boolean considerPausing = false;
-        if( 0 > pos_ms) {
-            pos_ms = 0;
+        if( 0 > video_pts.getLast() ) {
+            video_pts.setPTS(0);
             considerPausing = true;
-        } else if ( pos_ms > getDuration() ) {
-            pos_ms = getDuration();
+        } else if ( video_pts.getLast() > getDuration() ) {
+            video_pts.setPTS( getDuration() );
             considerPausing = true;
         }
         if( considerPausing && State.Playing == getState() ) {
-- 
cgit v1.2.3