From 6332e13b2f0aa9818d37802302f04c90a4fa4239 Mon Sep 17 00:00:00 2001 From: Sven Gothel Date: Sat, 10 Aug 2013 09:14:19 +0200 Subject: GLMediaPlayer: Add multithreaded decoding w/ textureCount > 2 where available EGL/FFMPeg. WIP! Off-thread decoding: If validated (impl) textureCount > 2, decoding happens on extra thread. If decoding requires GL context, a shared context is created for decoding thread. API Changes: - initGLStream(..): Adds 'textureCount' as argument. - TextureSequence.TexSeqEventListener.newFrameAvailable(..) exposes the new frame available - TextureSequence.TextureFrame exposes the PTS (video) Implementation: - 'int validateTextureCount(int)': implementation decides whether textureCount can be > 2, i.e. off-thread decoding allowed, default is NO w/ textureCount==2! - 'boolean requiresOffthreadGLCtx()': implementation decides whether shared context is required for off-thread decoding - 'syncFrame2Audio(TextureFrame frame)': implementation shall handle a/v sync, due to audio stream details (pts, buffered frames) - FFMPEGMediaPlayer extends GLMediaPlayerImpl, no more EGLMediaPlayerImpl (redundant) +++ - SyncedRingbuffer: Expose T[] array +++ TODO: - syncAV! - test Android --- .../opengl/util/av/impl/OMXGLMediaPlayer.java | 44 +++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java') diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java index aef98fcde..a21bb40a8 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java @@ -33,7 +33,6 @@ import java.net.URL; import javax.media.opengl.GL; import javax.media.opengl.GLException; -import javax.media.opengl.GLProfile; import com.jogamp.opengl.util.texture.TextureSequence; @@ -49,17 +48,17 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { static final boolean available; static { + available = false; + /** FIXME! // OMX binding is included in jogl_desktop and jogl_mobile GLProfile.initSingleton(); - available = initIDs0(); + available = initIDs0(); */ } public static final boolean isAvailable() { return available; } protected long moviePtr = 0; - protected TextureSequence.TextureFrame lastTex = null; - public OMXGLMediaPlayer() { super(TextureType.KHRImage, true); if(!available) { @@ -76,17 +75,15 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } @Override - protected TextureSequence.TextureFrame createTexImage(GL gl, int idx, int[] tex) { - final EGLTextureFrame eglTex = (EGLTextureFrame) super.createTexImage(gl, idx, tex); - _setStreamEGLImageTexture2D(moviePtr, idx, tex[idx], eglTex.getImage(), eglTex.getSync()); - lastTex = eglTex; + protected TextureSequence.TextureFrame createTexImage(GL gl, int texName) { + final EGLTextureFrame eglTex = (EGLTextureFrame) super.createTexImage(gl, texName); + _setStreamEGLImageTexture2D(moviePtr, texName, eglTex.getImage(), eglTex.getSync()); return eglTex; } @Override - protected void destroyTexImage(GL gl, TextureSequence.TextureFrame imgTex) { - lastTex = null; - super.destroyTexImage(gl, imgTex); + protected void destroyTexFrame(GL gl, TextureSequence.TextureFrame imgTex) { + super.destroyTexFrame(gl, imgTex); } @Override @@ -99,7 +96,7 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } @Override - protected void initGLStreamImpl(GL gl, int[] texNames) throws IOException { + protected void initGLStreamImpl(GL gl) throws IOException { if(0==moviePtr) { throw new GLException("OMX native instance null"); } @@ -119,6 +116,10 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { protected int getCurrentPositionImpl() { return 0!=moviePtr ? _getCurrentPosition(moviePtr) : 0; } + @Override + protected int getAudioPTSImpl() { + return getCurrentPositionImpl(); + } @Override protected boolean setPlaySpeedImpl(float rate) { @@ -168,24 +169,23 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } @Override - protected TextureSequence.TextureFrame getLastTextureImpl() { - return lastTex; - } - - @Override - protected TextureSequence.TextureFrame getNextTextureImpl(GL gl, boolean blocking) { + protected boolean getNextTextureImpl(GL gl, TextureFrame nextFrame, boolean blocking) { if(0==moviePtr) { throw new GLException("OMX native instance null"); } final int nextTex = _getNextTextureID(moviePtr, blocking); if(0 < nextTex) { - final TextureSequence.TextureFrame eglImgTex = texFrameMap.get(new Integer(_getNextTextureID(moviePtr, blocking))); + /* FIXME + final TextureSequence.TextureFrame eglImgTex = + texFrameMap.get(new Integer(_getNextTextureID(moviePtr, blocking))); if(null!=eglImgTex) { lastTex = eglImgTex; - } + } */ } - return lastTex; + return true; } + @Override + protected void syncFrame2Audio(TextureFrame frame) { } private String replaceAll(String orig, String search, String repl) { String dest=null; @@ -216,7 +216,7 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { private native void _setStream(long moviePtr, int textureNum, String path); private native void _activateStream(long moviePtr); - private native void _setStreamEGLImageTexture2D(long moviePtr, int i, int tex, long image, long sync); + private native void _setStreamEGLImageTexture2D(long moviePtr, int tex, long image, long sync); private native int _seek(long moviePtr, int position); private native void _setPlaySpeed(long moviePtr, float rate); private native void _play(long moviePtr); -- cgit v1.2.3 From c37629ea8fdcb11f7f8a18e37a4cde57d4ba6a01 Mon Sep 17 00:00:00 2001 From: Sven Gothel Date: Wed, 14 Aug 2013 07:02:59 +0200 Subject: GLMediaPlayer Multithreaded Decoding: GLMediaPlayer* (Part-3) - WIP - GLMediaPlayer - Remove State.Stopped and method stop() - redundant, use pause() / destroy() - Add notion of stream IDs - Add API doc: State / Stream-ID incl. html-anchor - Expose video/audio PTS, .. - Expose optional AudioSink - Min multithreaded textureCount is 4 (EGL* and FFMPEG*) - GLMediaPlayerImpl - Move AudioSink rel. impl. to this class, allowing a tight video implementation reusing logic. - Remove 'synchronized' methods, synchronize on State where applicable - implement new methods (see above) - playSpeed is handled partially in AudioSink. If it exeeds AudioSink's capabilities, drop audio and rely solely on video sync. - video sync (WIP) - video pts delay based on geometric weight - reset video SCR if 'out of range', resync w/ PTS - - FramePusher - allow interruption when pausing/stopping, while waiting for next avail free frame to decode. - FFMPEGMediaPlayer - Add proper AudioDataFormat negotiation AudioSink <-> libav - Parse libav's SampleFormat - Remove AudioSink interaction (moved to GLMediaPlayerImpl) - Tests (MovieSimple, MovieCube): - Add aid/vid selection - Add KeyListener for actions: seek(..), play()/pause(), setPlaySpeed(..) - Dump perf-string each 2s - TODO: - Add audio sync in AudioSink, similar to GLMediaPlayer's weighted video delay, here: drop audio frames. --- make/scripts/tests.sh | 6 +- .../com/jogamp/opengl/util/av/GLMediaPlayer.java | 146 ++-- .../opengl/util/av/GLMediaPlayerFactory.java | 5 +- .../opengl/util/texture/TextureSequence.java | 5 +- .../android/av/AndroidGLMediaPlayerAPI14.java | 53 +- .../jogamp/opengl/util/av/EGLMediaPlayerImpl.java | 2 +- .../jogamp/opengl/util/av/GLMediaPlayerImpl.java | 740 +++++++++++++++------ .../jogamp/opengl/util/av/NullGLMediaPlayer.java | 27 +- .../opengl/util/av/impl/FFMPEGMediaPlayer.java | 232 ++++--- .../opengl/util/av/impl/OMXGLMediaPlayer.java | 25 +- src/jogl/native/libav/ffmpeg_tool.h | 19 +- .../jogamp_opengl_util_av_impl_FFMPEGMediaPlayer.c | 223 ++++--- .../opengl/test/android/MovieCubeActivity0.java | 3 +- .../opengl/test/android/MovieSimpleActivity0.java | 3 +- .../opengl/test/android/MovieSimpleActivity1.java | 4 +- .../jogl/demos/es2/TextureSequenceCubeES2.java | 23 +- .../test/junit/jogl/demos/es2/av/MovieCube.java | 157 +++-- .../test/junit/jogl/demos/es2/av/MovieSimple.java | 228 +++++-- 18 files changed, 1283 insertions(+), 618 deletions(-) (limited to 'src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java') diff --git a/make/scripts/tests.sh b/make/scripts/tests.sh index ba4589d14..82e9fb92b 100644 --- a/make/scripts/tests.sh +++ b/make/scripts/tests.sh @@ -131,7 +131,9 @@ function jrun() { #D_ARGS="-Djogl.1thread=true -Djogl.debug.Threading" #D_ARGS="-Djogl.debug.DebugGL -Djogl.debug.TraceGL -Djogl.debug.GLContext.TraceSwitch -Djogl.debug=all" #D_ARGS="-Djogamp.debug.IOUtil -Djogl.debug.GLSLCode -Djogl.debug.GLMediaPlayer" + #D_ARGS="-Djogl.debug.GLMediaPlayer" #D_ARGS="-Djogl.debug.GLMediaPlayer -Djogl.debug.AudioSink" + #D_ARGS="-Djogl.debug.AudioSink" #D_ARGS="-Djogl.debug.GLArrayData" #D_ARGS="-Djogl.debug.GLDrawable" #D_ARGS="-Djogl.debug.EGLDisplayUtil -Dnativewindow.debug.GraphicsConfiguration -Djogl.debug.GLDrawable" @@ -314,8 +316,8 @@ function testawtswt() { # av demos # #testnoawt jogamp.opengl.openal.av.ALDummyUsage $* -testnoawt com.jogamp.opengl.test.junit.jogl.demos.es2.av.MovieCube $* -#testnoawt com.jogamp.opengl.test.junit.jogl.demos.es2.av.MovieSimple $* +#testnoawt com.jogamp.opengl.test.junit.jogl.demos.es2.av.MovieCube $* +testnoawt com.jogamp.opengl.test.junit.jogl.demos.es2.av.MovieSimple $* # # core/newt (testnoawt and testawt) diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java index a36bce305..fae88ea18 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java +++ b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java @@ -34,19 +34,37 @@ import javax.media.opengl.GL; import javax.media.opengl.GLException; import jogamp.opengl.Debug; +import jogamp.opengl.util.av.GLMediaPlayerImpl; import com.jogamp.opengl.util.texture.TextureSequence; /** - * Lifecycle of an GLMediaPlayer: + * GLMediaPlayer interface specifies a {@link TextureSequence} + * with a video stream as it's source. + *

+ * Audio maybe supported and played back internally or via an {@link AudioSink} implementation, + * if an audio stream is selected in {@link #initGLStream(GL, int, URLConnection, int, int)}. + *

+ *
GLMediaPlayer Lifecycle
 + *

+ * + * + * + * + * + * + * + *
action state before state after
{@link #initGLStream(GL, int, URLConnection, int, int)} Uninitialized Paused
{@link #play()} Paused Playing
{@link #pause()} Playing Paused
{@link #seek(int)} Playing, Paused Unchanged
{@link #destroy(GL)} ANY Uninitialized
+ *

+ *
Audio and video Stream IDs
 + *

* - * - * - * - * - * - * + * + * + * + * *
action state before state after
{@link #initGLStream(GL, int, URLConnection)} Uninitialized Stopped
{@link #start()} Stopped, Paused Playing
{@link #stop()} Playing, Paused Stopped
{@link #pause()} Playing Paused
{@link #destroy(GL)} ANY Uninitialized
value request get
{@link #STREAM_ID_NONE} mute not available
{@link #STREAM_ID_AUTO} auto unspecified
≥0 specific stream specific stream
+ *

*

* Current implementations (check each API doc link for details): *