From 23cf5279472d3ae1b2d8d1904e6b1f1e7fd8f012 Mon Sep 17 00:00:00 2001 From: Sven Göthel Date: Fri, 2 Feb 2024 08:27:49 +0100 Subject: Bug 1494 - GLMediaPlayer/GraphUI: Support Displaying Bitmap'ed Subtitles (PGS ..) via FFMPEGFMediaPlayer/FFmpeg FFMPEGFMediaPlayer related changes: - Add libswscale (6th FFmpeg lib used) for sws_getCachedContext(), sws_scale() and sws_freeContext(), used natively to convert the palette'ed bitmap into RGBA colorspace -> GL texture - Handling AVSubtitleRect.type SUBTITLE_BITMAP -- only handled if libswscale is available -- config/adjust texture object -- sws_scale palette'ed bitmap to texture -- intermediate memory is cached, may be resized and free'ed at destroy -- texture objects are managed and passed from GLMediaPlayerImpl, as they are also forwarded to player client via SubBitmapEvent - Passing the AVCodecID to GLMediaPlayerImpl, converted to our CodecID enum. - Unifying creation and opening of AVCodecContext with 'createOpenedAVCodecContext(..)' +++ SubtitleEvent* - SubTextEvent now also handles ASS.Dialogue (FFmpeg 4) besides ASS.Event (FFmpeg 5, 6, ..). +++ GLMediaPlayerImpl - Added ringbuffer subTexFree, managing Texture for bitmap'ed subtitles -- Uses 1 bitmap-subtitle Texture per used textureCount in cache, as one bitmap-subtile can be displayed per frame. Could be potentially reduced to just 2 .. but resources used are relatively low here. - Validating subTexFree + videoFramesFree usage, use blocking get/put ringbuffer due to utilization from different threads. - Receives subtitle content from native getNextPacket0() via callback, creates SubtitleEvent instance and passes it to a SubtitleEventListener - if exists. (See MediaButton example) -- SubBitmapEvent also gets its special SubBitmapEvent.TextureOwner to handle client releasing the event and allowing us to put back the Texture resource to 'subTexFree'. This passing through of the Texture object is probably a weakness of this lifecycle and requires the client to ensure SubtitleEvent.release() gets called. See MediaButton example! - Exposing CodecID, allowing clients like MediaButton to handle SubtitleEvent content according to codec --- .../opengl/util/av/impl/FFMPEGMediaPlayer.java | 56 ++++++++++++++++------ 1 file changed, 41 insertions(+), 15 deletions(-) (limited to 'src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java') diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java index e6784273e..974bdc10b 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java @@ -30,7 +30,6 @@ package jogamp.opengl.util.av.impl; import java.io.IOException; import java.io.PrintStream; -import java.nio.ByteBuffer; import java.security.PrivilegedAction; import com.jogamp.opengl.GL; @@ -41,21 +40,19 @@ import com.jogamp.common.av.AudioFormat; import com.jogamp.common.av.AudioSink; import com.jogamp.common.av.AudioSinkFactory; import com.jogamp.common.av.TimeFrameI; -import com.jogamp.common.os.Clock; import com.jogamp.common.util.IOUtil; import com.jogamp.common.util.PropertyAccess; import com.jogamp.common.util.SecurityUtil; import com.jogamp.gluegen.runtime.ProcAddressTable; import com.jogamp.opengl.util.GLPixelStorageModes; -import com.jogamp.opengl.util.av.ASSEventLine; import com.jogamp.opengl.util.av.GLMediaPlayer; +import com.jogamp.opengl.util.av.VideoPixelFormat; import com.jogamp.opengl.util.texture.Texture; import jogamp.common.os.PlatformPropsImpl; import jogamp.opengl.GLContextImpl; import jogamp.opengl.util.av.AudioSampleFormat; import jogamp.opengl.util.av.GLMediaPlayerImpl; -import jogamp.opengl.util.av.VideoPixelFormat; import jogamp.opengl.util.av.impl.FFMPEGDynamicLibraryBundleInfo.VersionedLib; /*** @@ -98,6 +95,7 @@ import jogamp.opengl.util.av.impl.FFMPEGDynamicLibraryBundleInfo.VersionedLib; *
  • avutil
    • *
  • avdevice (optional for video input devices)
    • *
  • swresample
    • + *
  • swscale (optional for bitmap'ed subtitles)
    • * *

    * @@ -105,10 +103,10 @@ import jogamp.opengl.util.av.impl.FFMPEGDynamicLibraryBundleInfo.VersionedLib; *

    * Currently we are binary compatible with the following major versions: * - * - * - * - * + * + * + * + * *
    ffmpegavcodecavformatavdeviceavutilswresample FFMPEG* class
    4 58 58 58 56 03 FFMPEGv0400
    5 59 59 59 57 04 FFMPEGv0500
    6 60 60 60 58 04 FFMPEGv0600
    ffmpegavcodecavformatavdeviceavutilswresampleswscale FFMPEG* class
    4 58 58 58 56 03 05 FFMPEGv0400
    5 59 59 59 57 04 06 FFMPEGv0500
    6 60 60 60 58 04 07 FFMPEGv0600
    *

    *

    @@ -203,6 +201,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { private static final int avCodecMajorVersionCC; private static final int avDeviceMajorVersionCC; private static final int swResampleMajorVersionCC; + private static final int swScaleMajorVersionCC; private static final boolean available; static { @@ -220,18 +219,21 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { avUtilMajorVersionCC = natives.getAvUtilMajorVersionCC0(); avDeviceMajorVersionCC = natives.getAvDeviceMajorVersionCC0(); swResampleMajorVersionCC = natives.getSwResampleMajorVersionCC0(); + swScaleMajorVersionCC = natives.getSwScaleMajorVersionCC0(); } else { avUtilMajorVersionCC = 0; avFormatMajorVersionCC = 0; avCodecMajorVersionCC = 0; avDeviceMajorVersionCC = 0; swResampleMajorVersionCC = 0; + swScaleMajorVersionCC = 0; } final VersionedLib avCodec = FFMPEGDynamicLibraryBundleInfo.avCodec; final VersionedLib avFormat = FFMPEGDynamicLibraryBundleInfo.avFormat; final VersionedLib avUtil = FFMPEGDynamicLibraryBundleInfo.avUtil; final VersionedLib avDevice = FFMPEGDynamicLibraryBundleInfo.avDevice; final VersionedLib swResample = FFMPEGDynamicLibraryBundleInfo.swResample; + final VersionedLib swScale = FFMPEGDynamicLibraryBundleInfo.swScale; // final boolean avDeviceLoaded = FFMPEGDynamicLibraryBundleInfo.avDeviceLoaded(); // final boolean swResampleLoaded = FFMPEGDynamicLibraryBundleInfo.swResampleLoaded(); final int avCodecMajor = avCodec.version.getMajor(); @@ -239,11 +241,13 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { final int avUtilMajor = avUtil.version.getMajor(); final int avDeviceMajor = avDevice.version.getMajor(); final int swResampleMajor = swResample.version.getMajor(); + final int swScaleMajor = swScale.version.getMajor(); libAVVersionGood = avCodecMajorVersionCC == avCodecMajor && avFormatMajorVersionCC == avFormatMajor && avUtilMajorVersionCC == avUtilMajor && ( avDeviceMajorVersionCC == avDeviceMajor || 0 == avDeviceMajor ) && - swResampleMajorVersionCC == swResampleMajor; + swResampleMajorVersionCC == swResampleMajor && + ( swScaleMajorVersionCC == swScaleMajor || 0 == swScaleMajor ); if( !libAVVersionGood ) { System.err.println("FFmpeg Not Matching Compile-Time / Runtime Major-Version"); } @@ -257,6 +261,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { avCodecMajorVersionCC = 0; avDeviceMajorVersionCC = 0; swResampleMajorVersionCC = 0; + swScaleMajorVersionCC = 0; libAVVersionGood = false; } available = libAVGood && libAVVersionGood && null != natives; @@ -270,6 +275,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { out.println("FFmpeg Util : "+FFMPEGDynamicLibraryBundleInfo.avUtil+" [cc "+avUtilMajorVersionCC+"]"); out.println("FFmpeg Device : "+FFMPEGDynamicLibraryBundleInfo.avDevice+" [cc "+avDeviceMajorVersionCC+", loaded "+FFMPEGDynamicLibraryBundleInfo.avDeviceLoaded()+"]"); out.println("FFmpeg Resample: "+FFMPEGDynamicLibraryBundleInfo.swResample+" [cc "+swResampleMajorVersionCC+", loaded "+FFMPEGDynamicLibraryBundleInfo.swResampleLoaded()+"]"); + out.println("FFmpeg Scale : "+FFMPEGDynamicLibraryBundleInfo.swScale+" [cc "+swScaleMajorVersionCC+", loaded "+FFMPEGDynamicLibraryBundleInfo.swScaleLoaded()+"]"); out.println("FFmpeg Class : "+(null!= natives ? natives.getClass().getSimpleName() : "n/a")); } @Override @@ -447,6 +453,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { @Override public Object run() { final ProcAddressTable pt = ctx.getGLProcAddressTable(); + final long procAddrGLTexImage2D = pt.getAddressFor("glTexImage2D"); final long procAddrGLTexSubImage2D = pt.getAddressFor("glTexSubImage2D"); final long procAddrGLGetError = pt.getAddressFor("glGetError"); final long procAddrGLFlush = pt.getAddressFor("glFlush"); @@ -458,7 +465,9 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { procAddrGLEnable = 0; } final long procAddrGLBindTexture = pt.getAddressFor("glBindTexture"); - natives.setGLFuncs0(moviePtr, procAddrGLTexSubImage2D, procAddrGLGetError, procAddrGLFlush, procAddrGLFinish, procAddrGLEnable, procAddrGLBindTexture); + natives.setGLFuncs0(moviePtr, procAddrGLTexImage2D, procAddrGLTexSubImage2D, + procAddrGLGetError, procAddrGLFlush, procAddrGLFinish, + procAddrGLEnable, procAddrGLBindTexture, gl.isNPOTTextureAvailable()); return null; } } ); audioQueueSize = AudioSink.DefaultQueueSizeWithVideo; @@ -983,7 +992,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final int getNextTextureImpl(final GL gl, final TextureFrame nextFrame) { + protected final int getNextTextureImpl(final GL gl, final TextureFrame vFrame, final Texture sTex, final boolean[] sTexUsed) { if(0==moviePtr) { throw new GLException("FFMPEG native instance null"); } @@ -994,15 +1003,32 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { // final Texture tex = nextFrame.getTexture(); // tex.enable(gl); // tex.bind(gl); - vTexID = nextFrame.getTexture().getTextureObject(); + vTexID = vFrame.getTexture().getTextureObject(); } /** Try decode up to 10 packets to find one containing video. */ for(int i=0; TimeFrameI.INVALID_PTS == vPTS && 10 > i; i++) { - vPTS = natives.readNextPacket0(moviePtr, getTextureTarget(), vTexID, getTextureFormat(), getTextureType(), GL.GL_TEXTURE_2D, 0); + int sTexID = 0; // invalid + int sTexWidth = 0; + int sTexHeight = 0; + if( null != gl && !sTexUsed[0] ) { + // glEnable() and glBindTexture() are performed in native readNextPacket0() + // final Texture tex = nextFrame.getTexture(); + // tex.enable(gl); + // tex.bind(gl); + vTexID = vFrame.getTexture().getTextureObject(); + if( null != sTex ) { + sTexID = sTex.getTextureObject(); + // FIXME: Disabled in native code, buggy on AMD GPU corrupting texture content + sTexWidth = sTex.getWidth(); + sTexHeight = sTex.getHeight(); + } + } + vPTS = natives.readNextPacket0(moviePtr, getTextureTarget(), vTexID, getTextureFormat(), getTextureType(), + GL.GL_TEXTURE_2D, sTexID, sTexWidth, sTexHeight, sTex, sTexUsed); } - if( null != nextFrame ) { - nextFrame.setPTS(vPTS); + if( null != vFrame ) { + vFrame.setPTS(vPTS); } return vPTS; } -- cgit v1.2.3