From e4690c09229e739b751107ff18a0dd83fd958f1d Mon Sep 17 00:00:00 2001 From: Jaideep Date: Thu, 9 Nov 2023 14:22:03 -0800 Subject: [PATCH 1/6] vp9 stride value fix --- .../InvertedColorsVideoRenderer.java | 65 +++++++++---------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java index 88714bc8..6af750d6 100644 --- a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java +++ b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java @@ -76,8 +76,8 @@ static class MyRenderer implements GLSurfaceView.Renderer { + " u=texture2D(Utex,vec2(nx,ny)).r;\n" + " v=texture2D(Vtex,vec2(nx,ny)).r;\n" - + " y=1.0-1.1643*(y-0.0625);\n" // this line produces the inverted effect - // + " y=1.1643*(y-0.0625);\n" // use this line instead if you want to have normal colors + // + " y=1.0-1.1643*(y-0.0625);\n" // this line produces the inverted effect + + " y=1.1643*(y-0.0625);\n" // use this line instead if you want to have normal colors + " u=u-0.5;\n" + " v=v-0.5;\n" + " r=y+1.5958*v;\n" + " g=y-0.39173*u-0.81290*v;\n" + " b=y+2.017*u;\n" @@ -202,48 +202,43 @@ void setupTextures(Frame frame) { textureHeight = frame.getHeight(); } + void GlTexSubImage2D(int width, int height, int stride, + ByteBuffer buf) { + if (stride == width) { + // Yay! We can upload the entire plane in a single GL call. + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, + height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } else { + for (int row = 0; row < height; ++row) { + buf.position( row * stride); + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, row, width, + 1, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } + } + } + void updateTextures(Frame frame) { int width = frame.getWidth(); int height = frame.getHeight(); int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; - int y_size = width * height; - int uv_size = half_width * half_height; - ByteBuffer bb = frame.getBuffer(); - // If we are reusing this frame, make sure we reset position and - // limit - bb.clear(); + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); - if (bb.remaining() == y_size + uv_size * 2) { - bb.position(0); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); + GlTexSubImage2D(width, height, frame.getYstride(), frame.getYplane()); - GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); - GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, - height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - bb); - - bb.position(y_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE1); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - - bb.position(y_size + uv_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE2); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - } else { - textureWidth = 0; - textureHeight = 0; - } + GLES20.glActiveTexture(GLES20.GL_TEXTURE2); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } From f3eca4a97812fc761d2b125c16cd9568d4ca257f Mon Sep 17 00:00:00 2001 From: Jaideep Date: Thu, 16 Nov 2023 09:52:49 -0800 Subject: [PATCH 2/6] Revert to Inverted line --- .../basicvideorenderer/InvertedColorsVideoRenderer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java index 6af750d6..9b84580e 100644 --- a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java +++ b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java @@ -76,8 +76,8 @@ static class MyRenderer implements GLSurfaceView.Renderer { + " u=texture2D(Utex,vec2(nx,ny)).r;\n" + " v=texture2D(Vtex,vec2(nx,ny)).r;\n" - // + " y=1.0-1.1643*(y-0.0625);\n" // this line produces the inverted effect - + " y=1.1643*(y-0.0625);\n" // use this line instead if you want to have normal colors + + " y=1.0-1.1643*(y-0.0625);\n" // this line produces the inverted effect + // + " y=1.1643*(y-0.0625);\n" // use this line instead if you want to have normal colors + " u=u-0.5;\n" + " v=v-0.5;\n" + " r=y+1.5958*v;\n" + " g=y-0.39173*u-0.81290*v;\n" + " b=y+2.017*u;\n" From d7bf9b7124b405c54f5f439e011310fcea9ef4e4 Mon Sep 17 00:00:00 2001 From: Jaideep Date: Thu, 16 Nov 2023 16:38:13 -0800 Subject: [PATCH 3/6] vp9 YUV rendering fix, based on using plane/size api --- .../InvertedColorsVideoRenderer.java | 1 - .../ReceiveFrameMetaDataRenderer.java | 62 +++++++--------- .../ScreenshotVideoRenderer.java | 65 ++++++++-------- .../ScreenshotVideoRenderer.kt | 74 +++++++++---------- 4 files changed, 96 insertions(+), 106 deletions(-) diff --git a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java index 9b84580e..a22a4a44 100644 --- a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java +++ b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java @@ -239,7 +239,6 @@ void updateTextures(Frame frame) { GLES20.glActiveTexture(GLES20.GL_TEXTURE2); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); - } @Override diff --git a/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java b/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java index 30015d80..8f1c3835 100644 --- a/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java +++ b/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java @@ -198,49 +198,43 @@ void setupTextures(Frame frame) { textureHeight = frame.getHeight(); } + void GlTexSubImage2D(int width, int height, int stride, + ByteBuffer buf) { + if (stride == width) { + // Yay! We can upload the entire plane in a single GL call. + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, + height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } else { + for (int row = 0; row < height; ++row) { + buf.position( row * stride); + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, row, width, + 1, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } + } + } + void updateTextures(Frame frame) { int width = frame.getWidth(); int height = frame.getHeight(); int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; - int y_size = width * height; - int uv_size = half_width * half_height; - - ByteBuffer bb = frame.getBuffer(); - // If we are reusing this frame, make sure we reset position and - // limit - bb.clear(); - if (bb.remaining() == y_size + uv_size * 2) { - bb.position(0); + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); - GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); - GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); + GlTexSubImage2D(width, height, frame.getYstride(), frame.getYplane()); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, - height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - bb); - - bb.position(y_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE1); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - - bb.position(y_size + uv_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE2); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - } else { - textureWidth = 0; - textureHeight = 0; - } + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + GLES20.glActiveTexture(GLES20.GL_TEXTURE2); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } @Override diff --git a/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java b/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java index 3ef02590..7374a241 100644 --- a/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java +++ b/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java @@ -198,44 +198,43 @@ void setupTextures(Frame frame) { textureHeight = frame.getHeight(); } + void GlTexSubImage2D(int width, int height, int stride, + ByteBuffer buf) { + if (stride == width) { + // Yay! We can upload the entire plane in a single GL call. + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, + height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } else { + for (int row = 0; row < height; ++row) { + buf.position( row * stride); + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, row, width, + 1, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } + } + } + void updateTextures(Frame frame) { int width = frame.getWidth(); int height = frame.getHeight(); int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; - int y_size = width * height; - int uv_size = half_width * half_height; - - ByteBuffer bb = frame.getBuffer(); - // If we are reusing this frame, make sure we reset position and - // limit - bb.clear(); - - if (bb.remaining() == y_size + uv_size * 2) { - bb.position(0); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, - height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - bb); - - bb.position(y_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE1); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - - bb.position(y_size + uv_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE2); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - } else { - textureWidth = 0; - textureHeight = 0; - } + + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); + GlTexSubImage2D(width, height, frame.getYstride(), frame.getYplane()); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE2); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } @Override diff --git a/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt b/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt index 6308d241..2eeb9ce3 100644 --- a/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt +++ b/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt @@ -127,48 +127,46 @@ void main(void) { textureHeight = frame.height } - private fun updateTextures(frame: Frame) { - val width = frame.width - val height = frame.height - val half_width = width + 1 shr 1 - val half_height = height + 1 shr 1 - val y_size = width * height - val uv_size = half_width * half_height - val bb = frame.buffer - // If we are reusing this frame, make sure we reset position and - // limit - bb.clear() - if (bb.remaining() == y_size + uv_size * 2) { - bb.position(0) - GLES20.glActiveTexture(GLES20.GL_TEXTURE0) - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]) - GLES20.glTexSubImage2D( - GLES20.GL_TEXTURE_2D, 0, 0, 0, width, - height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - bb - ) - bb.position(y_size) - GLES20.glActiveTexture(GLES20.GL_TEXTURE1) - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]) - GLES20.glTexSubImage2D( - GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb - ) - bb.position(y_size + uv_size) - GLES20.glActiveTexture(GLES20.GL_TEXTURE2) - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]) - GLES20.glTexSubImage2D( - GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb - ) + private fun GlTexSubImage2D(width: Int, height: Int, stride: Int, + buf: ByteBuffer) { + if (stride == width) { + // Yay! We can upload the entire plane in a single GL call. + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, + height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); } else { - textureWidth = 0 - textureHeight = 0 + for (row in 0 until height) { + buf.position( row * stride); + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, row, width, + 1, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } } } + private fun updateTextures(frame: Frame) { + val width = frame.getWidth(); + val height = frame.getHeight(); + val half_width = (width + 1) >> 1; + val half_height = (height + 1) >> 1; + + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); + GlTexSubImage2D(width, height, frame.getYstride(), frame.getYplane()); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE2); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); + } + } + override fun onSurfaceChanged(gl: GL10, width: Int, height: Int) { GLES20.glViewport(0, 0, width, height) viewportWidth = width From 289dcafd71e7332ade3aeb95af52dc75ded8ced3 Mon Sep 17 00:00:00 2001 From: Jaideep Date: Thu, 16 Nov 2023 17:46:53 -0800 Subject: [PATCH 4/6] One more file --- .../InvertedColorsVideoRenderer.java | 62 +++++++++---------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java b/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java index 3605f087..8aac9242 100644 --- a/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java +++ b/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java @@ -202,49 +202,43 @@ void setupTextures(Frame frame) { textureHeight = frame.getHeight(); } + void GlTexSubImage2D(int width, int height, int stride, + ByteBuffer buf) { + if (stride == width) { + // Yay! We can upload the entire plane in a single GL call. + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, + height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } else { + for (int row = 0; row < height; ++row) { + buf.position( row * stride); + GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, row, width, + 1, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + buf); + } + } + } + void updateTextures(Frame frame) { int width = frame.getWidth(); int height = frame.getHeight(); int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; - int y_size = width * height; - int uv_size = half_width * half_height; - - ByteBuffer bb = frame.getBuffer(); - // If we are reusing this frame, make sure we reset position and - // limit - bb.clear(); - if (bb.remaining() == y_size + uv_size * 2) { - bb.position(0); + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); - GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); - GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); + GlTexSubImage2D(width, height, frame.getYstride(), frame.getYplane()); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, - height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - bb); - - bb.position(y_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE1); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - - bb.position(y_size + uv_size); - GLES20.glActiveTexture(GLES20.GL_TEXTURE2); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, - half_width, half_height, GLES20.GL_LUMINANCE, - GLES20.GL_UNSIGNED_BYTE, bb); - } else { - textureWidth = 0; - textureHeight = 0; - } + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + GLES20.glActiveTexture(GLES20.GL_TEXTURE2); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } @Override From 83f0b2429c84859ad22ce741a8ea04d21fbb40a7 Mon Sep 17 00:00:00 2001 From: Jaideep Shah Date: Mon, 8 Jan 2024 13:29:34 -0800 Subject: [PATCH 5/6] compile errors --- .../ScreenshotVideoRenderer.kt | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt b/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt index 2eeb9ce3..60a8810d 100644 --- a/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt +++ b/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt @@ -133,38 +133,37 @@ void main(void) { // Yay! We can upload the entire plane in a single GL call. GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - buf); + buf) } else { for (row in 0 until height) { - buf.position( row * stride); + buf.position( row * stride) GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, row, width, 1, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - buf); + buf) } } } private fun updateTextures(frame: Frame) { - val width = frame.getWidth(); - val height = frame.getHeight(); - val half_width = (width + 1) >> 1; - val half_height = (height + 1) >> 1; + val width = frame.getWidth() + val height = frame.getHeight() + val half_width = width + 1 shr 1 + val half_height = height + 1 shr 1 - GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); - GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1) + GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1) - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]); - GlTexSubImage2D(width, height, frame.getYstride(), frame.getYplane()); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[0]) + GlTexSubImage2D(width, height, frame.getYstride(), frame.getYplane()) - GLES20.glActiveTexture(GLES20.GL_TEXTURE1); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + GLES20.glActiveTexture(GLES20.GL_TEXTURE1) + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]) + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()) - GLES20.glActiveTexture(GLES20.GL_TEXTURE2); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); - } + GLES20.glActiveTexture(GLES20.GL_TEXTURE2) + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]) + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()) } override fun onSurfaceChanged(gl: GL10, width: Int, height: Int) { @@ -431,4 +430,4 @@ void main(void) { view.setRenderer(renderer) view.renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY } -} \ No newline at end of file +} From 61e4eb7656e40260b3b8b91df25d0b21b7a9ea8b Mon Sep 17 00:00:00 2001 From: Jaideep Date: Tue, 9 Jan 2024 10:31:33 -0800 Subject: [PATCH 6/6] frame buffer size checks added, just in case. Also it will be compatible with SDK checks --- .../InvertedColorsVideoRenderer.java | 14 ++++++++++++-- .../InvertedColorsVideoRenderer.java | 13 +++++++++++-- .../ReceiveFrameMetaDataRenderer.java | 16 +++++++++++++--- .../ScreenshotVideoRenderer.java | 14 ++++++++++++-- .../livephotocapture/ScreenshotVideoRenderer.kt | 11 +++++++++++ 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java b/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java index 8aac9242..1373e229 100644 --- a/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java +++ b/Basic-Video-Driver-Java/app/src/main/java/com/tokbox/sample/basicvideodriver/InvertedColorsVideoRenderer.java @@ -225,6 +225,16 @@ void updateTextures(Frame frame) { int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; + ByteBuffer bb = frame.getBuffer(); + bb.clear(); + //check if buffer data is correctly sized. + if (bb.remaining() != frame.getYplaneSize() + frame.getUVplaneSize() * 2) { + textureWidth = 0; + textureHeight = 0; + return; + } + + bb.position(0); GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); @@ -234,11 +244,11 @@ void updateTextures(Frame frame) { GLES20.glActiveTexture(GLES20.GL_TEXTURE1); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); GLES20.glActiveTexture(GLES20.GL_TEXTURE2); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } @Override diff --git a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java index a22a4a44..7f0c2a5d 100644 --- a/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java +++ b/Basic-Video-Renderer-Java/app/src/main/java/com/tokbox/sample/basicvideorenderer/InvertedColorsVideoRenderer.java @@ -225,6 +225,15 @@ void updateTextures(Frame frame) { int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; + ByteBuffer bb = frame.getBuffer(); + bb.clear(); + //check if buffer data is correctly sized. + if (bb.remaining() != frame.getYplaneSize() + frame.getUVplaneSize() * 2) { + textureWidth = 0; + textureHeight = 0; + return; + } + bb.position(0); GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); @@ -234,11 +243,11 @@ void updateTextures(Frame frame) { GLES20.glActiveTexture(GLES20.GL_TEXTURE1); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); GLES20.glActiveTexture(GLES20.GL_TEXTURE2); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } @Override diff --git a/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java b/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java index 8f1c3835..90eaba7b 100644 --- a/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java +++ b/Frame-Metadata-Java/app/src/main/java/com/tokbox/sample/framemetadata/ReceiveFrameMetaDataRenderer.java @@ -214,13 +214,23 @@ void GlTexSubImage2D(int width, int height, int stride, } } } - + void updateTextures(Frame frame) { int width = frame.getWidth(); int height = frame.getHeight(); int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; + ByteBuffer bb = frame.getBuffer(); + bb.clear(); + //check if buffer data is correctly sized. + if (bb.remaining() != frame.getYplaneSize() + frame.getUVplaneSize() * 2) { + textureWidth = 0; + textureHeight = 0; + return; + } + + bb.position(0); GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); @@ -230,11 +240,11 @@ void updateTextures(Frame frame) { GLES20.glActiveTexture(GLES20.GL_TEXTURE1); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); GLES20.glActiveTexture(GLES20.GL_TEXTURE2); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } @Override diff --git a/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java b/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java index 7374a241..3360dcf8 100644 --- a/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java +++ b/Live-Photo-Capture-Java/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.java @@ -221,6 +221,16 @@ void updateTextures(Frame frame) { int half_width = (width + 1) >> 1; int half_height = (height + 1) >> 1; + ByteBuffer bb = frame.getBuffer(); + bb.clear(); + //check if buffer data is correctly sized. + if (bb.remaining() != frame.getYplaneSize() + frame.getUVplaneSize() * 2) { + textureWidth = 0; + textureHeight = 0; + return; + } + + bb.position(0); GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1); @@ -230,11 +240,11 @@ void updateTextures(Frame frame) { GLES20.glActiveTexture(GLES20.GL_TEXTURE1); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[1]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getUplane()); GLES20.glActiveTexture(GLES20.GL_TEXTURE2); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureIds[2]); - GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); + GlTexSubImage2D(half_width, half_height, frame.getUvStride(), frame.getVplane()); } @Override diff --git a/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt b/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt index 60a8810d..add4bcea 100644 --- a/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt +++ b/Live-Photo-Capture-Kotlin/app/src/main/java/com/tokbox/sample/livephotocapture/ScreenshotVideoRenderer.kt @@ -150,6 +150,17 @@ void main(void) { val half_width = width + 1 shr 1 val half_height = height + 1 shr 1 + val bb = frame.buffer + // If we are reusing this frame, make sure we reset position and + // limit + bb.clear() + if (bb.remaining() != frame.getYplaneSize() + frame.getUVplaneSize() * 2) { + textureWidth = 0 + textureHeight = 0 + return + } + + bb.position(0) GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1) GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1)