[react-native] Add option to TensorCamera to not use custom shaders to resize camera image texture (#5626)

* [react-native] Add option to not use custom shaders to resize camera image texture

* fix

Author: jinjingforever

tfjs-react-native/src/camera/camera.ts

@@ -135,12 +135,15 @@ export async function toTexture(
  * @param texture the texture to convert into a tensor
  * @param sourceDims source dimensions of input texture (width, height, depth)
  * @param targetShape desired shape of output tensor
+ * @param useCustomShadersToResize whether to use custom shaders to resize
+ *   texture.
  *
  * @doc {heading: 'Media', subheading: 'Camera'}
  */
 export function fromTexture(
     gl: WebGL2RenderingContext, texture: WebGLTexture, sourceDims: Dimensions,
-    targetShape: Dimensions, options: FromTextureOptions = {}): tf.Tensor3D {
+    targetShape: Dimensions, useCustomShadersToResize = false,
+    options: FromTextureOptions = {}): tf.Tensor3D {
   tf.util.assert(
       targetShape.depth === 3 || targetShape.depth === 4,
       () => 'fromTexture Error: target depth must be 3 or 4');
@@ -188,7 +191,8 @@ export function fromTexture(
           ' "bilinear" or "nearest_neighbor"');
 
   const resizedTexture = runResizeProgram(
-      gl, texture, sourceDims, targetShape, alignCorners, interpolation);
+      gl, texture, sourceDims, targetShape, alignCorners,
+      useCustomShadersToResize, interpolation);
   const downloadedTextureData =
       downloadTextureData(gl, resizedTexture, targetShape);
 

tfjs-react-native/src/camera/camera_stream.tsx

@@ -34,6 +34,7 @@ interface WrappedComponentProps {
 }
 
 interface Props {
+  useCustomShadersToResize: boolean;
   cameraTextureWidth: number;
   cameraTextureHeight: number;
   resizeWidth: number;
@@ -54,6 +55,7 @@ interface State {
 
 const DEFAULT_AUTORENDER = true;
 const DEFAULT_RESIZE_DEPTH = 3;
+const DEFAULT_USE_CUSTOM_SHADERS_TO_RESIZE = false;
 
 /**
  * A higher-order-component (HOC) that augments the [Expo.Camera](https://docs.expo.io/versions/latest/sdk/camera/)
@@ -70,10 +72,26 @@ const DEFAULT_RESIZE_DEPTH = 3;
  * __In addition to__ all the props taken by Expo.Camera. The returned
  * component takes the following props
  *
+ * - __use_custom_shaders_to_resize__: boolean — whether to use custom shaders
+ *   to resize the camera image to smaller dimensions that fit the output
+ *   tensor.
+ *   - If it is set to false (default and recommended), the resize will be done
+ *     by the underlying GL system when drawing the camera image texture to the
+ *     target output texture with TEXTURE_MIN_FILTER/TEXTURE_MAG_FILTER set to
+ *     gl.LINEAR, and there is no need to provide `cameraTextureWidth` and
+ *     `cameraTextureHeight` props below.
+ *   - If it is set to true (legacy), the resize will be done by the custom
+ *     shaders defined in `resize_bilinear_program_info.ts`. Setting it to true
+ *     also requires that client provide the correct `cameraTextureWidth` and
+ *     `cameraTextureHeight` props below. Unfortunately there is no official API
+ *     to get the camera texture size programmatically so they have to be
+ *     decided empirically. From our experience, it is hard to cover all cases
+ *     in this way because different devices models and/or preview sizes might
+ *     produce different camera texture sizes.
  * - __cameraTextureWidth__: number — the width the camera preview texture
- *   (see example and note below)
+ *   (see note above)
  * - __cameraTextureHeight__: number — the height the camera preview texture
- *   (see example and note below)
+ *   (see note above)
  * - __resizeWidth__: number — the width of the output tensor
  * - __resizeHeight__: number — the height of the output tensor
  * - __resizeDepth__: number — the depth (num of channels) of the output tensor.
@@ -127,31 +145,12 @@ const DEFAULT_RESIZE_DEPTH = 3;
  *   }
  *
  *   render() {
- *    // Currently expo does not support automatically determining the
- *    // resolution of the camera texture used. So it must be determined
- *    // empirically for the supported devices and preview size.
- *
- *    let textureDims;
- *    if (Platform.OS === 'ios') {
- *     textureDims = {
- *       height: 1920,
- *       width: 1080,
- *     };
- *    } else {
- *     textureDims = {
- *       height: 1200,
- *       width: 1600,
- *     };
- *    }
- *
  *    return <View>
  *      <TensorCamera
  *       // Standard Camera props
  *       style={styles.camera}
  *       type={Camera.Constants.Type.front}
  *       // Tensor related props
- *       cameraTextureHeight={textureDims.height}
- *       cameraTextureWidth={textureDims.width}
  *       resizeHeight={200}
  *       resizeWidth={152}
  *       resizeDepth={3}
@@ -251,10 +250,24 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
         resizeHeight,
         resizeWidth,
         resizeDepth,
-        cameraTextureHeight,
-        cameraTextureWidth,
       } = this.props;
 
+      // cameraTextureHeight and cameraTextureWidth props can be omitted when
+      // useCustomShadersToResize is set to false. Setting a default value to
+      // them here.
+      const cameraTextureHeight =
+        this.props.cameraTextureHeight != null
+          ? this.props.cameraTextureHeight
+          : 0;
+      const cameraTextureWidth =
+        this.props.cameraTextureWidth != null
+          ? this.props.cameraTextureWidth
+          : 0;
+      const useCustomShadersToResize =
+        this.props.useCustomShadersToResize != null
+          ? this.props.useCustomShadersToResize
+          : DEFAULT_USE_CUSTOM_SHADERS_TO_RESIZE;
+
       //
       //  Set up a generator function that yields tensors representing the
       // camera on demand.
@@ -279,7 +292,8 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
             gl,
             cameraTexture,
             textureDims,
-            targetDims
+            targetDims,
+            useCustomShadersToResize,
           );
           yield imageTensor;
         }
@@ -329,6 +343,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
       // Use this object to use typescript to check that we are removing
       // all the tensorCamera properties.
       const tensorCameraPropMap: Props = {
+        useCustomShadersToResize: null,
         cameraTextureWidth: null,
         cameraTextureHeight: null,
         resizeWidth: null,

tfjs-react-native/src/camera/camera_test.ts

@@ -66,7 +66,7 @@ describeWithFlags('toTexture', RN_ENVS, () => {
     const texture = await toTexture(gl, inTensor);
 
     const outTensor = fromTexture(
-        gl, texture, {width, height, depth}, {width, height, depth});
+        gl, texture, {width, height, depth}, {width, height, depth}, true);
 
     expectArraysEqual(await inTensor.data(), await outTensor.data());
     expectArraysEqual(inTensor.shape, outTensor.shape);
@@ -168,6 +168,7 @@ describeWithFlags('fromTexture:nearestNeighbor', RN_ENVS, () => {
           width: inShape[1],
           depth: inShape[2],
         },
+        true,
         {
           alignCorners: false,
           interpolation: 'nearest_neighbor',
@@ -192,6 +193,7 @@ describeWithFlags('fromTexture:nearestNeighbor', RN_ENVS, () => {
           width: inShape[1],
           depth: inShape[2],
         },
+        true,
         {
           alignCorners: true,
           interpolation: 'nearest_neighbor',
@@ -231,6 +233,7 @@ describeWithFlags('fromTexture:nearestNeighbor', RN_ENVS, () => {
              width: expectedShape[1],
              depth: expectedShape[2],
            },
+           true,
            {alignCorners: false, interpolation: 'nearest_neighbor'},
        );
 
@@ -267,6 +270,7 @@ describeWithFlags('fromTexture:nearestNeighbor', RN_ENVS, () => {
              width: expectedShape[1],
              depth: expectedShape[2],
            },
+           true,
            {alignCorners: true, interpolation: 'nearest_neighbor'},
        );
 
@@ -304,6 +308,7 @@ describeWithFlags('fromTexture:nearestNeighbor', RN_ENVS, () => {
           width: expectedShape[1],
           depth: expectedShape[2],
         },
+        true,
         {alignCorners: false, interpolation: 'nearest_neighbor'},
     );
 
@@ -342,6 +347,7 @@ describeWithFlags('fromTexture:nearestNeighbor', RN_ENVS, () => {
           width: expectedShape[1],
           depth: expectedShape[2],
         },
+        true,
         {alignCorners: true, interpolation: 'nearest_neighbor'},
     );
 
@@ -393,6 +399,7 @@ describeWithFlags('fromTexture:nearestNeighbor', RN_ENVS, () => {
           width: inShape[1],
           depth: 3,
         },
+        true,
         {
           alignCorners: true,
           interpolation: 'nearest_neighbor',
@@ -465,6 +472,7 @@ describeWithFlags('fromTexture:bilinear', RN_ENVS, () => {
           width: inShape[1],
           depth: inShape[2],
         },
+        true,
         {
           alignCorners: false,
           interpolation: 'bilinear',
@@ -489,6 +497,7 @@ describeWithFlags('fromTexture:bilinear', RN_ENVS, () => {
           width: inShape[1],
           depth: inShape[2],
         },
+        true,
         {
           alignCorners: true,
           interpolation: 'bilinear',
@@ -527,6 +536,7 @@ describeWithFlags('fromTexture:bilinear', RN_ENVS, () => {
           width: expectedShape[1],
           depth: expectedShape[2],
         },
+        true,
         {alignCorners: false, interpolation: 'bilinear'},
     );
 
@@ -562,6 +572,7 @@ describeWithFlags('fromTexture:bilinear', RN_ENVS, () => {
           width: expectedShape[1],
           depth: expectedShape[2],
         },
+        true,
         {alignCorners: true, interpolation: 'bilinear'},
     );
 
@@ -599,6 +610,7 @@ describeWithFlags('fromTexture:bilinear', RN_ENVS, () => {
           width: expectedShape[1],
           depth: expectedShape[2],
         },
+        true,
         {alignCorners: false, interpolation: 'bilinear'},
     );
 
@@ -636,6 +648,7 @@ describeWithFlags('fromTexture:bilinear', RN_ENVS, () => {
           width: expectedShape[1],
           depth: expectedShape[2],
         },
+        true,
         {alignCorners: true, interpolation: 'bilinear'},
     );
 
@@ -687,6 +700,7 @@ describeWithFlags('fromTexture:bilinear', RN_ENVS, () => {
           width: inShape[1],
           depth: 3,
         },
+        true,
         {
           alignCorners: true,
           interpolation: 'bilinear',

tfjs-react-native/src/camera/camera_webgl_util.ts

@@ -151,7 +151,7 @@ export function drawTexture(
     gl: WebGL2RenderingContext, texture: WebGLTexture,
     dims: {width: number, height: number}, flipHorizontal: boolean) {
   const {program, vao, vertices, uniformLocations} =
-      drawTextureProgram(gl, flipHorizontal);
+      drawTextureProgram(gl, flipHorizontal, false);
   gl.useProgram(program);
   gl.bindVertexArray(vao);
 
@@ -176,9 +176,11 @@ export function drawTexture(
 export function runResizeProgram(
     gl: WebGL2RenderingContext, inputTexture: WebGLTexture,
     inputDims: Dimensions, outputDims: Dimensions, alignCorners: boolean,
+    useCustomShadersToResize: boolean,
     interpolation: 'nearest_neighbor'|'bilinear') {
-  const {program, vao, vertices, uniformLocations} =
-      resizeProgram(gl, inputDims, outputDims, alignCorners, interpolation);
+  const {program, vao, vertices, uniformLocations} = useCustomShadersToResize ?
+      resizeProgram(gl, inputDims, outputDims, alignCorners, interpolation) :
+      drawTextureProgram(gl, false, true);
   gl.useProgram(program);
   // Set up geometry
   webgl_util.callAndCheck(gl, () => {
@@ -191,6 +193,12 @@ export function runResizeProgram(
   gl.uniform1i(uniformLocations.get('inputTexture'), 1);
   gl.activeTexture(gl.TEXTURE0 + 1);
   gl.bindTexture(gl.TEXTURE_2D, inputTexture);
+  if (!useCustomShadersToResize) {
+    const textureFilter =
+        interpolation === 'nearest_neighbor' ? gl.NEAREST : gl.LINEAR;
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, textureFilter);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, textureFilter);
+  }
 
   //
   // Set up output texture.
@@ -295,17 +303,18 @@ function createFrameBuffer(gl: WebGL2RenderingContext): WebGLFramebuffer {
   return fb;
 }
 
-function drawTextureProgram(
-    gl: WebGL2RenderingContext, flipHorizontal: boolean): ProgramObjects {
+export function drawTextureProgram(
+    gl: WebGL2RenderingContext, flipHorizontal: boolean,
+    flipVertical: boolean): ProgramObjects {
   if (!programCacheByContext.has(gl)) {
     programCacheByContext.set(gl, new Map());
   }
   const programCache = programCacheByContext.get(gl);
 
-  const cacheKey = `drawTexture_${flipHorizontal}`;
+  const cacheKey = `drawTexture_${flipHorizontal}_${flipVertical}`;
   if (!programCache.has(cacheKey)) {
     const vertSource =
-        drawTextureProgramInfo.vertexShaderSource(flipHorizontal);
+        drawTextureProgramInfo.vertexShaderSource(flipHorizontal, flipVertical);
     const fragSource = drawTextureProgramInfo.fragmentShaderSource();
 
     const vertices = drawTextureProgramInfo.vertices();

tfjs-react-native/src/camera/draw_texture_program_info.ts

@@ -15,8 +15,10 @@
  * =============================================================================
  */
 
-export function vertexShaderSource(flipHorizontal: boolean) {
+export function vertexShaderSource(
+    flipHorizontal: boolean, flipVertical: boolean) {
   const horizontalScale = flipHorizontal ? -1 : 1;
+  const verticalScale = flipVertical ? -1 : 1;
   return `#version 300 es
 precision highp float;
 
@@ -27,7 +29,8 @@ out vec2 uv;
 
 void main() {
   // Invert geometry to match the image orientation from the camera.
-  gl_Position = vec4(position * vec2(${horizontalScale}., -1.), 0, 1);
+  gl_Position = vec4(position * vec2(${horizontalScale}., ${
+      verticalScale}. * -1.), 0, 1);
   uv = texCoords;
 }`;
 }