Android CameraX Analyzer Image with format YUV_420_888 to OpenCV Mat

Question

使用 Android CameraX Analyzer ImageProxy 在后台使用 ImageReader 和默认的 YUV_420_888 图像格式。

我想在 OpenCV Mat 中转换它，以便在我的分析器中使用 OpenCV：

override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
    try {
      imageProxy.image?.let {
        // ImageProxy uses an ImageReader under the hood:
        // https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
        // That has a default format of YUV_420_888 if not changed that's the default
        // Android camera format.
        // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
        // https://developer.android.com/reference/android/media/ImageReader.html

        // Sanity check
        if (it.format == ImageFormat.YUV_420_888
            && it.planes.size == 3
        ) {
           // TODO - convert ImageProxy.image to Mat
        } else {
          // Manage other image formats
          // TODO - https://developer.android.com/reference/android/media/Image.html
        }
      }
    } catch (ise: IllegalStateException) {
      ise.printStackTrace()
    }
  }

我该怎么做？

Answer 1

查看其 GitHub 存储库中的 OpenCV JavaCamera2Frame 类，您可以像这样编写一个 Image 扩展函数：

（移植到 Kotlin）

// Ported from opencv private class JavaCamera2Frame
fun Image.yuvToRgba(): Mat {
  val rgbaMat = Mat()

  if (format == ImageFormat.YUV_420_888
      && planes.size == 3) {

    val chromaPixelStride = planes[1].pixelStride

    if (chromaPixelStride == 2) { // Chroma channels are interleaved
      assert(planes[0].pixelStride == 1)
      assert(planes[2].pixelStride == 2)
      val yPlane = planes[0].buffer
      val uvPlane1 = planes[1].buffer
      val uvPlane2 = planes[2].buffer
      val yMat = Mat(height, width, CvType.CV_8UC1, yPlane)
      val uvMat1 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane1)
      val uvMat2 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane2)
      val addrDiff = uvMat2.dataAddr() - uvMat1.dataAddr()
      if (addrDiff > 0) {
        assert(addrDiff == 1L)
        Imgproc.cvtColorTwoPlane(yMat, uvMat1, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV12)
      } else {
        assert(addrDiff == -1L)
        Imgproc.cvtColorTwoPlane(yMat, uvMat2, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV21)
      }
    } else { // Chroma channels are not interleaved
      val yuvBytes = ByteArray(width * (height + height / 2))
      val yPlane = planes[0].buffer
      val uPlane = planes[1].buffer
      val vPlane = planes[2].buffer

      yPlane.get(yuvBytes, 0, width * height)

      val chromaRowStride = planes[1].rowStride
      val chromaRowPadding = chromaRowStride - width / 2

      var offset = width * height
      if (chromaRowPadding == 0) {
        // When the row stride of the chroma channels equals their width, we can copy
        // the entire channels in one go
        uPlane.get(yuvBytes, offset, width * height / 4)
        offset += width * height / 4
        vPlane.get(yuvBytes, offset, width * height / 4)
      } else {
        // When not equal, we need to copy the channels row by row
        for (i in 0 until height / 2) {
          uPlane.get(yuvBytes, offset, width / 2)
          offset += width / 2
          if (i < height / 2 - 1) {
            uPlane.position(uPlane.position() + chromaRowPadding)
          }
        }
        for (i in 0 until height / 2) {
          vPlane.get(yuvBytes, offset, width / 2)
          offset += width / 2
          if (i < height / 2 - 1) {
            vPlane.position(vPlane.position() + chromaRowPadding)
          }
        }
      }

      val yuvMat = Mat(height + height / 2, width, CvType.CV_8UC1)
      yuvMat.put(0, 0, yuvBytes)
      Imgproc.cvtColor(yuvMat, rgbaMat, Imgproc.COLOR_YUV2RGBA_I420, 4)
    }
  }

  return rgbaMat
}

所以你可以写：

override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
    try {
      imageProxy.image?.let {
        // ImageProxy uses an ImageReader under the hood:
        // https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
        // That has a default format of YUV_420_888 if not changed that's the default
        // Android camera format.
        // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
        // https://developer.android.com/reference/android/media/ImageReader.html

        // Sanity check
        if (it.format == ImageFormat.YUV_420_888
            && it.planes.size == 3
        ) {        
          val rgbaMat = it.yuvToRgba()
        } else {
          // Manage other image formats
          // TODO - https://developer.android.com/reference/android/media/Image.html
        }
      }
    } catch (ise: IllegalStateException) {
      ise.printStackTrace()
    }
  }

Answer 2

 private Mat convertYUVtoMat(@NonNull Image img) {
    byte[] nv21;

    ByteBuffer yBuffer = img.getPlanes()[0].getBuffer();
    ByteBuffer uBuffer = img.getPlanes()[1].getBuffer();
    ByteBuffer vBuffer = img.getPlanes()[2].getBuffer();

    int ySize = yBuffer.remaining();
    int uSize = uBuffer.remaining();
    int vSize = vBuffer.remaining();

    nv21 = new byte[ySize + uSize + vSize];

    yBuffer.get(nv21, 0, ySize);
    vBuffer.get(nv21, ySize, vSize);
    uBuffer.get(nv21, ySize + vSize, uSize);

    Mat yuv = new Mat(img.getHeight() + img.getHeight()/2, img.getWidth(), CvType.CV_8UC1);
    yuv.put(0, 0, nv21);
    Mat rgb = new Mat();
    Imgproc.cvtColor(yuv, rgb, Imgproc.COLOR_YUV2RGB_NV21, 3);
    Core.rotate(rgb, rgb, Core.ROTATE_90_CLOCKWISE);
    return  rgb;
}

此方法将 Camerax API YUV_420_888 图像转换为 OpenCV 的 Mat (RGB) 对象。 （2021 年工作）

Answer 3

如果您需要获取 OpenCV Mat，@shadowsheep 解决方案就可以了。

但如果您想获取 Bitmap 并且不想将 opencv 库添加到您的项目中，您可以查看 android/camera-samples repo 中的 RenderScript 解决方案

我还在github 创建了单个Java 文件库。如果您想在没有任何行或像素步长的情况下获得正确的 ByteBuffer 以进行进一步处理（例如使用神经网络引擎），这将非常有用。

我也compared 所有这些方法。 OpenCV 是最快的。