在 Java Camera2 API 中使用 OpenCV

Question

我正在使用 Android OpenCV 构建实时对象检测应用程序。我正在使用带有 TextureView 的 Android Camera2 API 来捕获图像。我想添加 OpenCV 代码来做一些实时图像处理并预览结果。

这是我的拍照代码

 protected void takePicture() {
    if(null == cameraDevice) {
        Log.e(TAG, "cameraDevice is null");
        return;
    }
    CameraManager manager = (CameraManager) getSystemService(Context.CAMERA_SERVICE);
    try {
        CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraDevice.getId());
        Size[] jpegSizes = null;
        if (characteristics != null) {
            jpegSizes = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP).getOutputSizes(ImageFormat.JPEG);
        }
        int width = 640;
        int height = 480;
        if (jpegSizes != null && 0 < jpegSizes.length) {
            width = jpegSizes[0].getWidth();
            height = jpegSizes[0].getHeight();
        }
        ImageReader reader = ImageReader.newInstance(width, height, ImageFormat.JPEG, 1);
        List<Surface> outputSurfaces = new ArrayList<Surface>(2);
        outputSurfaces.add(reader.getSurface());
        outputSurfaces.add(new Surface(textureView.getSurfaceTexture()));
        final CaptureRequest.Builder captureBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
        captureBuilder.addTarget(reader.getSurface());
        captureBuilder.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO);
        // Orientation
        int rotation = getWindowManager().getDefaultDisplay().getRotation();
        captureBuilder.set(CaptureRequest.JPEG_ORIENTATION, ORIENTATIONS.get(rotation));
        String timeStamp = new SimpleDateFormat("ddMMyyyy_HHmmss").format(new Date());
        final File file = new File(Environment.getExternalStorageDirectory()+"/Billboard_" + timeStamp + ".jpg");

        // get the location from the NetworkProvider
        LocationManager lm = (LocationManager) this.getSystemService(Context.LOCATION_SERVICE);
        LocationListener locationListener = new LocationListener() {

            @Override
            public void onLocationChanged(Location location) {
                longitude = location.getLongitude();
                latitude = location.getLatitude();

                storeGeoCoordsToImage(file, location);

                Log.e(TAG, "Latitude = " + latitude);
                Log.e(TAG, "Longitude = " + longitude);
            }

            @Override
            public void onProviderDisabled(String provider) {}

            @Override
            public void onProviderEnabled(String provider) {}

            @Override
            public void onStatusChanged(String provider, int status,Bundle extras) {}

        };
        // update location listener
        lm.requestLocationUpdates(LocationManager.NETWORK_PROVIDER, 0, 0, locationListener);

        ImageReader.OnImageAvailableListener readerListener = new ImageReader.OnImageAvailableListener() {
            @Override
            public void onImageAvailable(ImageReader reader) {
                Image image = null;
                try {
                    image = reader.acquireLatestImage();

                    ByteBuffer buffer = image.getPlanes()[0].getBuffer();
                    byte[] bytes = new byte[buffer.capacity()];
                    buffer.get(bytes);

                    save(bytes);
                } catch (FileNotFoundException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    if (image != null) {
                        image.close();
                    }
                }
            }
            private void save(byte[] bytes) throws IOException {
                OutputStream output = null;
                try {
                    output = new FileOutputStream(file);
                    output.write(bytes);
                } finally {
                    if (null != output) {
                        output.close();
                    }
                }
            }
        };
        reader.setOnImageAvailableListener(readerListener, mBackgroundHandler);
        final CameraCaptureSession.CaptureCallback captureListener = new CameraCaptureSession.CaptureCallback() {
            @Override
            public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result) {
                super.onCaptureCompleted(session, request, result);
                Toast.makeText(MainActivity.this, "Saved:" + file, Toast.LENGTH_SHORT).show();
            createCameraPreview();
        }
    };
    cameraDevice.createCaptureSession(outputSurfaces, new CameraCaptureSession.StateCallback() {
        @Override
        public void onConfigured(CameraCaptureSession session) {
            try {
                session.capture(captureBuilder.build(), captureListener, mBackgroundHandler);
            } catch (CameraAccessException e) {
                e.printStackTrace();
            }
        }
        @Override
        public void onConfigureFailed(CameraCaptureSession session) {
        }
    }, mBackgroundHandler);
} catch (CameraAccessException e) {
    e.printStackTrace();
}

}

我想像这样添加一些 Java OpenCV 代码并在屏幕上预览结果。

Mat destination = new Mat();
Imgproc.cvtColor(source, destination, Imgproc.COLOR_RGB2GRAY);
Imgproc.equalizeHist(destination, destination);
Imgproc.canny(destination, destination, 50, 150);

我很困惑如何从相机预览中获取图像并对其进行一些图像处理，然后显示结果。

任何有关 OpenCV 和 Camera2 API 代码集成的帮助都会有所帮助。 谢谢。

Answer 1

因此，如果您想处理来自相机的照片，您应该：

1. 将图像从 JPEG 转换为 ARGB8888 格式，然后获取 从中获取图像垫；
2. 使用 OpenCV 处理您的图像
3. 使用listener将处理后的图像发送到UI

例如，你可以这样做：

    // you need to create some interface in your activity to update image on the screen
    // and initialize it before you will use it
    OnImageReadyListener onImageReadyListener = null; 
    //...

private final ImageReader.OnImageAvailableListener onImageAvailableListener = (ImageReader imReader) -> {

    final Image image = imReader.acquireLatestImage();

    // 1st step: convert the image to Mat
    Mat source = ImageConverter.ImageToMat(image);

    // 2nd step: process it with OpenCV
    Mat destination = new Mat();
    Imgproc.cvtColor(source, destination, Imgproc.COLOR_RGB2GRAY);
    Imgproc.equalizeHist(destination, destination);
    Imgproc.canny(destination, destination, 50, 150);

    // 3rd step: publish your result
    if(onImageReadyListener != null)
        onImageReadyListener.getImage(ImageConverter.MatToBitmap(destination));

    image.close();
};

所以，你的监听器可能看起来像这样：

// setup this interface in your activity where you will update your image
public interface OnImageReadyListener {
public getImage(Bitmap image); // then you should override this method
}

ImageConverter 类应该是这样的：

public class ImageConverter {

private static final String TAG = ImageConverter.class.getSimpleName();

// Convert bitmap from JPEG to ARGB8888 format
private static Bitmap JPEGtoARGB8888(Bitmap input ){
    Bitmap output = null;
    int size = input.getWidth() * input.getHeight();
    int[] pixels = new int[size];
    input.getPixels(pixels,0,input.getWidth(),0,0,input.getWidth(),input.getHeight());
    output = Bitmap.createBitmap(input.getWidth(),input.getHeight(), Bitmap.Config.ARGB_8888);
    output.setPixels(pixels, 0, output.getWidth(), 0, 0, output.getWidth(), output.getHeight());
    return output; // ARGB_8888 formated bitmap
}

// Get image Mat from Bitmap
private static Mat BitmapToMat(Bitmap bitmap){
    Bitmap bitmapARGB8888 = JPEGtoARGB8888(bitmap);
    Mat imageMat = new Mat();
    Utils.bitmapToMat(bitmapARGB8888, imageMat);
    return imageMat;
}

// Convert camera Image data to OpenCV image Mat(rix)
public static Mat ImageToMat(Image image){

    // check image
    if(image == null)
        return null;

    // store image to bytes array
    final ByteBuffer buffer = image.getPlanes()[0].getBuffer();
    final byte[] bytes = new byte[buffer.capacity()];
    buffer.get(bytes);

    // get bitmap from bytes and convert it to Mat
    Bitmap bitmap = BitmapFactory.decodeByteArray(bytes, 0, picture.getImageData().length);
    return BitmapToMat(bitmap);
}

// Inverse conversion after image processing to show it on device screen
public static Bitmap MatToBitmap(Mat image){
    Bitmap bitmap = null;
    Mat convertedMat = new Mat (image.height(), image.width(), CvType.CV_8U, new Scalar(4));
    try {
        Imgproc.cvtColor(image, convertedMat, Imgproc.COLOR_GRAY2RGBA, 4);
        bitmap = Bitmap.createBitmap(convertedMat.cols(), convertedMat.rows(), Bitmap.Config.ARGB_8888);
        Utils.matToBitmap(convertedMat, bitmap);
    }
    catch (CvException e){
        Log.d(TAG, e.getMessage());
    }
    return bitmap;
}
}

附：另外我建议阅读this article 以了解如何将 OpenCV 连接到 Android。