【问题标题】:How to Remap a YUY2 image?如何重新映射 YUY2 图像?
【发布时间】:2021-04-29 19:38:21
【问题描述】:

我有一个 YUY2 数据形式的扭曲图像,YUY2 属于 YUV 4:2:2(不是 4:2:0)家族。

我有 mapxmapy (height-720, width-1280),我从

cv::fisheye::initUndistortRectifyMap(K, D, cv::Mat::eye(3, 3, CV_64F), new_K, Size, CV_32FC1, mapx, mapy);

我怎样才能获得不失真的 YUY2? 我的最终目标是拥有未失真的 YUY2(不是 BGR)。

我想执行以下步骤:

cv::cvtColor(YUY, BGR, cv::COLOR_YUV2BGR_YUY2);
\\ then perform remapping
\\ and convert back to YUY

但是没有从 BGR2YUY_YUY2 转换。

有什么更聪明的方法吗?

【问题讨论】:

  • 格式不用转成BGR,只需要转成YUV 4:4:4,不失真的4:4:4,(再转回4:2 :2).
  • @Rotem,如何将 YUV422 转换为 YUV444 并返回?
  • 我想过将 YUY2 转换为平面格式(Y、U 和 V),并调整 U 和 V 的大小。这在 Python 中很简单,但我找不到使用 OpenCV 的好例子。调整大小很简单,但转换为平面需要复制并跳过列。最好的OpenCV solution 我能找到看起来很糟糕。使用 for 循环就很简单了。
  • @Rotem,所以您的意思是不转换为 YUV444?如果是的话,那么我也想过把它分成不同的平面。让我并排尝试这种方法。谢谢..
  • @Rotem,我在想如果我将YUV422图像作为单平面图像(2560x720),如图所示。并以某种方式将地图转换为(2560x720)。这样重新映射功能就可以按原样使用。如果地图准备好了,也可以实现实时性能。你说什么?

标签: c++ opencv yuv remap color-conversion


【解决方案1】:

您可以从 YUV 4:2:2 转换为 YUV 4:4:4,不扭曲 4:4:4,然后转换回 4:2:2。

转换阶段说明:
YUV422 -> YUV444 -> remap(YUV444) -> YUV422

我找不到用于从 YUV 4:2:2 转换为 YUV 4:4:4 的 OpenCV 函数。

通过简单的 for 循环实现转换非常简单:

//Convert YUYV to YUV (y,u,v,y,u,v,y,u,v...)
//The conversion is performed by duplicating each U and V element twice (equivalent to resize with nearest neighbor interpolation).
//The input type is CV_8UC1 (considered to be Grayscale image).
//The output type is CV_8UC3 (considered to be colored image with 3 channels).
static cv::Mat convertYuyv422toYuv444(const cv::Mat yuyv)
{
    int rows = yuyv.rows;
    int src_cols = yuyv.cols;
    size_t src_step = yuyv.step;
    const unsigned char *I = (unsigned char*)yuyv.data; //Pointer to source image.

    int dst_cols = src_cols / 2;
    cv::Mat yuv = cv::Mat(rows, dst_cols, CV_8UC3);
    size_t dst_step = yuv.step;
    unsigned char *J = (unsigned char*)yuv.data;    //Pointer to destination image.

    for (int y = 0; y < rows; y++)
    {
        const unsigned char *I0 = I + y*src_step;   //Points the beginning for the source row.
        unsigned char *J0 = J + y*dst_step;         //Points the beginning for the destination row.

        int srcx = 0;
        int dstx = 0;

        //yuyv -> yuvyuv
        //Convert 2 pixels per iteration
        for (int x = 0; x < src_cols / 2; x += 2)
        {
            unsigned char y0 = I0[srcx];
            unsigned char u0 = I0[srcx + 1];
            unsigned char y1 = I0[srcx + 2];
            unsigned char v0 = I0[srcx + 3];

            J0[dstx] = y0;
            J0[dstx + 1] = u0;
            J0[dstx + 2] = v0;
            J0[dstx + 3] = y1;
            J0[dstx + 4] = u0;  //Duplicate U
            J0[dstx + 5] = v0;  //Duplicate V

            srcx += 4;  //Source has 2 elements per pixel
            dstx += 6;  //Destination has 3 elements per pixel
        }
    }

    return yuv;
}

转换只是将每个 U 和 V 元素复制两次。
这不是最好的方法,但它被认为已经足够好了。
复制 U 和 V 相当于使用最近邻插值调整大小。


要将 YUV 4:4:4 转换回 YUV 4:2:2,您可以使用以下帖子中的代码示例:
Convert YUV4:4:4 to YUV4:2:2 images


已有支持各种颜色格式转换的优化库。

例如libswscale,但我认为这对您的需求来说有点过头了......


测试:

为了测试,我使用了您之前帖子中的输入(附上我的回答):
How to undistort I420 image data? Efficiently

由于我没有 YUYV 图像,所以我使用FFmpeg(命令行)来创建一个:

ffmpeg -i input_image.jpg -codec rawvideo -pix_fmt yuyv422 input_image_yuyv.yuv

我使用 MATLAB 代码将原始 input_image_yuyv.yuv 转换为 PNG。

MATLAB 实现以两种方式将 4:2:2 转换为 4:4:4,并验证复制 U 和 V 等效于使用最近邻插值调整大小。

MATLAB 代码还用于验证 C++ 实现的正确性。

I = imread('input_image.jpg');
[rows, cols, ch] = size(I); % rows = 1280, cols = 720

% Read the YUYV to 2560x720 matrix from a binary file
f = fopen('input_image_yuyv.yuv', 'r');
YUYV = fread(f, [cols*2, rows], '*uint8')';
fclose(f);

% Write YUYV to PNG image - to be used as C++ input.
imwrite(YUYV, 'YUYV.png');

%figure;imshow(YUYV);title('YUYV');impixelinfo
Y = YUYV(:, 1:2:end); % 1280x720
U = YUYV(:, 2:4:end); % 640x720
V = YUYV(:, 4:4:end); % 640x720

% figure;imshow(Y);title('in Y');impixelinfo
% figure;imshow(U);title('in U');impixelinfo
% figure;imshow(V);title('in V');impixelinfo

% Convert U and V to 4:4:4 format using imresize with Nearest Neighbor interpolation method (used as reference).
refU2 = imresize(U, [rows, cols], 'nearest');
refV2 = imresize(V, [rows, cols], 'nearest');
% figure;imshow(U2);title('reference inU full');impixelinfo
% figure;imshow(V2);title('reference inV full');impixelinfo

% Resize x2 in the horizontal axis by simple duplication:
U2 = zeros(rows, cols, 'uint8');
U2(:, 1:2:end) = U;
U2(:, 2:2:end) = U;
V2 = zeros(rows, cols, 'uint8');
V2(:, 1:2:end) = V;
V2(:, 2:2:end) = V;

% Verify that the simple duplication is equivalent to resize with Nearest Neighbor interpolation:
% display(isequal(U2, refU2) && isequal(V2, refV2)) % Equal!!!

% Build YUV444 3840x720 matrix:
YUV444 = zeros(rows, cols*3, 'uint8');
YUV444(:, 1:3:end) = Y;
YUV444(:, 2:3:end) = U2;
YUV444(:, 3:3:end) = V2;
%figure;imshow(YUV444);title('YUV444');impixelinfo

% Write the YUV444 image to binary file (used as reference for C++ implementation)
f = fopen('image_yuv444.yuv', 'w');
fwrite(f, YUV444', 'uint8');
fclose(f);

imwrite(YUV444, 'matlabYUV444.png');


% Test output (after executing C++ code).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
c_YUV444 = imread('yuv444.png');
display(isequal(YUV444, c_YUV444));
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

以下代码是C++实现的main()

int main()
{
    cv::Mat yuyv = cv::imread("YUYV.png", cv::IMREAD_GRAYSCALE);    //Read YUYV.png (created using MATLAB) as Grayscale

    cv::Mat yuv = convertYuyv422toYuv444(yuyv); //Convet yuyv to yuv (y,u,v,y,u,v...)

    //cv::imshow("yuyv", yuyv);
    cv::imwrite("yuv444.png", yuv); //Store YUV image for testing.

    //https://stackoverflow.com/questions/59876539/how-to-undistort-i420-image-data-efficiently
    //remap the YUV 4:4:4
    ///////////////////////////////////////////////////////////////////////////////
    int W = 1280, H = 720;  //Assume resolution of Y plane is 1280x720
    cv::Mat mapx;
    cv::Mat mapy;
    cv::Mat dst_yuv;

    cv::Matx33d K = cv::Matx33d(541.2152931632737, 0.0, 661.7479652584254,
        0.0, 541.0606969363056, 317.4524205037745,
        0.0, 0.0, 1.0);
    cv::Vec4d D = cv::Vec4d(-0.042166406281296365, -0.001223961942208027, -0.0017036710622692108, 0.00023929900459453295);
    cv::Size newSize = cv::Size(3400, 1940);
    cv::Matx33d new_K;
    cv::fisheye::estimateNewCameraMatrixForUndistortRectify(K, D, cv::Size(W, H), cv::Mat::eye(3, 3, CV_64F), new_K, 1, newSize);    // W,H are the distorted image size
    cv::fisheye::initUndistortRectifyMap(K, D, cv::Mat::eye(3, 3, CV_64F), new_K, newSize, CV_16SC2, mapx, mapy);

    cv::remap(yuv, dst_yuv, mapx, mapy, cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0, 128, 128));
    ///////////////////////////////////////////////////////////////////////////////

    //Convert for BGR - just for dispaly
    cv::Mat dst_bgr;
    cv::cvtColor(dst_yuv, dst_bgr, cv::COLOR_YUV2BGR);

    cv::imshow("yuv", yuv);
    cv::imshow("dst_yuv", dst_yuv);
    cv::imshow("dst_bgr", dst_bgr);
    cv::waitKey(0);
    cv::destroyAllWindows();

    cv::imwrite("dst_bgr.png", dst_bgr);    //Store BGR image for testing.

    return 0;
}

注意:
remapcv::BORDER_CONSTANTcv::Scalar(0, 128, 128) 一起使用:

cv::remap(yuv, dst_yuv, mapx, mapy, cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0, 128, 128));

C++ 结果(转换为 BGR 后):

【讨论】:

  • 非常感谢您提供如此细致的编译帮助。这些功能是有效的。由于我的目标是实时应用程序,我正在寻找精简的方法。让我看看是否可以在 mapx 和 mapy 中进行一些修改以使其适合 YUV422。如果我有地图,执行将是有效的。我不知道为什么 opencv 没有 BGR2YUV_YUY2 转换。如果可以,那么您的方法对我来说就足够了。
【解决方案2】:

我尝试在 mapx 和 mapy 中进行修改以使其适合 YUV422。结果在计算时间方面非常好。只需要实时重新映射一次。但质量不是最好的。

然后我通过 libswscale 尝试了 YUV422 -> YUV444 -> remap(YUV444) -> YUV422,但 YUV 转换又需要时间。

最后,我开发了用于 YUV 转换的 cuda 内核。我附在下面。

//  nvcc -c -o colorConversion.o colorConversion.cu `pkg-config --libs --cflags opencv4`
//  /usr/bin/g++ -g -O3 /home/jai/vscode/opencvCUDA/cuda3.cpp -o /home/jai/vscode/opencvCUDA/cuda3 colorConversion.o `pkg-config --libs --cflags opencv4` `pkg-config --libs --cflags gstreamer-1.0` `pkg-config --libs --cflags cuda-11.3` `pkg-config --libs --cflags cudart-11.3`

#include "colorConversion.h"

__global__ void kernel_YUY422toYUY(cv::cuda::PtrStepSz<uchar2> YUV422, cv::cuda::PtrStepSz<uchar3> YUV)
{
    int i = blockIdx.y;         // row
    int j = blockDim.x * blockIdx.x + threadIdx.x; // col
    if (threadIdx.x & 1) {  // odd  1,3,5
      // YUV[i * step3 + 3 * j]     = YUV422[i * step2 + 2 * j];         // Y0
      // YUV[i * step3 + 3 * j + 1] = YUV422[i * step2 + 2 * j - 1]; // Y0
      // YUV[i * step3 + 3 * j + 2] = YUV422[i * step2 + 2 * j + 1]; // Y0
      YUV(i, j).x = YUV422(i, j).x;
      YUV(i, j).y = YUV422(i, j - 1).y;
      YUV(i, j).z = YUV422(i, j).y;
    } else {    // even 0,2,4,
      // YUV[i * step3 + 3 * j]     = YUV422[i * step2 + 2 * j];         // Y0
      //  YUV[i * step3 + 3 * j + 1] = YUV422[i * step2 + 2 * j + 1]; // U0
       // YUV[i * step3 + 3 * j + 2] = YUV422[i * step2 + 2 * j + 3]; // V0
       YUV(i, j).x = YUV422(i, j).x;
       YUV(i, j).y = YUV422(i, j).y;
       YUV(i, j).z = YUV422(i, j+1).y;
    }
}

void YUY422toYUY(const cv::cuda::GpuMat &YUV422gpu, cv::cuda::GpuMat &YUVgpu)
{
    kernel_YUY422toYUY<<<dim3(2, YUVgpu.rows), dim3(YUVgpu.cols / 2)>>>(YUV422gpu, YUVgpu);
    //cudaSafeCall(cudaGetLastError());
}

__global__ void kernel_YUYtoYUY422(cv::cuda::PtrStepSz<uchar3> YUV, cv::cuda::PtrStepSz<uchar2> YUV422)
{
    int i = blockIdx.x;         // row
    int j = threadIdx.x*2; // col

    YUV422(i, j).x = YUV(i, j).x;
    YUV422(i, j).y = (YUV(i, j).y + YUV(i, j+1).y)/2;

    YUV422(i, j+1).x = YUV(i, j+1).x;
    YUV422(i, j+1).y = (YUV(i, j).z + YUV(i, j+1).z)/2;
}

void YUYtoYUY422(const cv::cuda::GpuMat &YUVgpu, cv::cuda::GpuMat &YUV422gpu)
{
    kernel_YUYtoYUY422<<<dim3(YUV422gpu.rows), dim3(YUV422gpu.cols / 2)>>>(YUVgpu, YUV422gpu);
    //cudaSafeCall(cudaGetLastError());
}

然后我使用以下代码行再次使用 CUDA 重新映射:

YUV422GPU.upload(YUV422);  // YUV422 #channel = 2
YUV1.create(H, W, CV_8UC3);
YUV2.create(H, W, CV_8UC3);

YUY422toYUY(YUV422GPU, YUV1);
cv::cuda::remap(YUV1, YUV2, mapxGPU, mapyGPU, interpolationMethod); // YUV remap
YUYtoYUY422(YUV2, YUV422GPU);
YUV422GPU.download(dst);    // dst is the final YUV422. 2 channel image

【讨论】:

    猜你喜欢
    • 1970-01-01
    • 1970-01-01
    • 1970-01-01
    • 2019-07-28
    • 2012-08-09
    • 1970-01-01
    • 1970-01-01
    • 1970-01-01
    • 1970-01-01
    相关资源
    最近更新 更多