nvidia xavier nx平台OpenCV对接硬件编码器

1. 前言

尝试在xavier nx平台做图像处理应用
应用摘要如下

1.捕捉摄像头设备
2.颜色转换格式
3.图像处理
4.编码与HW编码器

1~3使用OpenCV实现
我想实现的是通过图像处理的结果(例如。覆盖图像与边框)到HW编码器。
我引用了jetson_multimedia_api_reference，但是混淆了

HW: JetsonNX开发工具包，SW: Jetpack4.4

2. 参考python示例 & patch

import sys
import cv2

def read_cam():
    cap = cv2.VideoCapture("filesrc location=/home/nvidia/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! nvvidconv ! video/x-raw, format=BGRx ! videoconvert ! video/x-raw,format=BGR ! appsink  ")

    w = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
    h = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
    fps = cap.get(cv2.CAP_PROP_FPS)
    print('Src opened, %dx%d @ %d fps' % (w, h, fps))

    gst_out = "appsrc ! video/x-raw, format=BGR ! queue ! videoconvert ! video/x-raw,format=BGRx ! nvvidconv ! nvv4l2h264enc ! h264parse ! matroskamux ! filesink location=test.mkv "
    out = cv2.VideoWriter(gst_out, cv2.CAP_GSTREAMER, 0, float(fps), (int(w), int(h)))
    if not out.isOpened():
        print("Failed to open output")
        exit()

    if cap.isOpened():
        while True:
            ret_val, img = cap.read();
            if not ret_val:
                break;
            out.write(img);
            cv2.waitKey(1)
    else:
     print "pipeline open failed"

    print("successfully exit")
    cap.release()
    out.release()

if __name__ == '__main__':
    read_cam()

但现在需要拿编码器输出的数据进行后期处理
考虑使用多媒体api

参考以下patch

diff --git a/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp b/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp
index 49a9ab8..0613f0b 100644
--- a/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp
+++ b/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp
@@ -39,6 +39,8 @@
 #include <stdio.h>
 #include <stdlib.h>

+#include <opencv2/opencv.hpp>
+
 using namespace Argus;
 using namespace EGLStream;

@@ -259,7 +261,7 @@ bool ConsumerThread::threadInitialize()
     input_params.width = STREAM_SIZE.width();
     input_params.height = STREAM_SIZE.height();
     input_params.layout = NvBufferLayout_Pitch;
-    input_params.colorFormat = NvBufferColorFormat_NV12;
+    input_params.colorFormat = NvBufferColorFormat_ABGR32;
     input_params.nvbuf_tag = NvBufferTag_VIDEO_CONVERT;

     NvBufferCreateEx (&m_compositedFrame, &input_params);
@@ -333,8 +335,8 @@ bool ConsumerThread::threadExecute()
             if (!m_dmabufs[i])
             {
                 m_dmabufs[i] = iNativeBuffer->createNvBuffer(iEglOutputStreams[i]->getResolution(),
-                                                          NvBufferColorFormat_YUV420,
-                                                          NvBufferLayout_BlockLinear);
+                                                          NvBufferColorFormat_ABGR32,
+                                                          NvBufferLayout_Pitch);
                 if (!m_dmabufs[i])
                     CONSUMER_PRINT("\tFailed to create NvBuffer\n");
             }
@@ -349,7 +351,21 @@ bool ConsumerThread::threadExecute()
         {
             // Composite multiple input to one frame
             NvBufferComposite(m_dmabufs, m_compositedFrame, &m_compositeParam);
-            g_renderer->render(m_compositedFrame);
+            void *pdata = NULL;
+
+            NvBufferMemMap(m_compositedFrame, 0, NvBufferMem_Read, &pdata);
+            NvBufferMemSyncForCpu(m_compositedFrame, 0, &pdata);
+
+            cv::Mat imgbuf = cv::Mat(STREAM_SIZE.height(),
+                                     STREAM_SIZE.width(),
+                                     CV_8UC4, pdata);
+            cv::Mat display_img;
+            cvtColor(imgbuf, display_img, CV_RGBA2BGR);
+
+            NvBufferMemUnMap(m_compositedFrame, 0, &pdata);
+
+            cv::imshow("img", display_img);
+            cv::waitKey(1);
         }
         else
             g_renderer->render(m_dmabufs[0]);

Makefile:

+CPPFLAGS+=`pkg-config --cflags opencv`
+LDFLAGS+=`pkg-config --libs opencv`

3. OpenCV安装方法

1不要通过Jetpack安装OpenCV 3.3.1。默认安装。请取消勾选OpenCV 3.3.1
2 Jetpack将禁止安装样例包，如果你不选中OpenCV，请从
https://developer.nvidia.com/embedded/dlc/multimedia-api-r2821 2
3获取脚本https://github.com/AastaNV/JEP/blob/master/script/install_opencv3.4.0_TX2.sh
4执行脚本
mkdir OpenCV
$ ./install_opencv3.4.0_TX2.sh OpenCV

5应用补丁并重建09_camera_jpeg_capture
6运行
$ export DISPLAY=:0
09_camera_jpeg_capture$ ./camera_jpeg_capture --disable-jpg --cap-time 10

4. V4L2架构参考sample

参考12_camera_v4l2_cuda。
该示例用于v4l2采集

5. gstreamer示例

在opencv中，可以像VideoWriter一样使用gstreamer管道

/ Get resolution and framerate from capture
unsigned int width = cap.get (cv::CAP_PROP_FRAME_WIDTH);
unsigned int height = cap.get (cv::CAP_PROP_FRAME_HEIGHT);
unsigned int fps = cap.get (cv::CAP_PROP_FPS);

// Create the writer with gstreamer pipeline encoding into H264, muxing into mkv container and saving to file 
cv::VideoWriter gst_nvh264_writer("appsrc ! queue ! videoconvert ! video/x-raw,format=BGRx ! nvvidconv ! nvv4l2h264enc ! video/x-h264,format=byte-stream ! h264parse ! matroskamux ! filesink location=test-nvh264-writer.mkv ", 0, fps, cv::Size (width, height));
  if (!gst_nvh264_writer.isOpened ()) {
    std::cout << "Failed to open gst_nvh264 writer." << std::endl;
    return (-6);
  }

在循环中，使用以下命令推送处理过的帧(每个捕获帧一个):

gst_nvh264_writer.write(frame);

6. NvVideoEncoder

考虑在MMAPI中使用NvVideoEncoder
需要创建NvBuffer和复制cv::Mat数据到它。

方法如下:

1. 创建导入cv::Mat(ARGB32)的NvBuffer
2. 为YUV420M创建匹配编码器 输入格式的NvBuffer。
3. 创建NvVideoEncoder
4. 编码器输出平面缓冲区的出队列
5. 将ARGB转换为YUV420M

尝试将cv::Mat转换成NvBuffer
如果par.num_planes = 1，这是正确的
但这种方法似乎是无效的

cv::Mat src = cv::Mat::zeros(height, width, CV_8UC4);
ret = NvBufferGetParams(fd, par);    //fd is V4L2_PIX_FMT_ABGR32
for( unsigned int plane = 0; plane < par.num_planes; plane++){
    ret = NvBufferMemMap(fd, plane, NvBufferMem_Write, &vaddr);  // vaddr is void*
    if(ret == 0){
        for( unsigned int i = 0; i < par.height[plane]; i++){
            memcpy((uint8_t *)vaddr + i *par.pitch[plane], src.data[ i * src.step], src.width * sizeof((uint8_t)) * 4);
        }
    }
    NvBufferMemSyncForDevice (fd, plane, &vaddr);
}

7. CUDA过滤器

因为OpenCV的主要格式是BGR，在Jetson平台上并不支持BGR。
在处理之后，需要将其转换为RGBA，复制到NvBuffer，转换到YUV420并进行编码。
它可能不会带来好的性能。

有CUDA过滤器可以应用于RGBA缓冲区。
如果能在用例中使用CUDA过滤器，性能会更好

函数调用如下:

  //CUDA postprocess
    {
        EGLImageKHR egl_image;
        egl_image = NvEGLImageFromFd(egl_display, dmabuf_fd);
        CUresult status;
        CUeglFrame eglFrame;
        CUgraphicsResource pResource = NULL;
        cudaFree(0);
        status = cuGraphicsEGLRegisterImage(&pResource,
                    egl_image,
                    CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
        if (status != CUDA_SUCCESS)
        {
            printf("cuGraphicsEGLRegisterImage failed: %d \n",status);
        }
        status = cuGraphicsResourceGetMappedEglFrame(&eglFrame, pResource, 0, 0);
        status = cuCtxSynchronize();
        if (create_filter) {
            filter = cv::cuda::createSobelFilter(CV_8UC4, CV_8UC4, 1, 0, 3, 1, cv::BORDER_DEFAULT);
            //filter = cv::cuda::createGaussianFilter(CV_8UC4, CV_8UC4, cv::Size(31,31), 0, 0, cv::BORDER_DEFAULT);
            create_filter = false;
        }
        cv::cuda::GpuMat d_mat(h, w, CV_8UC4, eglFrame.frame.pPitch[0]);
        filter->apply (d_mat, d_mat);

        status = cuCtxSynchronize();
        status = cuGraphicsUnregisterResource(pResource);
        NvDestroyEGLImage(egl_display, egl_image);
    }

dmabuf_fd是RGBA格式。
经过处理后，可以将其转换回NV12并发送给硬件编码器。