瑞芯微RV1126——人脸识别源码分析

 VI_CHN_ATTR_S vi_chn_attr;

4、初始化视频处理属性

RGA_ATTR_S stRgaAttr;

5、初始化编码通道属性：

VENC_CHN_ATTR_S venc_chn_attr;

6、绑定数据源：

RK_MPI_SYS_Bind

7、开始捕获码流：

RK_MPI_VI_StartStream

8、执行三个对应的线程：

• pthread_create(&rockx_pid, NULL, rockx_vi_detect_thread, NULL);
• pthread_create(&venc_pid, NULL, rockx_vi_face_recognize_venc_thread, NULL);
• pthread_create(&rtsp_pid, NULL, rockx_venc_rtsp_thread, NULL);

9、销毁申请的系统资源;

3.核心代码分析

初始化vi通道属性；初始化视频处理属性；初始化编码通道属性；绑定数据源；开始捕获码流：

//初始化vi通道属性
  VI_CHN_ATTR_S vi_chn_attr;
  vi_chn_attr.pcVideoNode = pDeviceName;
  vi_chn_attr.u32BufCnt = u32BufCnt;
  vi_chn_attr.u32Width = u32Width;
  vi_chn_attr.u32Height = u32Height;//你的摄像头分辨率大小不要超过venc
  vi_chn_attr.enPixFmt = IMAGE_TYPE_NV12;
  vi_chn_attr.enBufType = VI_CHN_BUF_TYPE_MMAP;
  vi_chn_attr.enWorkMode = VI_WORK_MODE_NORMAL;
  ret = RK_MPI_VI_SetChnAttr(s32CamId, 0, &vi_chn_attr);//设置vi通道属性
  ret |= RK_MPI_VI_EnableChn(s32CamId, 0);//使能vi通道属性，让其生效
  if (ret)
  {
    printf("ERROR: create rkisp0 VI[0] error! ret=%d\n", ret);
    return 0;
  }

  //初始化rga属性
  RGA_ATTR_S stRgaAttr;
  stRgaAttr.bEnBufPool = RK_TRUE;
  stRgaAttr.u16BufPoolCnt = 2;
  stRgaAttr.u16Rotaion = 0;
  stRgaAttr.stImgIn.u32X = 0;
  stRgaAttr.stImgIn.u32Y = 0;
  stRgaAttr.stImgIn.imgType = IMAGE_TYPE_NV12;
  stRgaAttr.stImgIn.u32Width = u32Width;
  stRgaAttr.stImgIn.u32Height = u32Height;
  stRgaAttr.stImgIn.u32HorStride = u32Width;
  stRgaAttr.stImgIn.u32VirStride = u32Height;
  stRgaAttr.stImgOut.u32X = 0;
  stRgaAttr.stImgOut.u32Y = 0;
  stRgaAttr.stImgOut.imgType = IMAGE_TYPE_NV12;
  stRgaAttr.stImgOut.u32Width = disp_width;
  stRgaAttr.stImgOut.u32Height = disp_height;
  stRgaAttr.stImgOut.u32HorStride = disp_width;
  stRgaAttr.stImgOut.u32VirStride = disp_height;
  ret = RK_MPI_RGA_CreateChn(0, &stRgaAttr);//rga通道
  if (ret)
  {
    printf("ERROR: Create rga[0] falied! ret=%d\n", ret);
    return -1;
  }

  //初始化编码属性
  VENC_CHN_ATTR_S venc_chn_attr;
  memset(&venc_chn_attr, 0, sizeof(VENC_CHN_ATTR_S));
  venc_chn_attr.stVencAttr.u32PicWidth = disp_width;
  venc_chn_attr.stVencAttr.u32PicHeight = disp_height;
  venc_chn_attr.stVencAttr.u32VirWidth = disp_width;
  venc_chn_attr.stVencAttr.u32VirHeight = disp_height;
  venc_chn_attr.stVencAttr.imageType = IMAGE_TYPE_NV12;
  venc_chn_attr.stVencAttr.enType = RK_CODEC_TYPE_H264;
  venc_chn_attr.stVencAttr.u32Profile = 66;
  venc_chn_attr.stRcAttr.enRcMode = VENC_RC_MODE_H264CBR;//恒定的编码码率类型
  venc_chn_attr.stRcAttr.stH264Cbr.u32Gop = 30;
  venc_chn_attr.stRcAttr.stH264Cbr.u32BitRate = disp_width * disp_height * 3;
  venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateDen = 1;
  venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateNum = 25;
  venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateDen = 1;
  venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateNum = 25;
  ret = RK_MPI_VENC_CreateChn(0, &venc_chn_attr);//创建编码通道
  if (ret)
  {
    printf("ERROR: Create venc failed!\n");
    exit(0);
  }

  //初始化mpp通道
  MPP_CHN_S vi_chn;
  MPP_CHN_S rga_chn;
  vi_chn.enModId = RK_ID_VI;
  vi_chn.s32ChnId = 0;
  rga_chn.enModId = RK_ID_RGA;
  rga_chn.s32ChnId = 0;
  ret = RK_MPI_SYS_Bind(&vi_chn, &rga_chn);//绑定vi和rga通道
  if (ret != 0)
  {
    printf("[VI] vi id: %d bind venc id: %d, ret: %d error\n", vi_chn.s32ChnId, rga_chn.s32ChnId, ret);
    return -1;
  }

初始化三个线程

  //初始化三个线程id
  pthread_t rockx_pid;
  pthread_t venc_pid;
  pthread_t rtsp_pid;

  //创建人脸检测线程
  pthread_create(&rockx_pid, NULL, rockx_vi_detect_thread, NULL);
  //人脸识别线程
  pthread_create(&venc_pid, NULL, rockx_vi_face_recognize_venc_thread, NULL);
  //人脸编码rtsp传输线程
  pthread_create(&rtsp_pid, NULL, rockx_venc_rtsp_thread, NULL);

人脸检测线程

void *rockx_vi_detect_thread(void *args)
{
  //自动释放线程资源
  pthread_detach(pthread_self());

  //创建一个类对象thread_map
  S_THREAD_MAP thread_map;
  //对thread_map进行初始化
  get_thread_map(0, &thread_map);

  //定义了一个map类型database_face_map对象
  map<string, rockx_face_feature_t> database_face_map = thread_map.thread_map;
  //定义迭代器database_iter
  map<string, rockx_face_feature_t>::iterator database_iter;

  //定义一个缓冲区
  MEDIA_BUFFER src_mb = NULL;
  //人脸模式枚举变量定义
  rockx_module_t data_version;
  data_version = ROCKX_MODULE_FACE_DETECTION_V2;

  //定义一个人脸执行返回结果变量
  rockx_ret_t rockx_ret;
  //定义人脸检测处理指针变量
  rockx_handle_t face_det_handle;
  //定义人脸识别特征提取处理指针变量
  rockx_handle_t face_recognize_handle;
  //定义人脸特征点定位处理指针变量
  rockx_handle_t face_5landmarks_handle;
  //定义了人脸标记检测处理指针变量
  rockx_handle_t face_masks_det_handle;
  //定义人脸配置结构体指针变量
  rockx_config_t *config = rockx_create_config();//获取人脸配置值
  //添加配置人脸模型存放路径，这里是存放在共享目录下：/mnt/nfs/rockx_data/
  rockx_add_config(config, ROCKX_CONFIG_DATA_PATH, "/mnt/nfs/rockx_data/");
  //创建使用人脸模型数据，来处理人脸检测
  rockx_ret = rockx_create(&face_det_handle, data_version, config,
                           sizeof(rockx_config_t));
  //判断是否创建使用人脸模型数据 来处理人脸检测是否成功                         
  if (rockx_ret != ROCKX_RET_SUCCESS)
  {
    printf("init face_detect error %d\n", rockx_ret);
    return NULL;
  }
  //使用人脸模型数据来人脸识别特征提取
  rockx_ret = rockx_create(&face_recognize_handle, ROCKX_MODULE_FACE_RECOGNIZE,
                           config, sizeof(rockx_config_t));
  //识别是否成功                         
  if (rockx_ret != ROCKX_RET_SUCCESS)
  {
    printf("init face_recognize error %d\n", rockx_ret);
    return NULL;
  }
  //使用模型算法数据来做人脸特征点定位处理
  rockx_ret = rockx_create(&face_5landmarks_handle,
                           ROCKX_MODULE_FACE_LANDMARK_5, config, 0);
  //判断是否处理成功                         
  if (rockx_ret != ROCKX_RET_SUCCESS)
  {
    printf("init rockx module ROCKX_MODULE_FACE_LANDMARK_68 error %d\n",
           rockx_ret);
  }

  // rockx_handle_t face_masks_det_handle;进行标记处理
  rockx_ret = rockx_create(&face_masks_det_handle,
                           ROCKX_MODULE_FACE_MASKS_DETECTION, config, 0);
  if (rockx_ret != ROCKX_RET_SUCCESS)
  {
    printf("init rockx module ROCKX_MODULE_FACE_MASKS_DETECTION error %d\n",
           rockx_ret);
  }
  //定义人脸图片结构体变量，并进行成员赋值
  rockx_image_t input_image;
  input_image.width = 1920;
  input_image.height = 1080;
  input_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12;
  bool is_recognize = false;

  string predict = "";

  //rockx_face_result_t face_result;

  int ret;
  //做轮询操作
  while (!quit)
  {
#if 1
    //从指定通道中获取数据缓冲区
    src_mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, 0, -1);
    if (!src_mb)
    {
      printf("ERROR: RK_MPI_SYS_GetMediaBuffer get null buffer!\n");
      break;
    }
    //从指定的MEDIA_BUFFER中获取缓冲区数据大小
    input_image.size = RK_MPI_MB_GetSize(src_mb);
    input_image.data = (unsigned char *)RK_MPI_MB_GetPtr(src_mb);//从指定的MEDIA_BUFFER中获取缓冲区数据指针
#endif

#if 1

    //rockx_face_result_group_t face_result_group;
    //memset(&face_result_group, 0, sizeof(face_result_group));
    //定义人脸处理结果结构体变量
    rockx_object_array_t face_array;
    memset(&face_array, 0, sizeof(face_array));

    //开始人脸检测
    rockx_ret = rockx_face_detect(face_det_handle, &input_image, &face_array, NULL);
    if (rockx_ret != ROCKX_RET_SUCCESS)
    {
      printf("rockx_face_detect ERROR %d\n", rockx_ret);
    }
    //进行互斥处理
    set_rockx_face_array(face_array);
    //判断检测人脸特征数量值是否大于0
    if (face_array.count > 0)
    {
      //rockx_queue->putRockxFaceArray(face_array);
      printf("face_count : %d\n", face_array.count);
      for (int i = 0; i < face_array.count; i++)
      {
        if (1)
        {
          int is_false_face;
          //进行人脸过滤处理
          ret = rockx_face_filter(face_5landmarks_handle, &input_image,
                                  &face_array.object[i].box, &is_false_face);
          if (ret != ROCKX_RET_SUCCESS)
          {
            printf("rockx_face_filter error %d\n", ret);
          }
          if (is_false_face)
            continue;
        }
#if 1
        //人脸检测结果（包括人脸、车牌、头部、物体等）变量定义
        rockx_object_t max_face;
        rockx_object_t cur_face = face_array.object[i];
        //进行人脸区域计算处理操作
        int cur_face_box_area = (cur_face.box.right - cur_face.box.left) *
                                (cur_face.box.bottom - cur_face.box.top);
        int max_face_box_area = (max_face.box.right - max_face.box.left) *
                                (max_face.box.bottom - max_face.box.top);
        if (cur_face_box_area > max_face_box_area)
        {
          max_face = cur_face;
        }

        //检测输出处理
        rockx_image_t out_img;
        memset(&out_img, 0, sizeof(rockx_image_t));
        //进行面部矫正对齐
        ret = rockx_face_align(face_5landmarks_handle, &input_image,
                               &(max_face.box), NULL, &out_img);
        if (ret != ROCKX_RET_SUCCESS)
        {
          printf("face_align failed\n");
        }
        //人脸特征结果变量定义
        rockx_face_feature_t out_feature;
        //获取人脸特征
        rockx_face_recognize(face_recognize_handle, &out_img, &out_feature);

        for (database_iter = database_face_map.begin();
             database_iter != database_face_map.end(); database_iter++)
        {
          float similarity;
          //比较两个人脸特征的相似性
          ret = rockx_face_feature_similarity(&database_iter->second,
                                              &out_feature, &similarity);
          printf("simple_value = %lf\n", similarity);
          //判断预测精度
          if (similarity <= 1.0)
          {
            is_recognize = true;
            //predict_name_bak = database_iter->first;
            predict = database_iter->first;
            break;
          }
          else
          {
            is_recognize = false;
            predict = "";
            continue;
          }
        }
        if (is_recognize == true)
        {
          predict = database_iter->first;
        }
        else
        {
          predict = "";
        }
        set_rockx_prdict_name(predict);

#endif
      }
    }
#endif
    //释放缓冲区
    RK_MPI_MB_ReleaseBuffer(src_mb);
    src_mb = NULL;
  }
  //释放相关人脸处理数据模块
  rockx_destroy(face_det_handle);
  rockx_destroy(face_recognize_handle);
  rockx_destroy(face_5landmarks_handle);

  return NULL;
}

人脸识别线程

void *rockx_vi_face_recognize_venc_thread(void *args)
{
  pthread_detach(pthread_self());//线程资源自动释放

  MEDIA_BUFFER mb = NULL; //媒体缓存区
  int ret;
  float x_rate = (float)1280 / 1920;
  float y_rate = (float)720 / 1080;

  while (!quit)
  {
    //从指定通道中获取数据缓冲区
    mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_RGA, 0, -1);
    if (!mb)
    {
      printf("ERROR: RK_MPI_SYS_GetMediaBuffer get null buffer!\n");
      break;
    }
      //获取人脸处理结果
    rockx_object_array_t face_array = get_rockx_face_array();
    //创建mat对象，并创建了720x1080的像素块，每个像素每个通道的位数都是8位，一个字节的。上述CV_8UC3中的8表示8位、UC表示uchar类型、1表示1个通道
    Mat tmp_img = Mat(720, 1280, CV_8UC1, RK_MPI_MB_GetPtr(mb));
#if 1
    //对人脸x,y，w,h进行处理
    for (int i = 0; i < face_array.count; i++)
    {
      int x = face_array.object[i].box.left * x_rate;
      int y = face_array.object[i].box.top * y_rate;
      int w = (face_array.object[i].box.right - face_array.object[i].box.left) * x_rate;
      int h = (face_array.object[i].box.bottom - face_array.object[i].box.top) * y_rate;

      if (x < 0)
        x = 0;
      if (y < 0)
        y = 0;

      while ((uint32_t)(x + w) >= 1280)
      {
        w -= 16;
      }
      while ((uint32_t)(y + h) >= 720)
      {
        h -= 16;
      }
      //获取人脸预测名字
      string predict_name = get_rockx_prdict_name();
      printf("predict_name = %s\n", predict_name.c_str());
      nv12_border((char *)RK_MPI_MB_GetPtr(mb), 1280, 720, x, y, w, h, 255, 0, 255);

      int baseline;
      //计算人脸名字文本大小
      Size text_size = getTextSize(predict_name, 2, 2, 2, &baseline);
      Point origin;
      origin.x = tmp_img.cols / 4 - text_size.width / 4;
      origin.y = tmp_img.rows / 4 + text_size.height / 4;
      //把名字字符填充到文本框里面去
      cv::putText(tmp_img, predict_name, origin, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(255, 0, 255), 3);
    }
#endif
    //释放对应的资源
    RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, 0, mb);
    RK_MPI_MB_ReleaseBuffer(mb);
    mb = NULL;
  }
  return NULL;
}

人脸编码rtsp传输线程

void *rockx_venc_rtsp_thread(void *args)
{
  pthread_detach(pthread_self());
  MEDIA_BUFFER mb = NULL;

  while (!quit)
  {
    mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, 0, -1);
    if (!mb)
    {
      printf("ERROR: RK_MPI_SYS_GetMediaBuffer get null buffer!\n");
      break;
    }
    //rtsp来传输码流
    rtsp_tx_video(g_rtsp_session, (unsigned char *)RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), RK_MPI_MB_GetTimestamp(mb));
    RK_MPI_MB_ReleaseBuffer(mb);
    rtsp_do_event(g_rtsplive);
  }

  return NULL;
}