VideoPipe可视化视频结构化框架更新总结（2023-3-30）
项目地址：https://github.com/sherlockchou86/video_pipe_c
往期文章：https://www.cnblogs.com/xiaozhi_5638/p/16969546.html
最近有多个更新，有兴趣的扫码加群交流。
新增实例分割相关支持
增加了基于mask-rcnn的实例分割插件和相关sample。
1 #include "VP.h" 2 3 #include "../nodes/vp_file_src_node.h" 4 #include "../nodes/infers/vp_mask_rcnn_detector_node.h" 5 #include "../nodes/track/vp_sort_track_node.h" 6 #include "../nodes/osd/vp_osd_node_v3.h" 7 #include "../nodes/vp_screen_des_node.h" 8 9 #include "../utils/analysis_board/vp_analysis_board.h" 10 11 /* 12 * ## mask rcnn sample ## 13 * image segmentation using mask rcnn. 14 */ 15 16 #if mask_rcnn_sample 17 18 int main() { 19 VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); 20 VP_LOGGER_INIT(); 21 22 // create nodes 23 auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/19.mp4", 0.6); 24 auto mask_rcnn_detector = std::make_shared<vp_nodes::vp_mask_rcnn_detector_node>("mask_rcnn_detector", "./models/mask_rcnn/frozen_inference_graph.pb", "./models/mask_rcnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt", "./models/coco_80classes.txt"); 25 auto track_0 = std::make_shared<vp_nodes::vp_sort_track_node>("sort_track_0"); 26 auto osd_v3_0 = std::make_shared<vp_nodes::vp_osd_node_v3>("osd_v3_0", "../third_party/paddle_ocr/font/NotoSansCJKsc-Medium.otf"); 27 auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0); 28 29 // construct pipeline 30 mask_rcnn_detector->attach_to({file_src_0}); 31 track_0->attach_to({mask_rcnn_detector}); 32 osd_v3_0->attach_to({track_0}); 33 screen_des_0->attach_to({osd_v3_0}); 34 35 file_src_0->start(); 36 37 // for debug purpose 38 vp_utils::vp_analysis_board board({file_src_0}); 39 board.display(); 40 } 41 42 43 #endif
上面代码效果图如下：
新增语义分割相关支持
新增了基于ENet网络的语义分割插件和sample。
1 #include "VP.h" 2 3 #include "../nodes/vp_file_src_node.h" 4 #include "../nodes/infers/vp_enet_seg_node.h" 5 #include "../nodes/osd/vp_seg_osd_node.h" 6 #include "../nodes/vp_screen_des_node.h" 7 8 #include "../utils/analysis_board/vp_analysis_board.h" 9 10 /* 11 * ## enet seg sample ## 12 * semantic segmentation based on ENet. 13 * 1 input, 2 outputs including orignal frame and mask frame. 14 */ 15 16 #if enet_seg_sample 17 18 int main() { 19 VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); 20 VP_LOGGER_INIT(); 21 22 // create nodes 23 auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/21.mp4"); 24 auto enet_seg = std::make_shared<vp_nodes::vp_enet_seg_node>("enet_seg", "models/enet-cityscapes/enet-model.net"); 25 auto seg_osd_0 = std::make_shared<vp_nodes::vp_seg_osd_node>("seg_osd_0", "models/enet-cityscapes/enet-classes.txt", "models/enet-cityscapes/enet-colors.txt"); 26 auto screen_des_mask = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_mask", 0, true, vp_objects::vp_size(400, 225)); 27 auto screen_des_original = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_original", 0, false, vp_objects::vp_size(400, 225)); 28 29 // construct pipeline 30 enet_seg->attach_to({file_src_0}); 31 seg_osd_0->attach_to({enet_seg}); 32 screen_des_mask->attach_to({seg_osd_0}); 33 screen_des_original->attach_to({seg_osd_0}); 34 35 file_src_0->start(); 36 37 // for debug purpose 38 vp_utils::vp_analysis_board board({file_src_0}); 39 board.display(); 40 } 41 42 #endif
上面代码效果图如下：
新增多级推理插件sample
多个检测、分类插件串联，不同分类器作用于不同的主目标：
1 #include "VP.h" 2 3 #include "../nodes/vp_file_src_node.h" 4 #include "../nodes/infers/vp_yolo_detector_node.h" 5 #include "../nodes/infers/vp_classifier_node.h" 6 #include "../nodes/osd/vp_osd_node.h" 7 #include "../nodes/vp_screen_des_node.h" 8 #include "../utils/analysis_board/vp_analysis_board.h" 9 10 /* 11 * ## multi detectors and classifiers sample ## 12 * show multi infer nodes work together. 13 * 1 detector and 2 classifiers applied on primary class ids(1/2/3). 14 */ 15 16 #if multi_detectors_and_classifiers_sample 17 18 int main() { 19 VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); 20 VP_LOGGER_INIT(); 21 22 // create nodes 23 auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "test_video/20.mp4", 0.6); 24 /* primary detector */ 25 // labels for detector model 26 // 0 - person 27 // 1 - car 28 // 2 - bus 29 // 3 - truck 30 // 4 - 2wheel 31 auto primary_detector = std::make_shared<vp_nodes::vp_yolo_detector_node>("primary_detector", "models/det_cls/yolov3-tiny-2022-0721_best.weights", "models/det_cls/yolov3-tiny-2022-0721.cfg", "models/det_cls/yolov3_tiny_5classes.txt", 416, 416, 1); 32 /* secondary classifier 1, applied to car(1)/bus(2)/truck(3) only */ 33 auto _1st_classifier = std::make_shared<vp_nodes::vp_classifier_node>("1st_classifier", "models/det_cls/vehicle/resnet18-batch=N-type_view_0322_nhwc.onnx", "", "models/det_cls/vehicle/vehicle_types.txt", 224, 224, 1, std::vector<int>{1, 2, 3}, 10, false, 1, cv::Scalar(), cv::Scalar(), true, true); 34 /* secondary classifier 2, applied to car(1)/bus(2)/truck(3) only */ 35 auto _2nd_classifier = std::make_shared<vp_nodes::vp_classifier_node>("2nd_classifier", "models/det_cls/vehicle/resnet18-batch=N-color_view_0322_nhwc.onnx", "", "models/det_cls/vehicle/vehicle_colors.txt", 224, 224, 1, std::vector<int>{1, 2, 3}, 10, false, 1, cv::Scalar(), cv::Scalar(), true, true); 36 auto osd_0 = std::make_shared<vp_nodes::vp_osd_node>("osd_0"); 37 auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_o", 0); 38 39 // construct pipeline 40 primary_detector->attach_to({file_src_0}); 41 _1st_classifier->attach_to({primary_detector}); 42 _2nd_classifier->attach_to({_1st_classifier}); 43 osd_0->attach_to({_2nd_classifier}); 44 screen_des_0->attach_to({osd_0}); 45 46 // start 47 file_src_0->start(); 48 49 // for debug purpose 50 vp_utils::vp_analysis_board board({file_src_0}); 51 board.display(); 52 } 53 54 #endif
上面代码运行效果如下：
新增图片源输入插件
支持以图片方式输入（文件或UDP），频率可调、各个通道互相独立。
1 #include "VP.h" 2 3 #include "../nodes/vp_image_src_node.h" 4 #include "../nodes/infers/vp_yolo_detector_node.h" 5 #include "../nodes/osd/vp_osd_node.h" 6 #include "../nodes/vp_split_node.h" 7 #include "../nodes/vp_screen_des_node.h" 8 9 #include "../utils/analysis_board/vp_analysis_board.h" 10 11 /* 12 * ## image_src_sample ## 13 * show how vp_image_src_node works, read image from local file or receive image from remote via udp. 14 */ 15 16 #if image_src_sample 17 18 int main() { 19 VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); 20 VP_LOGGER_INIT(); 21 22 // create nodes 23 auto image_src_0 = std::make_shared<vp_nodes::vp_image_src_node>("image_file_src_0", 0, "./images/test_%d.jpg", 1, 0.4); // read 1 image EVERY 1 second from local files, such as test_0.jpg,test_1.jpg 24 /* sending command for test: `gst-launch-1.0 filesrc location=16.mp4 ! qtdemux ! avdec_h264 ! videoconvert ! videoscale ! video/x-raw,width=416,height=416 ! videorate ! video/x-raw,framerate=1/1 ! jpegenc ! rtpjpegpay ! udpsink host=ip port=6000` */ 25 auto image_src_1 = std::make_shared<vp_nodes::vp_image_src_node>("image_udp_src_1", 1, "6000", 3); // receive 1 image EVERY 3 seconds from remote via udp , such as 127.0.0.1:6000 26 auto yolo_detector = std::make_shared<vp_nodes::vp_yolo_detector_node>("yolo_detector", "models/det_cls/yolov3-tiny-2022-0721_best.weights", "models/det_cls/yolov3-tiny-2022-0721.cfg", "models/det_cls/yolov3_tiny_5classes.txt"); 27 auto osd = std::make_shared<vp_nodes::vp_osd_node>("osd"); 28 auto split = std::make_shared<vp_nodes::vp_split_node>("split_by_channel", true); // split by channel index (important!) 29 auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0); 30 auto screen_des_1 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_1", 1); 31 32 // construct pipeline 33 yolo_detector->attach_to({image_src_0, image_src_1}); 34 osd->attach_to({yolo_detector}); 35 split->attach_to({osd}); 36 screen_des_0->attach_to({split}); 37 screen_des_1->attach_to({split}); 38 39 image_src_0->start(); // start read from local file 40 image_src_1->start(); // start receive from remote via udp 41 42 // for debug purpose 43 vp_utils::vp_analysis_board board({image_src_0, image_src_1}); 44 board.display(); 45 } 46 47 #endif
上面代码运行效果如下：
新增图片结果输出插件
支持以图片格式输出结果（文件或UDP），频率可调、各通道互相独立。
1 #include "VP.h" 2 3 #include "../nodes/vp_file_src_node.h" 4 #include "../nodes/infers/vp_yunet_face_detector_node.h" 5 #include "../nodes/infers/vp_sface_feature_encoder_node.h" 6 #include "../nodes/osd/vp_face_osd_node_v2.h" 7 #include "../nodes/vp_screen_des_node.h" 8 #include "../nodes/vp_image_des_node.h" 9 10 #include "../utils/analysis_board/vp_analysis_board.h" 11 12 /* 13 * ## image_des_sample ## 14 * show how vp_image_des_node works, save image to local file or push image to remote via udp. 15 */ 16 17 #if image_des_sample 18 19 int main() { 20 VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); 21 VP_LOGGER_INIT(); 22 23 // create nodes 24 auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/10.mp4", 0.6); 25 auto yunet_face_detector_0 = std::make_shared<vp_nodes::vp_yunet_face_detector_node>("yunet_face_detector_0", "./models/face/face_detection_yunet_2022mar.onnx"); 26 auto sface_face_encoder_0 = std::make_shared<vp_nodes::vp_sface_feature_encoder_node>("sface_face_encoder_0", "./models/face/face_recognition_sface_2021dec.onnx"); 27 auto osd_0 = std::make_shared<vp_nodes::vp_face_osd_node_v2>("osd_0"); 28 auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0); 29 30 /* save to file, `%d` is placeholder for filename index */ 31 //auto image_des_0 = std::make_shared<vp_nodes::vp_image_des_node>("image_file_des_0", 0, "./images/%d.jpg", 3); 32 33 /* push via udp, receiving command for test: `gst-launch-1.0 udpsrc port=6000 ! application/x-rtp,encoding-name=jpeg ! rtpjpegdepay ! jpegparse ! jpegdec ! queue ! videoconvert ! autovideosink` */ 34 auto image_des_0 = std::make_shared<vp_nodes::vp_image_des_node>("image_udp_des_0", 0, "192.168.77.248:6000", 3, vp_objects::vp_size(400, 200)); 35 36 // construct pipeline 37 yunet_face_detector_0->attach_to({file_src_0}); 38 sface_face_encoder_0->attach_to({yunet_face_detector_0}); 39 osd_0->attach_to({sface_face_encoder_0}); 40 screen_des_0->attach_to({osd_0}); 41 image_des_0->attach_to({osd_0}); 42 43 file_src_0->start(); 44 45 // for debug purpose 46 vp_utils::vp_analysis_board board({file_src_0}); 47 board.display(); 48 } 49 50 #endif
上面代码运行效果如下：
更多更新扫码加入微信群，及时获取通知。