代码收藏家 yolov5和orbslam2结合的实现方法 [使用UNIX域socket实现python和c++通信]
yolov5和orbslam2结合的实现方法
前言
摩西摩西,首先我们先来介绍一下什么是UNIX域的socket吧!
参考网上的文章, UNIX Domain SOCKET 是在Socket架构上发展起来的用于同一台主机的进程间通讯（IPC）。它不需要经过网络协议栈，不需要打包拆包、计算校验和、维护序列号应答等。只是将应用层数据从一个进程拷贝到另一个进程。UNIX Domain SOCKET有SOKCET_DGRAM和SOCKET_STREAM两种模式，类似于UDP和TCP，但是面向消息的UNIX socket也是可靠的，消息既不会丢失也不会顺序错乱。
原文链接：https://blog.csdn.net/zhangkun2609/article/details/84188465
简单来说,它就是unix系统在写成时就有的协议,可以用于进程之间进行通信。他的原理其实和先用python把信息写在一个文件里，再由c++程序去那个文件里读取的道理差不多，只是UNIX域的socket速度更快也更安全 。熟悉ROS的小伙伴应该会觉得这个操作好像和ROS里的消息传输有点像，但是UNIX域socket不会经过网络层等上层的网络，因此速度更快。
（ 主要是看起来更有逼格了，毕业论文里这种名词一写，不明觉厉好吧）
那么我用这个东西干了什么呢？
看过我之前博客的小伙伴应该知道，我毕业设计做的是动态场景下的语义SLAM，我的实现路径简单来说就是先用yolov5检测出动态物体和静态物体，然后把物体框数据传递到orbslam2里面。再在orbslam2中，把动态物体里面的特征点都剔除掉，这可以提高orbslam2系统在高动态环境下的一个表现。
我之前的实现方法是 先运行yolo5，得到物体框的txt格式的数据，再用orbslam2去读取这些数据，然后做后续工作。之前的整个流程是异步的、不实时的，怎么想都感觉不够高级，后来咨询了师兄，采用了UNIX域socket的通信方法来实现。
1、python3中的UNIX域socket实现
先来介绍下python3里面怎么写UNIX域socket吧。
参考博客：
https://blog.csdn.net/zkreats/article/details/50836423
因为我主要是要用python发送消息给c++，因此用python作为server端。
使用的主要就是socket这个模块，import socket即可。
socket模块中有好几种通信协议，可以实现包括TCP/IP在内的多种通信，本文主要讲的是UNIX域协议。
下面我介绍几个常用的函数,基本上会用这几个已经够了.
#这句代码是起始代码，第一个参数代表的是使用UNIX域socket，第二个参数代表的是使用流式的方式来传输数据
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
#这几句代码的作用是绑定通信的地址(在c++里也同样绑定这个地址就可以传数据了)
serverAddr = '/home/jy/server_socket'
if os.path.exists(serverAddr):
os.unlink(serverAddr)# 如果套接字存在，则删除
server.bind(serverAddr) #绑定要监听的端口
#意为最多可以同时监听五个端口
server.listen(5)
#就是建立一个连接啦
conn,addr = server.accept()
#发送数据
conn.send('Hi! I am python. I am glad to hear you!'.encode())
#接受数据并取前30位
data = conn.recv(1024)[0:30]
data = data.decode("ascii")
#关闭传输
conn.close()
传一个我测试时候用的代码吧,可以跑。
在基本的接发数据的基础上加了一个判断是否发送的东西
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import socket
# 建立一个服务端
import sys
import os
serverAddr = '/home/jy/server_socket' # 套接字存放路径及名称
# serverAddr = '/home/jy/tmp/hhh_socket' # 套接字存放路径及名称
server = socket.socket(socket.AF_UNIX,socket.SOCK_STREAM)
print(server)
if os.path.exists(serverAddr):
os.unlink(serverAddr)# 如果套接字存在，则删除
server.bind(serverAddr) #绑定要监听的端口
if server.listen(5): #最多监听5个客户端
print >> sys.stderr, 'socket.listen error'
# server.listen(5) #开始监听 表示可以使用五个链接排队
# print(server.listen(5))
while True:# conn就是客户端链接过来而在服务端为期生成的一个链接实例
print ('waiting for connecting')
conn,addr = server.accept() #等待链接,多个链接的时候就会出现问题,其实返回了两个值
print(conn,addr)
flag = 1
while True:
if flag==1:
conn.send('Hi! I am python. I am glad to hear you!'.encode()) #然后再发送数据
flag = 2
try:
# data = conn.recv(1024) #接收数据
data = conn.recv(1024)[0:30] #!
# print(data)
data = data.decode("ascii")
# print(type(data))
#因为data = conn.recv(1024)[0:30]收了30个bytes，为了可以能够在后面的判断里可以相等起来，这里把多余的空位手动补全了
a = b'1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
a = a.decode("ascii")
# print(type(a))
if a==data :#如果收到的信息是“1”就做出点不一样的花样
print("wow")
conn.send('Hi! I am python. I am sad to hear you!'.encode()) #然后再发送数据
else:
print('In python:',data) #打印接收到的数据
#如果server不给client回应,则client不会继续发消息,!!:要看read和write的顺序
conn.send('Hi! I am python. I am happy to hear you!'.encode()) #然后再发送数据
# conn.send(data.upper()) #然后再发送数据
except ConnectionResetError as e:
print('关闭了正在占线的链接！')
break
conn.close()
2、C++中的UNIX域socket实现
首先还是介绍一些常用的句子(我也只会这些了):
sockfd = socket(AF_UNIX, SOCK_STREAM, 0)//这句话就是新建一个socket
//这三句话就是定义使用的socket传输的协议是UNIX域的,并且定义了传输地址(和python对应)
address.sun_family = AF_UNIX;
strcpy(address.sun_path, "/home/jy/server_socket");
len = sizeof(address);
result = connect(sockfd, (struct sockaddr *)&address, len);
//往send_buf里写入"ok",再把send_buf传输出去
sprintf(send_buf,"ok");
byte=write(sockfd, send_buf, sizeof(send_buf);//如果有错则byte会赋值为-1
//接收数据,存在ch_recv里面
byte=read(sockfd,&ch_recv,1000)
//清除send_buf和ch_recv
memset(send_buf,0,sizeof(send_buf));//clear char[]
memset(ch_recv,0,sizeof(ch_recv));//clear char[]
//关闭socket
close(sockfd);
再放一个c++里的client的例子
#include <sys/types.h>
#include <sys/socket.h>
#include <stdio.h>
#include <sys/un.h>
#include <unistd.h>
#include <stdlib.h>
// #include<ctime>
#include <iostream>
#include <chrono>
using namespace std;
int main(int argc,char *argv[])
{
int sockfd;
int len;
struct sockaddr_un address;
int result;
int i,byte;
char send_buf[50],ch_recv[1024];
if((sockfd = socket(AF_UNIX, SOCK_STREAM, 0))==-1)//创建socket，指定通信协议为AF_UNIX,数据方式SOCK_STREAM
{
perror("socket");
exit(EXIT_FAILURE);
}
//配置server_address
address.sun_family = AF_UNIX;
strcpy(address.sun_path, "/home/jy/server_socket");
len = sizeof(address);
result = connect(sockfd, (struct sockaddr *)&address, len);
if(result == -1)
{
printf("ensure the server is up\n");
perror("connect");
exit(EXIT_FAILURE);
}
int k = 1 ;
while(1)
{
std::chrono::steady_clock::time_point t1 = std::chrono::steady_clock::now();
sleep(1);
sprintf(send_buf,"ok");//用sprintf事先把消息写到send_buf
if((byte=write(sockfd, send_buf, sizeof(send_buf)))==-1)
{
perror("write");
exit(EXIT_FAILURE);
}
if((byte=read(sockfd,&ch_recv,1000))==-1)
{
perror("read");
exit(EXIT_FAILURE);
}
// printf("In C++: %s",ch_recv);
// cout << ch_recv
cout << "********new*********" << endl;
string ch_recv_string = ch_recv;
cout << ch_recv_string << endl;
memset(send_buf,0,sizeof(send_buf));//clear char[]
memset(ch_recv,0,sizeof(ch_recv));//clear char[]
k = k+1;
std::chrono::steady_clock::time_point t2 = std::chrono::steady_clock::now();
double ttrack= std::chrono::duration_cast<std::chrono::duration<double> >(t2 - t1).count();
// cout << "per img time is " << ttrack << endl;
}
close(sockfd);
return 0;
}
3、附件:C++的server和python的client
为了方便其他和我做的东西不一样的网友,我把我写的C++的server和python的client也放在这里,需要自取。
#c++ server
#include <sys/types.h>
#include <sys/socket.h>
#include <stdio.h>
#include <sys/un.h>
#include <unistd.h>
#include <stdlib.h>
int main(int argc,char *argv[])
{
int server_sockfd, client_sockfd;
int server_len, client_len;
struct sockaddr_un server_address;
struct sockaddr_un client_address;
int i,byte;
char ch_send,recv_buf[128];
unlink("/home/jy/server_socket"); //解除原有server_socket对象链接
server_sockfd = socket(AF_UNIX, SOCK_STREAM, 0);//创建socket，指定通信协议为AF_UNIX,数据方式SOCK_STREAM
//配置server_address
server_address.sun_family = AF_UNIX;
strcpy(server_address.sun_path, "/home/jy/server_socket");
server_len = sizeof(server_address);
bind(server_sockfd, (struct sockaddr *)&server_address, server_len);
listen(server_sockfd, 5);
printf("server waiting for client connect\n");
client_len = sizeof(client_address);
//accept函数接收客户端求情，存储客户端地址信息、客户端地址大小
client_sockfd = accept(server_sockfd,(struct sockaddr *)&client_address, (socklen_t *)&client_len);
printf("the server wait form client data\n");
for(i=0,ch_send='1';i<5;i++,ch_send++)
{
//从client_sockfd读取客户端发来的消息
if((byte=read(client_sockfd, recv_buf, sizeof(recv_buf)))==-1)
{
perror("read");
exit(EXIT_FAILURE);
}
printf("the massage receiver from client is: %s\n",recv_buf);
sleep(1);
//向客户端发送消息
if((byte=write(client_sockfd,&ch_send,1))==-1)
{
perror("write");
exit(EXIT_FAILURE);
}
}
close(client_sockfd);
unlink("server socket");
}
#python-client
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import socket# 客户端 发送一个数据，再接收一个数据
import sys
import os
serverAddr = './uds_socket' # 套接字存放路径及名称
client = socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) #声明socket类型，同时生成链接对象
client.connect(serverAddr)
while True:
msg = '欢迎访问菜鸟教程！' #strip默认取出字符串的头尾空格
client.send(msg.encode('utf-8')) #发送一条信息 python3 只接收btye流
data = client.recv(1024) #接收一个信息，并指定接收的大小 为1024字节
print('recv:',data.decode()) #输出我接收的信息
client.close() #关闭这个链接
4、yolov5和orbslam2结合的实现方法(这个得结合之前的博客来看)
碍于我毕设还没写论文和答辩呢,所以也不好说太多,大致说一下思路吧.
1.1、在orbslam2代码的rgb_tum.cc文件开头就添加上socket的初始化
int sockfd;
int len;
struct sockaddr_un address;
int result;
int i,byte;
char send_buf[128],ch_recv[1024];
if((sockfd = socket(AF_UNIX, SOCK_STREAM, 0))==-1)//创建socket，指定通信协议为AF_UNIX,数据方式SOCK_STREAM
{
perror("socket");
exit(EXIT_FAILURE);
}
//配置server_address
address.sun_family = AF_UNIX;
strcpy(address.sun_path, "/home/jy/server_socket");
len = sizeof(address);
result = connect(sockfd, (struct sockaddr *)&address, len);
if(result == -1)
{
printf("ensure the server is up\n");
perror("connect");
exit(EXIT_FAILURE);
}
1.2、在orbslam2代码的rgb_tum.cc文件中添加两个函数
LoadBoundingBoxFromPython 函数的作用是从一句物体框数据中读取出我们需要的数据并储存起来.
MakeDetect_result 函数的作用是从一片物体框数据中分割出一句一句的物体框数据,然后调用LoadBoundingBoxFromPython函数来分别读取.
//在一句话中提取出四个边框值和物体类别,such as: left:1 top:134 right:269 bottom:478 class:person 0.79
void LoadBoundingBoxFromPython(const string& resultFromPython, std::pair<vector<double>, int>& detect_result){
if(resultFromPython.empty())
{
cerr << "no string from python! " << endl;
}
// cout << "here is LoadBoundingBoxFromPython " << endl;
vector<double> result_parameter;
int sum = 0, num_bit = 0;
for (char c : resultFromPython) {//读取数字. 例如读取"748",先读7,再7*10+8=78,再78*10+4,最后读到空格结束
if (c >= '0' && c <= '9') {
num_bit = c - '0';
sum = sum * 10 + num_bit;
} else if (c == ' ') {
result_parameter.push_back(sum);
sum = 0;
num_bit = 0;
}
}
detect_result.first = result_parameter;
// cout << "detect_result.first size is : " << detect_result.first.size() << endl;
string idx_begin = "class:";//读取物体类别
int idx = resultFromPython.find(idx_begin);
string idx_end = "0.";
int idx2 = resultFromPython.find(idx_end);
string class_label;
for (int j = idx + 6; j < idx2-1; ++j){
class_label += resultFromPython[j];
}
int class_id = -1;//存入识别物体的种类
if (class_label == "tv" || //低动态物体(在程序中可以假设为一直静态的物体):tv,refrigerator
class_label == "refrigerator" ||
class_label == "teddy bear"||
class_label == "laptop") {
class_id = 1;
}
if (class_label == "chair" || //中动态物体,在程序中不做先验动态静态判断
class_label == "car"){
class_id =2;
}
if (class_label == "person") { //高动态物体:人,动物等
class_id = 3;
}
detect_result.second = class_id;
// cout << "LoadBoundingBoxFromPython class id is: " << class_id << endl;
}
void MakeDetect_result(vector<std::pair<vector<double>, int>>& detect_result , int sockfd){
detect_result.clear();
std::pair<vector<double>, int> detect_result_str;
int byte;
char send_buf[128],ch_recv[1024];
sprintf(send_buf,"ok");//用sprintf事先把消息写到send_buf
if((byte=write(sockfd, send_buf, sizeof(send_buf)))==-1)
{
perror("write");
exit(EXIT_FAILURE);
}
if((byte=read(sockfd,&ch_recv,1000))==-1)
{
perror("read");
exit(EXIT_FAILURE);
}
// cout << "**ch_recv is : \n" << ch_recv << endl;
char *ptr;//char[]可读可写,可以修改字符串的内容。char*可读不可写，写入就会导致段错误
ptr = strtok(ch_recv, "*");//字符串分割函数
while(ptr != NULL){
printf("ptr=%s\n",ptr);
if ( strlen(ptr)>20 ){//试图去除乱码,乱码原因未知...好像并不能去除,留着吧,心理安慰下
// cout << strlen(ptr) << endl;
string ptr_str = ptr;
LoadBoundingBoxFromPython(ptr_str,detect_result_str);
}
detect_result.emplace_back(detect_result_str);
// cout << "hh: " << ptr_str << endl;
ptr = strtok(NULL, "*");
}
// cout << "detect_result size is : " << detect_result.size() << endl;
// for (int k=0; k<detect_result.size(); ++k)
// cout << "detect_result is : \n " << detect_result[k].second << endl;
}
2.1、修改yolov5的detect.py的代码
干了两件事情,一是为了加速程序运行速度,删掉了储存图片到文件的代码;
第二件事情是加上了socket,一次把一帧的物体框数据都传输到c++里面.
# -*- coding: UTF-8 -*-
import socket
# 建立一个服务端
import sys
import os
import argparse
import time
from pathlib import Path
import cv2
import torch
import torch.backends.cudnn as cudnn
from numpy import random
from models.experimental import attempt_load
from utils.datasets import LoadStreams, LoadImages
from utils.general import check_img_size, check_requirements, check_imshow, non_max_suppression, apply_classifier, \
scale_coords, xyxy2xywh, strip_optimizer, set_logging, increment_path
from utils.plots import plot_one_box
from utils.torch_utils import select_device, load_classifier, time_synchronized
serverAddr = '/home/jy/server_socket' # 套接字存放路径及名称
# serverAddr = '/home/jy/tmp/hhh_socket' # 套接字存放路径及名称
server = socket.socket(socket.AF_UNIX,socket.SOCK_STREAM)
print(server)
if os.path.exists(serverAddr):
os.unlink(serverAddr)# 如果套接字存在，则删除
server.bind(serverAddr) #绑定要监听的端口
if server.listen(5): #最多监听5个客户端
print >> sys.stderr, 'socket.listen error'
print ('waiting for connecting')
conn,addr = server.accept() #等待链接,多个链接的时候就会出现问题,其实返回了两个值
print(conn,addr)
def detect(save_img=False):
source, weights, view_img, save_txt, imgsz = opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size
webcam = source.isnumeric() or source.endswith('.txt') or source.lower().startswith(
('rtsp://', 'rtmp://', 'http://'))
# Directories
save_dir = Path(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir
(save_dir / 'detect_result' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # ***3
# Initialize
set_logging()
device = select_device(opt.device)
half = device.type != 'cpu' # half precision only supported on CUDA
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
stride = int(model.stride.max()) # model stride
imgsz = check_img_size(imgsz, s=stride) # check img_size
if half:
model.half() # to FP16
# Second-stage classifier
classify = False
if classify:
modelc = load_classifier(name='resnet101', n=2) # initialize
modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']).to(device).eval()
# Set Dataloader
vid_path, vid_writer = None, None
if webcam:
view_img = check_imshow()
cudnn.benchmark = True # set True to speed up constant image size inference
dataset = LoadStreams(source, img_size=imgsz, stride=stride)
else:
save_img = True
dataset = LoadImages(source, img_size=imgsz, stride=stride)
# Get names and colors
names = model.module.names if hasattr(model, 'module') else model.names
colors = [[random.randint(0, 255) for _ in range(3)] for _ in names]
# Run inference
if device.type != 'cpu':
model(torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(next(model.parameters()))) # run once
t0 = time.time()
for path, img, im0s, vid_cap in dataset:
img = torch.from_numpy(img).to(device)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
# Inference
t1 = time_synchronized()
pred = model(img, augment=opt.augment)[0]
# Apply NMS
pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms)
t2 = time_synchronized()
# Apply Classifier
if classify:
pred = apply_classifier(pred, modelc, img, im0s)
# Process detections
for i, det in enumerate(pred): # detections per image
if webcam: # batch_size >= 1
p, s, im0, frame = path[i], '%g: ' % i, im0s[i].copy(), dataset.count
else:
p, s, im0, frame = path, '', im0s, getattr(dataset, 'frame', 0)
p = Path(p) # to Path
save_path = str(save_dir / p.name) # img.jpg
txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}')
result_path = str(save_dir / 'detect_result' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # **1
s += '%gx%g ' % img.shape[2:] # print string
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh
# print ("*************:\n",len(det)) #what is this?
if len(det):
# Rescale boxes from img_size to im0 size
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round()
# Print results
for c in det[:, -1].unique():
n = (det[:, -1] == c).sum() # detections per class
s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string
# Write results
result_send = ""
for *xyxy, conf, cls in reversed(det):
if save_txt: # Write to file
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh
# print(" ")
# print("left:",int(xyxy[0])," top:",int(xyxy[1])," right:",int(xyxy[2])," bottom:",int(xyxy[3]))
str_result ="left:" + str(int(xyxy[0])) + " top:" + str(int(xyxy[1])) + " right:" + str(int(xyxy[2])) + " bottom:"+ str(int(xyxy[3])) + " class:"
line = (cls, *xywh, conf) if opt.save_conf else (cls, *xywh) # label format
with open(result_path + '.txt', 'a') as f2: #***2
f2.write(str_result)
if save_img or view_img: # Add bbox to image
label = f'{names[int(cls)]} {conf:.2f}'
# print(type(label)) #label is str
with open(result_path + '.txt', 'a') as f2: #***2
f2.write(label)
f2.write("\n")
# plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=3)
str_result = str_result + label + "*"
result_send = result_send + str_result
data = conn.recv(1024)[0:3].decode("ascii") #
print('**In python :',data)
print("*******result is : \n",result_send)
conn.send(result_send.encode())
# Print time (inference + NMS)
print(f'{s}Done. ({t2 - t1:.3f}s)')
if save_txt or save_img:
s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
#jy: 4 labels saved to runs/detect/exp6/labels
print(f"Results saved to {save_dir}{s}")
print(f'Done. ({time.time() - t0:.3f}s)')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)')#jy:输入可改source和weights
parser.add_argument('--source', type=str, default='data/images', help='source') # file/folder, 0 for webcam
parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)')#default is 640,change to 32 speed up up!
parser.add_argument('--conf-thres', type=float, default=0.5, help='object confidence threshold')#default is 0.25
parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS')
parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
parser.add_argument('--view-img', action='store_true', help='display results')
parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels')
parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3')#--class 0 :is class person
parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
parser.add_argument('--augment', action='store_true', help='augmented inference')
parser.add_argument('--update', action='store_true', help='update all models')
parser.add_argument('--project', default='runs/detect', help='save results to project/name')
parser.add_argument('--name', default='exp', help='save results to project/name')
parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
opt = parser.parse_args()
print(opt)
check_requirements()
with torch.no_grad():
if opt.update: # update all models (to fix SourceChangeWarning)
for opt.weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']:
detect()
strip_optimizer(opt.weights)
else:
detect()
# print("nonono***********************is here")
5、总结
总之，本次内容就到这里了。有需要、有问题再留言吧。
溜～～
来源：山楂没我渣
