使用 Intel Realsense D415 进行物体识别

Question

您好，我正在开展一个使用 python 和 tensorflow 模型训练的项目，我希望摄像头能够检测摄像头中经过训练的图像并显示摄像头到该物体的距离，例如书笔。 这是我正在使用的代码：

我的问题是如何在此代码中添加距离测量并将其显示在识别的图像上

import pyrealsense2 as rs import numpy as np import cv2 import tensorflow as tf

Configure depth and color streams
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.color, 1280, 720, rs.format.bgr8, 30)

print("[INFO] Starting streaming...")
pipeline.start(config)
print("[INFO] Camera ready.")

print("[INFO] Loading model...")
PATH_TO_CKPT = "frozen_inference_graph_coco.pb"


detection_graph = tf.Graph()
with detection_graph.as_default():
    od_graph_def = tf.compat.v1.GraphDef()
    with tf.compat.v1.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.compat.v1.import_graph_def(od_graph_def, name='')
    sess = tf.compat.v1.Session(graph=detection_graph)

image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')

detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')

detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')

num_detections = detection_graph.get_tensor_by_name('num_detections:0')

print("[INFO] Model loaded.")
colors_hash = {}
while True:
    frames = pipeline.wait_for_frames()
    color_frame = frames.get_color_frame()

    color_image = np.asanyarray(color_frame.get_data())
    scaled_size = (color_frame.width, color_frame.height)
    image_expanded = np.expand_dims(color_image, axis=0)
    (boxes, scores, classes, num) = sess.run([detection_boxes, detection_scores, detection_classes, num_detections],
                                             feed_dict={image_tensor: image_expanded})

    boxes = np.squeeze(boxes)
    classes = np.squeeze(classes).astype(np.int32)
    scores = np.squeeze(scores)

    for idx in range(int(num)):
        class_ = classes[idx]
        score = scores[idx]
        box = boxes[idx]

        if class_ not in colors_hash:
            colors_hash[class_] = tuple(np.random.choice(range(256), size=3))

        if score > 0.6:
            left = int(box[1] * color_frame.width)
            top = int(box[0] * color_frame.height)
            right = int(box[3] * color_frame.width)
            bottom = int(box[2] * color_frame.height)

            p1 = (left, top)
            p2 = (right, bottom)
            r, g, b = colors_hash[class_]
            cv2.rectangle(color_image, p1, p2, (int(r), int(g), int(b)), 2, 1)

    cv2.namedWindow('RealSense', cv2.WINDOW_AUTOSIZE)
    cv2.imshow('RealSense', color_image)
    cv2.waitKey(1)

print("[INFO] stop streaming ...")
pipeline.stop()

PS：我是初学者，我在pycharm工作

Answer 1

因此您首先需要启用深度相机流，然后将深度流与颜色流对齐。这将使您能够获得(1280, 720, 3) 图像和深度矩阵（仅当您使用cv2.applyColourMap 或rs.colorizer() 对图像进行着色时，深度矩阵才为3 通道）。

现在，让我们假设您使用示例代码中提到的 TensorFlow 在图像中检测到 (x1, y1, x2, y2) 处的对象。您现在可以查询深度矩阵 (original, not colorised) 关于此框坐标内的深度值。这将为您提供盒子位置的原始深度值。但是，您还需要另一条信息，即深度标尺，您可以使用profile.get_device().first_depth_sensor().get_depth_scale() 从深度传感器获取它。当您将深度矩阵中区域x1, y1, x2, y2 内的深度值乘以深度比例时，您将得到对象的深度。

您可以参考tutorial notebook。

您可以查看此 Python 脚本的样板代码。我省略了关于对象检测的部分，因为这是你已经做过的事情。

import pyrealsense2 as rs
import cv2
import numpy as np

cv2.namedWindow("Colour Image", cv2.WINDOW_AUTOSIZE)
cv2.namedWindow("Depth Image", cv2.WINDOW_AUTOSIZE)

pipeline = rs.pipeline()
config = rs.config()

wrapper = rs.pipeline_wrapper(pipeline)
profile = config.resolve(wrapper)

colorizer = rs.colorizer()
depth_scale = profile.get_device().first_depth_sensor().get_depth_scale()

print(f"Depth scale:{depth_scale}")

config.enable_stream(rs.stream.depth, 1280, 720, rs.format.z16, 30)
config.enable_stream(rs.stream.color, 1280, 720, rs.format.bgr8, 30)
pipeline.start(config)

align = rs.align(rs.stream.color)

while True:
    frames = pipeline.wait_for_frames()
    frames = align.process(frames)
    color_frame = frames.get_color_frame()
    depth_frame = frames.get_depth_frame()

    color_image = np.asanyarray(color_frame.get_data())
    depth_image = np.asanyarray(depth_frame.get_data())

    # Insert your object detection code here

    # I assume that I detected an object at [200, 200, 400, 400]

    # Draw the bounding box
    cv2.rectangle(color_image, (200, 200), (400, 400), (0, 255, 255), 1)
    cv2.rectangle(depth_image, (200, 200), (400, 400), (0, 255, 255), 1)

    depth = depth_image[200:400, 200:400].astype(float)

    depth = depth * depth_scale

    dist = cv2.mean(depth)

    cv2.putText(color_image, f"Depth: {dist} m", (200, 200), cv2.FONT_HERSHEY_PLAIN, 1.0, (0,0,0), 1)
    cv2.imshow("Colour Image", color_image)
    cv2.imshow("Depth Image", depth_image)



    if cv2.waitKey(1) & 0xFF == 27: # Escape key closes the window(s)
        break

pipeline.stop()
cv2.destroyAllWindows()