OpenVINO MULTI:MYRIAD with sequential inference is inefficiency and usually shows "XLink_sem_wait:94" and "XLinkResetRemote:257" logs
I have integrated OpenVINO and PyQt5 to do the inference job as shown in the image on Windows 11 with openvino_2021.4.689 version.
I reference this GitHub to finish YOLOv4 inference with NCS2.
The following is my inference engine code.
from openvino.inference_engine import IECore
import cv2
import numpy as np
import global_variable as gv
import ngraph as ng
import logging
import threading
import os
import sys
from collections import deque
from argparse import ArgumentParser, SUPPRESS
from math import exp as exp
from time import perf_counter
from enum import Enum
def deside_cut_size(height, width):
global x, y, w, h
if((height, width) == (1536, 868)):
x, y = 767, 0 # Where the crop point starts.
w, h = 769, 637 # The size of crop image.
elif((height, width) == (1264, 596)):
x, y = 495, 0 # Where the crop point starts.
w, h = 769, 503 # The size of crop image.
elif((height, width) == (1280, 868)):
x, y = 766, 0 # Where the crop point starts.
w, h = 514, 638 # The size of crop image.
elif((height, width) == (1152, 868)):
x, y = 766, 0 # Where the crop point starts.
w, h = 386, 636 # The size of crop image.
else:
x, y = int(height / 2), 0 # Where the crop point starts.
w, h = int(height / 2), int(width - 100) # The size of crop image.
def build_argparser():
parser = ArgumentParser(add_help=False)
args = parser.add_argument_group('Options')
args.add_argument('-h', '--help', action='help', default=SUPPRESS, help='Show this help message and exit.')
args.add_argument("-m", "--model", help="Required. Path to an .xml file with a trained model.",
default="model/frozen_darknet_yolov4_model.xml", type=str)
args.add_argument("-i", "--input", help="Required. Path to an image/video file. (Specify 'cam' to work with "
"camera)", default=gv.gInImgPath, type=str)
args.add_argument("-l", "--cpu_extension",
help="Optional. Required for CPU custom layers. Absolute path to a shared library with "
"the kernels implementations.", type=str, default=None)
args.add_argument("-d", "--device",
help="Optional. Specify the target device to infer on; CPU, GPU, FPGA, HDDL or MYRIAD is"
" acceptable. The sample will look for a suitable plugin for device specified. "
"Default value is CPU", default="MYRIAD", type=str)
args.add_argument("--labels", help="Optional. Labels mapping file", default=None, type=str)
args.add_argument("-t", "--prob_threshold", help="Optional. Probability threshold for detections filtering",
default=0.5, type=float)
args.add_argument("-iout", "--iou_threshold", help="Optional. Intersection over union threshold for overlapping "
"detections filtering", default=0.4, type=float)
args.add_argument("-r", "--raw_output_message", help="Optional. Output inference results raw values showing",
default=False, action="store_true")
args.add_argument("-nireq", "--num_infer_requests", help="Optional. Number of infer requests",
default=1, type=int)
args.add_argument("-nstreams", "--num_streams",
help="Optional. Number of streams to use for inference on the CPU or/and GPU in throughput mode "
"(for HETERO and MULTI device cases use format <device1>:<nstreams1>,<device2>:<nstreams2> "
"or just <nstreams>)",
default="", type=str)
args.add_argument("-nthreads", "--number_threads",
help="Optional. Number of threads to use for inference on CPU (including HETERO cases)",
default=None, type=int)
args.add_argument("-loop_input", "--loop_input", help="Optional. Iterate over input infinitely",
action='store_true')
args.add_argument("-no_show", "--no_show", help="Optional. Don't show output", action='store_true')
args.add_argument("--keep_aspect_ratio", action="store_true", default=False,
help='Optional. Keeps aspect ratio on resize.')
return parser
class YoloParams:
# ------------------------------------------- Extracting layer parameters ------------------------------------------
# Magic numbers are copied from yolo samples
def __init__(self, param, side):
self.num = 3 if 'num' not in param else int(param['num'])
self.coords = 4 if 'coords' not in param else int(param['coords'])
self.classes = 80 if 'classes' not in param else int(param['classes'])
self.side = side
self.anchors = [10.0, 13.0, 16.0, 30.0, 33.0, 23.0, 30.0, 61.0, 62.0, 45.0, 59.0, 119.0, 116.0, 90.0, 156.0,
198.0,
373.0, 326.0] if 'anchors' not in param else param['anchors']
self.isYoloV3 = False
if param.get('mask'):
mask = param['mask']
self.num = len(mask)
maskedAnchors = []
for idx in mask:
maskedAnchors += [self.anchors[idx * 2], self.anchors[idx * 2 + 1]]
self.anchors = maskedAnchors
self.isYoloV3 = True # Weak way to determine but the only one.
class Modes(Enum):
USER_SPECIFIED = 0
MIN_LATENCY = 1
class Mode():
def __init__(self, value):
self.current = value
def next(self):
if self.current.value + 1 < len(Modes):
self.current = Modes(self.current.value + 1)
else:
self.current = Modes(0)
class ModeInfo():
def __init__(self):
self.last_start_time = perf_counter()
self.last_end_time = None
self.frames_count = 0
self.latency_sum = 0
def scale_bbox(x, y, height, width, class_id, confidence, im_h, im_w, is_proportional):
if is_proportional:
scale = np.array([min(im_w/im_h, 1), min(im_h/im_w, 1)])
offset = 0.5*(np.ones(2) - scale)
x, y = (np.array([x, y]) - offset) / scale
width, height = np.array([width, height]) / scale
xmin = int((x - width / 2) * im_w)
ymin = int((y - height / 2) * im_h)
xmax = int(xmin + width * im_w)
ymax = int(ymin + height * im_h)
# Method item() used here to convert NumPy types to native types for compatibility with functions, which don't
# support Numpy types (e.g., cv2.rectangle doesn't support int64 in color parameter)
return dict(xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, class_id=class_id.item(), confidence=confidence.item())
def parse_yolo_region(predictions, resized_image_shape, original_im_shape, params, threshold, is_proportional):
# ------------------------------------------ Validating output parameters ------------------------------------------
_, _, out_blob_h, out_blob_w = predictions.shape
assert out_blob_w == out_blob_h, "Invalid size of output blob. It sould be in NCHW layout and height should " \
"be equal to width. Current height = {}, current width = {}" \
"".format(out_blob_h, out_blob_w)
# ------------------------------------------ Extracting layer parameters -------------------------------------------
orig_im_h, orig_im_w = original_im_shape
resized_image_h, resized_image_w = resized_image_shape
objects = list()
size_normalizer = (resized_image_w, resized_image_h) if params.isYoloV3 else (params.side, params.side)
bbox_size = params.coords + 1 + params.classes
# ------------------------------------------- Parsing YOLO Region output -------------------------------------------
for row, col, n in np.ndindex(params.side, params.side, params.num):
# Getting raw values for each detection bounding box
bbox = predictions[0, n*bbox_size:(n+1)*bbox_size, row, col]
x, y, width, height, object_probability = bbox[:5]
class_probabilities = bbox[5:]
if object_probability < threshold:
continue
# Process raw value
x = (col + x) / params.side
y = (row + y) / params.side
# Value for exp is very big number in some cases so following construction is using here
try:
width = exp(width)
height = exp(height)
except OverflowError:
continue
# Depends on topology we need to normalize sizes by feature maps (up to YOLOv3) or by input shape (YOLOv3)
width = width * params.anchors[2 * n] / size_normalizer[0]
height = height * params.anchors[2 * n + 1] / size_normalizer[1]
class_id = np.argmax(class_probabilities)
confidence = class_probabilities[class_id]*object_probability
if confidence < threshold:
continue
objects.append(scale_bbox(x=x, y=y, height=height, width=width, class_id=class_id, confidence=confidence,
im_h=orig_im_h, im_w=orig_im_w, is_proportional=is_proportional))
return objects
def intersection_over_union(box_1, box_2):
width_of_overlap_area = min(box_1['xmax'], box_2['xmax']) - max(box_1['xmin'], box_2['xmin'])
height_of_overlap_area = min(box_1['ymax'], box_2['ymax']) - max(box_1['ymin'], box_2['ymin'])
if width_of_overlap_area < 0 or height_of_overlap_area < 0:
area_of_overlap = 0
else:
area_of_overlap = width_of_overlap_area * height_of_overlap_area
box_1_area = (box_1['ymax'] - box_1['ymin']) * (box_1['xmax'] - box_1['xmin'])
box_2_area = (box_2['ymax'] - box_2['ymin']) * (box_2['xmax'] - box_2['xmin'])
area_of_union = box_1_area + box_2_area - area_of_overlap
if area_of_union == 0:
return 0
return area_of_overlap / area_of_union
def resize(image, size, keep_aspect_ratio, interpolation=cv2.INTER_LINEAR):
if not keep_aspect_ratio:
return cv2.resize(image, size, interpolation=interpolation)
iw, ih = image.shape[0:2][::-1]
w, h = size
scale = min(w/iw, h/ih)
nw = int(iw*scale)
nh = int(ih*scale)
image = cv2.resize(image, (nw, nh), interpolation=interpolation)
new_image = np.full((size[1], size[0], 3), 128, dtype=np.uint8)
dx = (w-nw)//2
dy = (h-nh)//2
new_image[dy:dy+nh, dx:dx+nw, :] = image
return new_image
def preprocess_frame(frame, input_height, input_width, nchw_shape, keep_aspect_ratio):
in_frame = resize(frame, (input_width, input_height), keep_aspect_ratio)
if nchw_shape:
in_frame = in_frame.transpose((2, 0, 1)) # Change data layout from HWC to CHW
in_frame = np.expand_dims(in_frame, axis=0)
return in_frame
def get_objects(output, net, new_frame_height_width, source_height_width, prob_threshold, is_proportional):
objects = list()
function = ng.function_from_cnn(net)
for layer_name, out_blob in output.items():
out_blob = out_blob.buffer.reshape(net.outputs[layer_name].shape)
params = [x._get_attributes() for x in function.get_ordered_ops() if x.get_friendly_name() == layer_name][0]
layer_params = YoloParams(params, out_blob.shape[2])
objects += parse_yolo_region(out_blob, new_frame_height_width, source_height_width, layer_params,
prob_threshold, is_proportional)
return objects
def filter_objects(objects, iou_threshold, prob_threshold):
# Filtering overlapping boxes with respect to the --iou_threshold CLI parameter
objects = sorted(objects, key=lambda obj : obj['confidence'], reverse=True)
for i in range(len(objects)):
if objects[i]['confidence'] == 0:
continue
for j in range(i + 1, len(objects)):
if intersection_over_union(objects[i], objects[j]) > iou_threshold:
objects[j]['confidence'] = 0
return tuple(obj for obj in objects if obj['confidence'] >= prob_threshold)
def async_callback(status, callback_args):
request, frame_id, frame_mode, frame, start_time, completed_request_results, empty_requests, \
mode, event, callback_exceptions = callback_args
try:
if status != 0:
raise RuntimeError('Infer Request has returned status code {}'.format(status))
completed_request_results[frame_id] = (frame, request.output_blobs, start_time, frame_mode == mode.current)
if mode.current == frame_mode:
empty_requests.append(request)
except Exception as e:
callback_exceptions.append(e)
event.set()
def put_highlighted_text(frame, message, position, font_face, font_scale, color, thickness):
cv2.putText(frame, message, position, font_face, font_scale, (255, 255, 255), thickness + 1) # white border
cv2.putText(frame, message, position, font_face, font_scale, color, thickness)
def await_requests_completion(requests):
for request in requests:
request.wait()
def openvino_inference_engine():
args = build_argparser().parse_args()
# ------------- 1. Plugin initialization for specified device and load extensions library if specified -------------
ie = IECore()
#------------------------------search_device start------------------------------#
if(gv.gfMULTI == 1):
tmp_array = '' # Combine all MYRIAD device names.
cmd_array = '' # Output the command for MULTI:MYRIAD.
for device in ie.available_devices:
if(device.split("MYRIAD.")[0] == ''):
tmp_array = tmp_array + device + ','
cmd_array = "MULTI:" + tmp_array[:len(tmp_array) - 1]
args.device = cmd_array
print(args.device)
#------------------------------search_device end------------------------------#
config_user_specified = {}
config_min_latency = {}
devices_nstreams = {}
if args.num_streams:
devices_nstreams = {device: args.num_streams for device in ['CPU', 'GPU'] if device in args.device} \
if args.num_streams.isdigit() \
else dict([device.split(':') for device in args.num_streams.split(',')])
if 'CPU' in args.device:
if args.cpu_extension:
ie.add_extension(args.cpu_extension, 'CPU')
if args.number_threads is not None:
config_user_specified['CPU_THREADS_NUM'] = str(args.number_threads)
if 'CPU' in devices_nstreams:
config_user_specified['CPU_THROUGHPUT_STREAMS'] = devices_nstreams['CPU'] \
if int(devices_nstreams['CPU']) > 0 \
else 'CPU_THROUGHPUT_AUTO'
config_min_latency['CPU_THROUGHPUT_STREAMS'] = '1'
if 'GPU' in args.device:
if 'GPU' in devices_nstreams:
config_user_specified['GPU_THROUGHPUT_STREAMS'] = devices_nstreams['GPU'] \
if int(devices_nstreams['GPU']) > 0 \
else 'GPU_THROUGHPUT_AUTO'
config_min_latency['GPU_THROUGHPUT_STREAMS'] = '1'
# -------------------- 2. Reading the IR generated by the Model Optimizer (.xml and .bin files) --------------------
net = ie.read_network(args.model, os.path.splitext(args.model)[0] + ".bin")
# ---------------------------------- 3. Load CPU extension for support specific layer ------------------------------
# ---------------------------------------------- 4. Preparing inputs -----------------------------------------------
input_blob = next(iter(net.input_info))
# Read and pre-process input images
if net.input_info[input_blob].input_data.shape[1] == 3:
input_height, input_width = net.input_info[input_blob].input_data.shape[2:]
nchw_shape = True
else:
input_height, input_width = net.input_info[input_blob].input_data.shape[1:3]
nchw_shape = False
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.strip() for x in f]
else:
labels_map = None
input_stream = 0 if args.input == "cam" else args.input
image = cv2.imread(input_stream)
deside_cut_size(image.shape[1], image.shape[0])
crop_img = image[y:y+h, x:x+w]
cv2.imwrite('img/tmp.jpg', crop_img)
input_stream = 'img/tmp.jpg'
mode = Mode(Modes.USER_SPECIFIED)
cap = cv2.VideoCapture(input_stream)
wait_key_time = 1
# ----------------------------------------- 5. Loading model to the plugin -----------------------------------------
exec_nets = {}
exec_nets[Modes.USER_SPECIFIED] = ie.load_network(network=net, device_name=args.device,
config=config_user_specified,
num_requests=args.num_infer_requests)
exec_nets[Modes.MIN_LATENCY] = ie.load_network(network=net, device_name=args.device.split(":")[-1].split(",")[0],
config=config_min_latency,
num_requests=1)
empty_requests = deque(exec_nets[mode.current].requests)
completed_request_results = {}
next_frame_id = 0
next_frame_id_to_show = 0
mode_info = { mode.current: ModeInfo() }
event = threading.Event()
callback_exceptions = []
# ----------------------------------------------- 6. Doing inference -----------------------------------------------
while (cap.isOpened() \
or completed_request_results \
or len(empty_requests) < len(exec_nets[mode.current].requests)) \
and not callback_exceptions:
if next_frame_id_to_show in completed_request_results:
frame, output, start_time, is_same_mode = completed_request_results.pop(next_frame_id_to_show)
next_frame_id_to_show += 1
if is_same_mode:
mode_info[mode.current].frames_count += 1
objects = get_objects(output, net, (input_height, input_width), frame.shape[:-1], args.prob_threshold,
args.keep_aspect_ratio)
objects = filter_objects(objects, args.iou_threshold, args.prob_threshold)
origin_im_size = frame.shape[:-1]
for obj in objects:
# Validation bbox of detected object
obj['xmax'] = min(obj['xmax'], origin_im_size[1])
obj['ymax'] = min(obj['ymax'], origin_im_size[0])
obj['xmin'] = max(obj['xmin'], 0)
obj['ymin'] = max(obj['ymin'], 0)
color = (0, 0, 255)
det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \
str(obj['class_id'])
if(det_label == '0'):
out_det_label = "Normal"
elif(det_label == '1'):
out_det_label = "MP"
else:
out_det_label = det_label
cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2)
cv2.putText(frame,
"#" + out_det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + '%',
(obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 2, color, 2)
if not args.no_show:
cv2.imwrite('img/tmp.jpg', frame) # Write image for showing the inference result on UI.
elif empty_requests and cap.isOpened():
start_time = perf_counter()
ret, frame = cap.read()
if not ret:
if args.loop_input:
cap.open(input_stream)
else:
cap.release()
continue
request = empty_requests.popleft()
# resize input_frame to network size
in_frame = preprocess_frame(frame, input_height, input_width, nchw_shape, args.keep_aspect_ratio)
# Start inference
request.set_completion_callback(py_callback=async_callback,
py_data=(request,
next_frame_id,
mode.current,
frame,
start_time,
completed_request_results,
empty_requests,
mode,
event,
callback_exceptions))
request.async_infer(inputs={input_blob: in_frame})
next_frame_id += 1
else:
event.wait()
if callback_exceptions:
raise callback_exceptions[0]
for exec_net in exec_nets.values():
await_requests_completion(exec_net.requests)
My design is that the user can choose an image and click the button, then the inference result will show up.
And this step can be sequentially redone again and again.
Afterwards I try to use MULTI:MYRIAD with two to three NCS2s, in order to accelerate the inference time.
But using two or three NCS2s always has worse time consumption than only one NCS2.
And usually show "XLink_sem_wait:94" and "XLinkResetRemote:257" logs while redoing the inference job.
[35mE: [global] [ 0] [] XLink_sem_wait:94 XLink_sem_inc(sem) method call failed with an error: -1[0m
[35mE: [global] [ 0] [] XLinkResetRemote:257 can't wait dispatcherClosedSem
[0m
It can normally show the inference result even if the above logs appear, but I believe that this is why MULTI:MYRIAD doesn't work well.
Solution
The optimum way to use this Multi-plugin with multiple devices is by configuring the individual devices and creating the Multi-Device on top.
For example:
myriad1_config = {}
myriad2_config = {}
ie.set_config(config=myriad1_config, device_name="MYRIAD.3.1-ma2480")
ie.set_config(config=myriad2_config, device_name="MYRIAD.3.3-ma2480")
# Load the network to the multi-device, specifying the priorities
exec_net = ie.load_network(
network=net, device_name="MULTI", config={"MULTI_DEVICE_PRIORITIES": "MYRIAD.3.1-ma2480,MYRIAD.3.3-ma2480"}
)
# Query the optimal number of requests
nireq = exec_net.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS")
It is suggested to use -nireq 10. This should give maximum FPS. You may refer to the object_detection_demo.py to see how this -nireq argument is used.
The proper command should be:
python object_detection_demo.py i 0 -d MULTI:MYRIAD.3.1-ma2480,MYRIAD.3.3-ma2480 -m yolov4.xml -at yolo -nireq 10
- WSGIServerException: [Errno 8] nodename nor servname provided, or not known
- Python/Flask Login form throws 500 error on IIS
- Matplotlib plt.show() isn't showing graph
- How to avoid duplicate processing in python API server?
- Extract points/coordinates from a polygon in Shapely
- How do I detect collision in pygame?
- Typehint a method that returns new instance using a superclass classmethod
- How to process requests from multiiple users using ML model and FastAPI?
- How can I clear the interpreter console?
- What are the reasons for the differences in output results when using the groupby function in the Python pandas package?
- How to antialias picture to get rid of the jerkiness in the image
- Is there any difference between list(array) and array.tolist() in Python?
- How to move x-axis on top of the plot in plotnine?
- Pulling Zillow Rent Data from Zillow API
- Safest way to incrementally append to a file?
- Unable to install mysqlclient package on EC2 instance
- FastAPI - How to get app instance inside a router?
- SQLAlchemy: engine, connection and session difference
- Generating binary arrays with alternating values based on change indices in NumPy
- How can i solve this error: ValueError: The feature names should match those that were passed
- 'super' object has no attribute '__sklearn_tags__'
- Python: How to copy all attibutes from base class to derived one
- mitm proxy is closed, but the port is still occupied
- How to find pg_config path
- Why does "pip install" inside Python raise a SyntaxError?
- Plotting a matplotlib pie chart using 2 columns
- ImportError: No module named langchain.llms
- How do I disable the security certificate check in Python requests
- How can I import Psycopg2 into a Lambda Function on AWS?
- Django built in Logout view `Method Not Allowed (GET): /users/logout/`