JeVoisBase  1.21
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Loading...
Searching...
No Matches
PyPostDAMOyolo.py
Go to the documentation of this file.
1import pyjevois
2if pyjevois.pro: import libjevoispro as jevois
3else: import libjevois as jevois
4
5import numpy as np
6import cv2
7
8## Python DNN post-processor for DAMO YOLO
9#
10# Adapted from https://github.com/PINTO0309/PINTO_model_zoo/blob/main/334_DAMO-YOLO/demo/demo_DAMO-YOLO_onnx.py and
11# https://github.com/tinyvision/DAMO-YOLO/blob/master/tools/demo.py
12#
13# @author Laurent Itti
14#
15# @email itti\@usc.edu
16# @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
17# @copyright Copyright (C) 2023 by Laurent Itti, iLab and the University of Southern California
18# @mainurl http://jevois.org
19# @supporturl http://jevois.org/doc
20# @otherurl http://iLab.usc.edu
21# @license GPL v3
22# @distribution Unrestricted
23# @restrictions None
24# @ingroup pydnn
26 # ###################################################################################################
27 ## Constructor
28 def __init__(self):
29 # results of process(), held here for use by report():
30 self.classIds = []
31 self.confidences = []
32 self.boxes = []
33
34 # map from class index to class name:
35 self.classmap = None
36
37 # ###################################################################################################
38 ## JeVois parameters initialization. These can be set by users in the GUI or JeVois model zoo file
39 def init(self):
40 pc = jevois.ParameterCategory("DNN Post-Processing Options", "")
41
42 self.classes = jevois.Parameter(self, 'classes', 'str',
43 "Path to text file with names of object classes",
44 '', pc)
45 self.classes.setCallback(self.loadClassesloadClasses)
46
47 self.nms = jevois.Parameter(self, 'nms', 'float',
48 "Non-maximum suppression intersection-over-union threshold in percent",
49 45.0, pc)
50
51 self.maxnbox = jevois.Parameter(self, 'maxnbox', 'uint',
52 "Max number of top-scoring boxes to report",
53 500, pc)
54
55 self.cthresh = jevois.Parameter(self, 'cthresh', 'float',
56 "Detection/classification score threshold, in percent confidence",
57 20.0, pc)
58
59 # ###################################################################################################
60 ## Freeze some parameters that should not be changed at runtime
61 def freeze(self, doit):
62 self.classes.freeze(doit)
63
64 # ###################################################################################################
65 ## Parameter callback: Load class names when 'classes' parameter value is changed by model zoo
66 def loadClasses(self, filename):
67 if filename:
68 jevois.LINFO(f"Loading {filename}...")
69 f = open(pyjevois.share + '/' + filename, 'rt') # will throw if file not found
70 self.classmap = f.read().rstrip('\n').split('\n')
71
72 # ###################################################################################################
73 ## Multiclass non-maximum suppression as implemented by PINTO0309
74 def multiclass_nms(self, bboxes, scores, score_th, nms_th, max_num):
75 num_classes = scores.shape[1]
76 bboxes = np.broadcast_to(bboxes[:, None], (bboxes.shape[0], num_classes, 4), )
77 valid_mask = scores > score_th
78 bboxes = bboxes[valid_mask]
79 scores = scores[valid_mask]
80
81 np_labels = valid_mask.nonzero()[1]
82
83 indices = cv2.dnn.NMSBoxes(bboxes.tolist(), scores.tolist(), score_th, nms_th)
84 # Note to Pinto0309: I believe NMSBoxes expects boxes as (x, y, w, h) but our boxes here are (x1, y1, x2, y2)
85
86 if max_num > 0:
87 indices = indices[:max_num]
88
89 if len(indices) > 0:
90 bboxes = bboxes[indices]
91 scores = scores[indices]
92 np_labels = np_labels[indices]
93 return bboxes, scores, np_labels
94 else:
95 return np.array([]), np.array([]), np.array([])
96
97 # ###################################################################################################
98 ## Post-processing as implemented by PINTO0309
99 def postprocess(self, scores, bboxes, score_th, nms_th):
100 batch_size = bboxes.shape[0]
101 for i in range(batch_size):
102 if not bboxes[i].shape[0]: continue
103 bboxes, scores, class_ids = self.multiclass_nms(bboxes[i], scores[i], score_th, nms_th, self.maxnbox.get())
104
105 return bboxes, scores, class_ids
106 # Note: if batch size > 1 this would only return the last results; anyway, batch size always 1 on JeVois
107
108 # ###################################################################################################
109 ## Get network outputs
110 # outs is a list of numpy arrays for the network's outputs.
111 # preproc is a handle to the pre-processor that was used, useful to recover transforms from original image
112 # to cropped/resized network inputs.
113 def process(self, outs, preproc):
114 if (len(outs) != 2): jevois.LFATAL("Need 2 outputs: scores, bboxes")
115
116 # To draw boxes, we will need to:
117 # - scale from [0..1]x[0..1] to blobw x blobh
118 # - scale and center from blobw x blobh to input image w x h, provided by PreProcessor::b2i()
119 # - when using the GUI, we further scale and translate to OpenGL display coordinates using GUIhelper::i2d()
120 # Here we assume that the first blob sets the input size.
121 bw, bh = preproc.blobsize(0)
122
123 boxes, self.confidences, self.classIds = self.postprocess(outs[0], outs[1], self.cthresh.get() * 0.01,
124 self.nms.get() * 0.01)
125
126 # Now clamp boxes to be within blob, and adjust the boxes from blob size to input image size:
127 self.boxes.clear()
128
129 for b in boxes:
130 x1, y1, x2, y2 = b
131
132 # Clamp box coords to within network's input blob:
133 x1 = float(min(bw - 1, max(0, x1)))
134 x2 = float(min(bw - 1, max(0, x2)))
135 y1 = float(min(bh - 1, max(0, y1)))
136 y2 = float(min(bh - 1, max(0, y2)))
137
138 # Scale box from input blob to input image:
139 x1, y1 = preproc.b2i(x1, y1, 0)
140 x2, y2 = preproc.b2i(x2, y2, 0)
141
142 self.boxes.append( [x1, y1, x2, y2] )
143
144 # Note: further scaling and translation to OpenGL display coords is handled internally by GUI helper
145 # using GUIhelper::i2d() when we call helper.drawRect(), etc below in report()
146
147 # ###################################################################################################
148 ## Helper to get class name and confidence as a clean string, and a color that varies with class name
149 def getLabel(self, id, conf):
150 if self.classmap is None or id < 0 or id >= len(self.classmap): categ = 'unknown'
151 else: categ = self.classmap[id]
152
153 color = jevois.stringToRGBA(categ, 255)
154
155 return ( ("%s: %.2f" % (categ, conf * 100.0)), color & 0xffffffff)
156
157 # ###################################################################################################
158 ## Report the latest results obtained by process() by drawing them
159 # outimg is None or a RawImage to draw into when in Legacy mode (drawing to an image sent to USB)
160 # helper is None or a GUIhelper to do OpenGL drawings when in JeVois-Pro GUI mode
161 # overlay is True if user wishes to see overlay text
162 # idle is true if keyboard/mouse have been idle for a while, which typically would reduce what is displayed
163 #
164 # Note that report() is called on every frame even though the network may run slower or take some time to load and
165 # initialize, thus you should be prepared for report() being called even before process() has ever been called
166 # (i.e., create some class member variables to hold the reported results, initialize them to some defaults in your
167 # constructor, report their current values here, and update their values in process()).
168 def report(self, outimg, helper, overlay, idle):
169
170 # Legacy JeVois mode: Write results into YUYV RawImage to send over USB:
171 if outimg is not None:
172 if overlay:
173 for i in range(len(self.classIds)):
174 label, color = self.getLabel(self.classIds[i], self.confidences[i])
175 x1, y1, x2, y2 = self.boxes[i]
176 jevois.drawRect(outimg, x1, y1, x2 - x1, y2 - y1, 2, jevois.YUYV.LightGreen)
177 jevois.writeText(outimg, label, x1 + 6, y1 + 2, jevois.YUYV.LightGreen, jevois.Font.Font10x20)
178
179 # JeVois-Pro mode: Write the results as OpenGL overlay boxes and text on top of the video:
180 if helper is not None:
181 if overlay:
182 for i in range(len(self.classIds)):
183 label, color = self.getLabel(self.classIds[i], self.confidences[i])
184 x1, y1, x2, y2 = self.boxes[i]
185 helper.drawRect(x1, y1, x2, y2, color & 0xffffffff, True)
186 helper.drawText(x1 + 3, y1 + 3, label, color & 0xffffffff)
187
188 # Could here send serial messages, or do some other processing over classIds, confidences and boxes
189
Python DNN post-processor for DAMO YOLO.
freeze(self, doit)
Freeze some parameters that should not be changed at runtime.
init(self)
JeVois parameters initialization.
loadClasses(self, filename)
Parameter callback: Load class names when 'classes' parameter value is changed by model zoo.
postprocess(self, scores, bboxes, score_th, nms_th)
Post-processing as implemented by PINTO0309.
process(self, outs, preproc)
Get network outputs outs is a list of numpy arrays for the network's outputs.
multiclass_nms(self, bboxes, scores, score_th, nms_th, max_num)
Multiclass non-maximum suppression as implemented by PINTO0309.
report(self, outimg, helper, overlay, idle)
Report the latest results obtained by process() by drawing them outimg is None or a RawImage to draw ...
getLabel(self, id, conf)
Helper to get class name and confidence as a clean string, and a color that varies with class name.