27 #include <opencv2/dnn.hpp>
31 namespace jevois {
namespace dnn {
namespace yunet {
36 PriorBox(cv::Size
const & input_shape, cv::Size
const & output_shape)
39 in_w = input_shape.width;
40 in_h = input_shape.height;
41 out_w = output_shape.width;
42 out_h = output_shape.height;
44 cv::Size feature_map_2nd { int(
int((in_w+1)/2)/2), int(
int((in_h+1)/2)/2) };
45 cv::Size feature_map_3rd { int(feature_map_2nd.width/2), int(feature_map_2nd.height/2) };
46 cv::Size feature_map_4th { int(feature_map_3rd.width/2), int(feature_map_3rd.height/2) };
47 cv::Size feature_map_5th { int(feature_map_4th.width/2), int(feature_map_4th.height/2) };
48 cv::Size feature_map_6th { int(feature_map_5th.width/2), int(feature_map_5th.height/2) };
51 feature_map_sizes.push_back(feature_map_3rd);
52 feature_map_sizes.push_back(feature_map_4th);
53 feature_map_sizes.push_back(feature_map_5th);
54 feature_map_sizes.push_back(feature_map_6th);
57 for (
size_t i = 0; i < feature_map_sizes.size(); ++i)
59 cv::Size feature_map_size = feature_map_sizes[i];
60 std::vector<float> min_size = min_sizes[i];
62 for (
int _h = 0; _h < feature_map_size.height; ++_h)
64 for (
int _w = 0; _w < feature_map_size.width; ++_w)
66 for (
size_t j = 0; j < min_size.size(); ++j)
68 float s_kx = min_size[j] / in_w;
69 float s_ky = min_size[j] / in_h;
71 float cx = (_w + 0.5) * steps[i] / in_w;
72 float cy = (_h + 0.5) * steps[i] / in_h;
74 Box anchor = { cx, cy, s_kx, s_ky };
75 priors.push_back(anchor);
85 std::vector<Face>
decode(cv::Mat
const & loc, cv::Mat
const & conf, cv::Mat
const & iou,
86 float const ignore_score = 0.3)
88 std::vector<Face> dets;
90 float const * loc_v = (
float const *)(loc.data);
91 float const * conf_v = (
float const *)(conf.data);
92 float const * iou_v = (
float const *)(iou.data);
93 for (
size_t i = 0; i < priors.size(); ++i)
96 float cls_score = conf_v[i*2+1];
97 float iou_score = iou_v[i];
100 if (iou_score < 0.f) iou_score = 0.f;
else if (iou_score > 1.f) iou_score = 1.f;
101 float score = std::sqrt(cls_score * iou_score);
104 if (score < ignore_score) {
continue; }
110 float cx = (priors[i].x + loc_v[i*14+0] * variance[0] * priors[i].width) * out_w;
111 float cy = (priors[i].y + loc_v[i*14+1] * variance[0] * priors[i].height) * out_h;
112 float w = priors[i].width * exp(loc_v[i*14+2] * variance[0]) * out_w;
113 float h = priors[i].height * exp(loc_v[i*14+3] * variance[1]) * out_h;
114 float x1 = cx - w / 2;
115 float y1 = cy -
h / 2;
119 float x_re = (priors[i].x + loc_v[i*14+ 4] * variance[0] * priors[i].width) * out_w;
120 float y_re = (priors[i].y + loc_v[i*14+ 5] * variance[0] * priors[i].height) * out_h;
121 float x_le = (priors[i].x + loc_v[i*14+ 6] * variance[0] * priors[i].width) * out_w;
122 float y_le = (priors[i].y + loc_v[i*14+ 7] * variance[0] * priors[i].height) * out_h;
123 float x_n = (priors[i].x + loc_v[i*14+ 8] * variance[0] * priors[i].width) * out_w;
124 float y_n = (priors[i].y + loc_v[i*14+ 9] * variance[0] * priors[i].height) * out_h;
125 float x_mr = (priors[i].x + loc_v[i*14+10] * variance[0] * priors[i].width) * out_w;
126 float y_mr = (priors[i].y + loc_v[i*14+11] * variance[0] * priors[i].height) * out_h;
127 float x_ml = (priors[i].x + loc_v[i*14+12] * variance[0] * priors[i].width) * out_w;
128 float y_ml = (priors[i].y + loc_v[i*14+13] * variance[0] * priors[i].height) * out_h;
137 dets.push_back(face);
143 std::vector<std::vector<float>>
const min_sizes
145 {10.0f, 16.0f, 24.0f},
148 {128.0f, 192.0f, 256.0f}
150 std::vector<int>
const steps { 8, 16, 32, 64 };
151 std::vector<float>
const variance { 0.1, 0.2 };
158 std::vector<cv::Size> feature_map_sizes;
159 std::vector<Box> priors;
160 std::vector<Box> generate_priors();
179 if (outs.size() != 3)
LFATAL(
"Need exactly 3 outputs, received " << outs.size());
180 if (outs[0].rows < 16 || outs[0].cols != 2)
LFATAL(
"loc size " << outs[0].size() <<
" instead of 2xN_Anchors");
181 if (outs[1].rows < 16 || outs[1].cols != 1)
LFATAL(
"conf size " << outs[1].size() <<
" instead of 1xN_Anchors");
182 if (outs[2].rows < 16 || outs[2].cols != 14)
LFATAL(
"iou size " << outs[2].size() <<
" instead of 14xN_Anchors");
186 float const nmsThreshold =
nms::get() * 0.01F;
194 cv::Size
const bsiz = preproc->
blobsize(0);
200 std::vector<jevois::dnn::yunet::Face> dets = itsPriorBox->decode(outs[2], outs[0], outs[1], boxThreshold);
203 itsDetections.clear();
206 std::vector<cv::Rect> face_boxes;
207 std::vector<float> face_scores;
208 for (
auto const & d : dets) { face_boxes.push_back(d.bbox_tlwh); face_scores.push_back(d.score); }
210 std::vector<int> keep_idx;
211 cv::dnn::NMSBoxes(face_boxes, face_scores, confThreshold, nmsThreshold, keep_idx, 1.f,
top::get());
212 for (
size_t i = 0; i < keep_idx.size(); i++) itsDetections.emplace_back(dets[keep_idx[i]]);
221 cv::Point2f tl = b.tl(); preproc->
b2i(tl.x, tl.y);
222 cv::Point2f br = b.br(); preproc->
b2i(br.x, br.y);
223 b.x = tl.x; b.y = tl.y; b.width = br.x - tl.x; b.height = br.y - tl.y;
225 preproc->
b2i(f.landmarks.right_eye.x, f.landmarks.right_eye.y);
226 preproc->
b2i(f.landmarks.left_eye.x, f.landmarks.left_eye.y);
227 preproc->
b2i(f.landmarks.nose_tip.x, f.landmarks.nose_tip.y);
228 preproc->
b2i(f.landmarks.mouth_right.x, f.landmarks.mouth_right.y);
229 preproc->
b2i(f.landmarks.mouth_left.x, f.landmarks.mouth_left.y);
236 bool JEVOIS_UNUSED_PARAM(idle))
241 std::string
const scorestr =
jevois::sformat(
"face: %.1f%%", f.score * 100.0F);
244 if (outimg && overlay)
246 unsigned int const col = jevois::yuyv::LightGreen;
247 unsigned int const rad = 5;
260 f.landmarks.mouth_right.x, f.landmarks.mouth_right.y, 2, col);
267 ImU32
const col = 0x80ff80ff;
268 float const rad = 5.0f;
271 helper->
drawRect(b.x, b.y, b.x + b.width, b.y + b.height, col,
true);
272 helper->
drawText(b.x + 3.0f, b.y + 3.0f, scorestr.c_str(), col);
275 helper->
drawCircle(f.landmarks.right_eye.x, f.landmarks.right_eye.y, rad*2, IM_COL32(255,0,0,255));
276 helper->
drawCircle(f.landmarks.left_eye.x, f.landmarks.left_eye.y, rad*2, IM_COL32(0,0,255,255));
277 helper->
drawCircle(f.landmarks.nose_tip.x, f.landmarks.nose_tip.y, rad, IM_COL32(0,255,0,255));
278 helper->
drawCircle(f.landmarks.mouth_right.x, f.landmarks.mouth_right.y, rad, IM_COL32(255,0,255,255));
279 helper->
drawCircle(f.landmarks.mouth_left.x, f.landmarks.mouth_left.y, rad, IM_COL32(0,255,255,255));
280 helper->
drawLine(f.landmarks.mouth_left.x, f.landmarks.mouth_left.y,
281 f.landmarks.mouth_right.x, f.landmarks.mouth_right.y, IM_COL32(255,255,255,255));