JeVoisBase  1.21
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Loading...
Searching...
No Matches
Saliency.C
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
19
26
28#include <jevois/Debug/Log.H>
29#include <jevois/Debug/Timer.H>
32
33#include <cstdlib>
34#include <future>
35#include <functional> // for placeholders
36
37#define WEIGHT_SCALEBITS ((env_size_t) 8)
38
39// ##############################################################################################################
40static int computeGist(const char * tagName, env_size_t clev, env_size_t slev, struct env_image* submap,
41 const struct env_image * JEVOIS_UNUSED_PARAM(center),
42 const struct env_image * JEVOIS_UNUSED_PARAM(surround), void * vdata)
43{
44 //LINFO(tagName << ": cs=" << clev << '-' << slev << " submap " << submap->dims.w << 'x' << submap->dims.h);
45
46 // We should be called 72 times, once for each of our 72 feature maps. Our gist contains 16 values (4x4 grid) for each
47 // of the maps, so 72*16=1152 values in total. Each single channel has 6 maps. First, parse the tagname to get the
48 // offset in our gist vector at the single channel level:
49
50 Saliency::visitor_data * vd = reinterpret_cast<Saliency::visitor_data *>(vdata);
51
52 env_size_t const onechan = 6 * 16;
53 env_size_t offset; unsigned int bitshift = 0;
54 switch (tagName[0])
55 {
56 case 'r':
57 if (tagName[2] == 'd') { offset = 0; bitshift = 6; } // red/green
58 else { offset = (std::atoi(tagName + 10) + 6) * onechan; bitshift = 4; } // reichardt(%d/%d) with number 1-based
59 break;
60 case 'b': offset = onechan; bitshift = 6; break; // blue/yellow
61 case 'i': offset = 2 * onechan; bitshift = 7; break; // intensity
62 case 's': offset = (std::atoi(tagName + 10) + 2) * onechan; bitshift = 0; break; // steerable(%d/%d), 1-based
63 case 'f': offset = 7 * onechan; bitshift = 5; break; // flicker
64 default: LFATAL("Unknown channel " << tagName);
65 }
66
67 // Now we need to compute an additional offset between 0 and 5 depending on clev and slev:
68 // clev in { 2, 3, 4 }, delta = slev - clev in { 3, 4 }
69 // FIXME: use envp to get the levels, etc
70 env_size_t const delta = slev - clev;
71 offset += ((delta - vd->envp->cs_del_min) * 3 + clev - vd->envp->cs_lev_min) * 16;
72 if (offset + 16 > vd->gist_size) LFATAL("gist offset " << offset << " out of range");
73
74 // All right, fill in the data:
75 env_grid_average(submap, vd->gist + offset, bitshift, 4, 4);
76
77 return 0;
78}
79
80// ##############################################################################################################
81Saliency::Saliency(std::string const & instance) :
82 jevois::Component(instance), gist_size(72 * 16), itsProfiler("Saliency", 100, LOG_DEBUG), itsInputDone(true)
83{
85
86 env_init_integer_math(&imath, &envp);
87
88 env_img_init_empty(&prev_input);
89 env_pyr_init(&prev_lowpass5, 0);
90 env_motion_channel_init(&motion_chan, &envp);
91
98 gist = new unsigned char[gist_size];
99
100 itsVisitorData.gist = gist;
101 itsVisitorData.gist_size = gist_size;
102 itsVisitorData.envp = &envp;
103}
104
105// ##############################################################################################################
107{
108 delete [] gist;
109 env_img_make_empty(&prev_input);
110 env_pyr_make_empty(&prev_lowpass5);
111 env_motion_channel_destroy(&motion_chan);
112}
113
114// ##############################################################################################################
115void Saliency::combine_output(struct env_image* chanOut, const intg32 iweight, struct env_image* result)
116{
117 if (!env_img_initialized(chanOut)) return;
118
119 intg32* const sptr = env_img_pixelsw(chanOut);
120 const env_size_t sz = env_img_size(chanOut);
121
122 // Lock here so that we combine one channel at a time:
123 std::lock_guard<std::mutex> _(itsMtx);
124
125 if (!env_img_initialized(result))
126 {
127 env_img_resize_dims(result, chanOut->dims);
128 intg32* const dptr = env_img_pixelsw(result);
129 for (env_size_t i = 0; i < sz; ++i)
130 {
131 sptr[i] = (sptr[i] >> WEIGHT_SCALEBITS) * iweight;
132 dptr[i] = sptr[i];
133 }
134 }
135 else
136 {
137 ENV_ASSERT(env_dims_equal(chanOut->dims, result->dims));
138 intg32* const dptr = env_img_pixelsw(result);
139 const env_size_t sz = env_img_size(result);
140 for (env_size_t i = 0; i < sz; ++i)
141 {
142 sptr[i] = (sptr[i] >> WEIGHT_SCALEBITS) * iweight;
143 dptr[i] += sptr[i];
144 }
145 }
146}
147
148#define SALUPDATE(envval, param) \
149 prev = envp.envval; envp.envval = saliency::param::get(); if (envp.envval != prev) nuke = true;
150
151// ##############################################################################################################
152void Saliency::processStart(struct env_dims const & dims, bool do_gist)
153{
154 // Mark our input image as being processed:
155 {
156 std::unique_lock<std::mutex> ulck(itsRawImageMtx);
157 itsInputDone = false;
158 }
159
160 bool nuke = false; size_t prev;
161
162 // Update envp with our current parameters:
163 envp.chan_c_weight = saliency::cweight::get();
164 envp.chan_i_weight = saliency::iweight::get();
165 envp.chan_o_weight = saliency::oweight::get();
166 envp.chan_f_weight = saliency::fweight::get();
167 envp.chan_m_weight = saliency::mweight::get();
168
169 SALUPDATE(cs_lev_min, centermin);
170 envp.cs_lev_max = envp.cs_lev_min + 2;
171
172 SALUPDATE(cs_del_min, deltamin);
173 envp.cs_del_max = envp.cs_del_min + 1;
174
175 SALUPDATE(output_map_level, smscale);
176
177 envp.motion_thresh = saliency::mthresh::get();
178
179 envp.flicker_thresh = saliency::fthresh::get();
180
181 prev = envp.multiscale_flicker;
182 envp.multiscale_flicker = saliency::msflick::get() ? 1 : 0;
183 if (envp.multiscale_flicker != prev) nuke = true;
184
185 env_params_validate(&envp);
186
187 // Zero-out all our internals:
194 memset(gist, 0, gist_size);
195
196 // Reject bad images:
197 if (dims.w < 32 || dims.h < 32) LFATAL("input dims " << dims.w << 'x' << dims.h << " too small -- REJECTED");
198 if (dims.w > 2048 || dims.h > 2048) LFATAL("input dims " << dims.w << 'x' << dims.h << " too large -- REJECTED");
199
200 // Check whether the input size or critical params just changed, and if so invalidate our previous stored data:
201 if (env_img_initialized(&prev_input) && (prev_input.dims.w != dims.w || prev_input.dims.h != dims.h)) nuke = true;
202
203 if (nuke)
204 {
205 env_img_make_empty(&prev_input);
206 env_pyr_make_empty(&prev_lowpass5);
207 env_motion_channel_destroy(&motion_chan);
208 env_motion_channel_init(&motion_chan, &envp);
209 }
210
211 // Install hook for gist computation, if desired:
212 if (do_gist) { envp.user_data_preproc = &itsVisitorData; envp.submapPreProc = &computeGist; }
213 else { envp.user_data_preproc = nullptr; envp.submapPreProc = nullptr; }
214}
215
216// ##############################################################################################################
218{
219 std::unique_lock<std::mutex> ulck(itsRawImageMtx);
220 if (itsInputDone) return; // we are done already
221 itsRawImageCond.wait(ulck, [&]() { return itsInputDone; } );
222}
223
224// ##############################################################################################################
225void Saliency::process(cv::Mat const & input, bool do_gist)
226{
227 static env_chan_status_func * statfunc = nullptr;
228 static void * statdata = nullptr;
229
230 // We here do what env_mt_visual_cortex_inut used to do in the original envision code, but using lambdas instead of
231 // the c-based jobs:
232 struct env_dims dims = { (env_size_t)input.cols, (env_size_t)input.rows };
233 processStart(dims, do_gist);
234 struct env_rgb_pixel * inpixels = reinterpret_cast<struct env_rgb_pixel *>(input.data);
235
236 const intg32 total_weight = env_total_weight(&envp);
237 ENV_ASSERT(total_weight > 0);
238
239 /* We want to compute
240
241 * weight
242 * img * ------------
243 * total_weight
244 *
245 *
246 * To do that without overflowing, we compute it as
247 *
248 *
249 * weight 256
250 * img * ------------ * ---
251 * total_weight 256
252 *
253 * img weight * 256
254 * = ( --- ) * ( ------------ )
255 * 256 total_weight
256 *
257 * where 256 is an example of (1<<WEIGHT_SCALEBITS) for
258 * WEIGHT_SCALEBITS=8.
259 */
260
261 // We can get the color channel started right away:
262 std::future<void> colorfut;
263 if (envp.chan_c_weight > 0)
264 colorfut = jevois::async([&](){
265 env_chan_color("color", &envp, &imath, inpixels, dims, statfunc, statdata, &color);
266 combine_output(&color, envp.chan_c_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
267 });
268
269 // Compute luminance image:
270 struct env_image bwimg; env_img_init(&bwimg, dims);
271 env_c_luminance_from_byte(inpixels, dims.w * dims.h, imath.nbits, env_img_pixelsw(&bwimg));
272
273 // Notify anyone that was waiting to free the raw input that we are done with it:
274 itsInputDone = true; itsRawImageCond.notify_all();
275
276 // Compute a luminance pyramid:
277 struct env_pyr lowpass5; env_pyr_init(&lowpass5, env_max_pyr_depth(&envp));
278 env_pyr_build_lowpass_5(&bwimg, envp.cs_lev_min, &imath, &lowpass5);
279
280 // Now parallelize the other channels:
281 std::future<void> motfut;
282 if (envp.chan_m_weight > 0)
283 motfut = jevois::async([&](){
284 env_mt_motion_channel_input(&motion_chan, "motion", bwimg.dims, &lowpass5, statfunc, statdata, &motion);
285 combine_output(&motion, envp.chan_m_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
286 });
287
288 std::future<void> orifut;
289 if (envp.chan_o_weight > 0)
290 orifut = jevois::async([&](){
291 env_mt_chan_orientation("orientation", &bwimg, statfunc, statdata, &ori);
292 combine_output(&ori, envp.chan_o_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
293 });
294
295 std::future<void> flickfut;
296 if (envp.chan_f_weight > 0)
297 flickfut = jevois::async([&](){
298 if (envp.multiscale_flicker)
299 env_chan_msflicker("flicker", &envp, &imath, bwimg.dims, &prev_lowpass5, &lowpass5,
300 statfunc, statdata, &flicker);
301 else
302 env_chan_flicker("flicker", &envp, &imath, &prev_input, &bwimg, statfunc, statdata, &flicker);
303
304 combine_output(&flicker, envp.chan_f_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
305
306 if (envp.multiscale_flicker) env_pyr_copy_src_dst(&lowpass5, &prev_lowpass5);
307 else env_pyr_make_empty(&prev_lowpass5);
308 });
309
310 // Intensity is the fastest one and we here just run it in the current thread:
311 if (envp.chan_i_weight > 0)
312 {
313 env_chan_intensity("intensity", &envp, &imath, bwimg.dims, &lowpass5, 1, statfunc, statdata, &intens);
314 combine_output(&intens, envp.chan_i_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
315 }
316
317 // Wait for all channels to finish up:
318 JEVOIS_WAIT_GET_FUTURE(colorfut);
320 JEVOIS_WAIT_GET_FUTURE(flickfut);
322
323 // Cleanup and get ready for next frame:
324 if (!envp.multiscale_flicker) env_img_swap(&prev_input, &bwimg); else env_img_make_empty(&prev_input);
325
326 if (statfunc) (*statfunc)(statdata, "saliency", &salmap);
327
328 // We transfer our lowpass5 to the motion channel as unshifted prev:
329 env_pyr_swap(&lowpass5, &motion_chan.unshifted_prev);
330 env_pyr_make_empty(&lowpass5);
331
332 env_img_make_empty(&bwimg);
333 /*
334 env_visual_cortex_rescale_ranges(&salmap, &intens, &color, &ori, &flicker, &motion);
335 */
336}
337
338// ##############################################################################################################
339void Saliency::process(jevois::RawImage const & input, bool do_gist)
340{
341 itsProfiler.start();
342
343 static env_chan_status_func * statfunc = nullptr;
344 static void * statdata = nullptr;
345
346 // We here do what env_mt_visual_cortex_inut used to do in the original envision code, but using lambdas instead of
347 // the c-based jobs:
348 struct env_dims dims = { input.width, input.height };
349 processStart(dims, do_gist);
350 itsProfiler.checkpoint("processStart");
351
352 // Compute Lum, RG, BY, parallelizing over rows:
353 const intg32 lumthresh = (3*255) / 10;
354 struct env_image rgimg; env_img_init(&rgimg, dims);
355 struct env_image byimg; env_img_init(&byimg, dims);
356 struct env_image bwimg; env_img_init(&bwimg, dims);
357
358 int const nthreads = 4;
359 int hh = dims.h / nthreads;
360 std::vector<std::future<void> > rgbyfut;
361 unsigned char const * inpix = input.pixels<unsigned char>();
362 intg32 * rgpix = env_img_pixelsw(&rgimg);
363 intg32 * bypix = env_img_pixelsw(&byimg);
364 intg32 * bwpix = env_img_pixelsw(&bwimg);
365 for (int i = 0; i < nthreads-1; ++i)
366 rgbyfut.push_back(jevois::async([&](int ii) {
367 int offset = dims.w * hh * ii;
368 convertYUYVtoRGBYL(dims.w, hh, inpix + offset*2, rgpix + offset, bypix + offset, bwpix + offset,
369 lumthresh, imath.nbits);
370 }, i));
371
372 // Do the last bit in the current thread:
373 int offset = dims.w * hh * (nthreads - 1);
374 convertYUYVtoRGBYL(dims.w, dims.h - hh * (nthreads-1), inpix + offset*2, rgpix + offset, bypix + offset,
375 bwpix + offset, lumthresh, imath.nbits);
376
377 const intg32 total_weight = env_total_weight(&envp);
378 ENV_ASSERT(total_weight > 0);
379
380 // We can get the color channels started right away. Here we split rg and by into two threads then combine later in a
381 // manner similar to what env_chan_color_rgby() does:
382 const env_size_t firstlevel = envp.cs_lev_min;
383 const env_size_t depth = env_max_pyr_depth(&envp);
384 std::future<void> rgfut, byfut;
385 struct env_image byOut = env_img_initializer;
386
387 // Wait for rgbylum computation to be complete:
388 for (auto & f : rgbyfut) f.get();
389 rgbyfut.clear();
390 itsProfiler.checkpoint("rgby");
391
392 // Notify anyone that was waiting to free the raw input that we are done with it:
393 itsInputDone = true; itsRawImageCond.notify_all();
394
395 // Launch RG and BY in threads:
396 if (envp.chan_c_weight > 0)
397 {
398 rgfut = jevois::async([&]() {
399 struct env_pyr rgpyr;
400 env_pyr_init(&rgpyr, depth);
401 env_pyr_build_lowpass_5(&rgimg, firstlevel, &imath, &rgpyr);
402 env_chan_intensity("red/green", &envp, &imath, rgimg.dims, &rgpyr, 0, statfunc, statdata, &color);
403 env_pyr_make_empty(&rgpyr);
404 });
405
406 byfut = jevois::async([&]() {
407 struct env_pyr bypyr;
408 env_pyr_init(&bypyr, depth);
409 env_pyr_build_lowpass_5(&byimg, firstlevel, &imath, &bypyr);
410 env_chan_intensity("blue/yellow", &envp, &imath, byimg.dims, &bypyr, 0, statfunc, statdata, &byOut);
411 env_pyr_make_empty(&bypyr);
412 });
413 }
414
415 // Compute a luminance pyramid:
416 struct env_pyr lowpass5; env_pyr_init(&lowpass5, env_max_pyr_depth(&envp));
417 env_pyr_build_lowpass_5(&bwimg, envp.cs_lev_min, &imath, &lowpass5);
418
419 itsProfiler.checkpoint("lowpass pyr");
420
421 // Now parallelize the other channels:
422 std::future<void> motfut;
423 if (envp.chan_m_weight > 0)
424 motfut = jevois::async([&]() {
425 env_mt_motion_channel_input(&motion_chan, "motion", bwimg.dims, &lowpass5, statfunc, statdata, &motion);
426 combine_output(&motion, envp.chan_m_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
427 });
428
429 std::future<void> orifut;
430 if (envp.chan_o_weight > 0)
431 orifut = jevois::async([&]() {
432 env_mt_chan_orientation("orientation", &bwimg, statfunc, statdata, &ori);
433 combine_output(&ori, envp.chan_o_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
434 });
435
436 std::future<void> flickfut;
437 if (envp.chan_f_weight > 0)
438 flickfut = jevois::async([&]() {
439 if (envp.multiscale_flicker)
440 env_chan_msflicker("flicker", &envp, &imath, bwimg.dims, &prev_lowpass5, &lowpass5,
441 statfunc, statdata, &flicker);
442 else
443 env_chan_flicker("flicker", &envp, &imath, &prev_input, &bwimg, statfunc, statdata, &flicker);
444
445 combine_output(&flicker, envp.chan_f_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
446
447 if (envp.multiscale_flicker) env_pyr_copy_src_dst(&lowpass5, &prev_lowpass5);
448 else env_pyr_make_empty(&prev_lowpass5);
449 });
450
451 // Intensity is the fastest one and we here just run it in the current thread:
452 if (envp.chan_i_weight > 0)
453 {
454 env_chan_intensity("intensity", &envp, &imath, bwimg.dims, &lowpass5, 1, statfunc, statdata, &intens);
455 combine_output(&intens, envp.chan_i_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
456 }
457 itsProfiler.checkpoint("intens");
458
459 // Wait for all channels to finish up:
460 if (rgfut.valid()) rgfut.get();
461 itsProfiler.checkpoint("red-green");
462
463 if (byfut.valid())
464 {
465 byfut.get();
466
467 // Finish up the color channel by combining rg and by:
468 const intg32 * const byptr = env_img_pixels(&byOut);
469 intg32 * const dptr = env_img_pixelsw(&color);
470 const env_size_t sz = env_img_size(&color);
471 for (env_size_t i = 0; i < sz; ++i) dptr[i] = (dptr[i] + byptr[i]) >> 1;
472
474
475 if (statfunc) (*statfunc)(statdata, "color", &color);
476 env_img_make_empty(&byOut);
477
478 // Add color channel to the saliency map:
479 combine_output(&color, envp.chan_c_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
480 }
481 itsProfiler.checkpoint("blue-yellow");
482
483 if (orifut.valid()) orifut.get();
484 itsProfiler.checkpoint("orientation");
485
486 if (flickfut.valid()) flickfut.get();
487 itsProfiler.checkpoint("flicker");
488
489 if (motfut.valid()) motfut.get();
490 itsProfiler.checkpoint("motion");
491
492 // Cleanup and get ready for next frame:
493 if (!envp.multiscale_flicker) env_img_swap(&prev_input, &bwimg); else env_img_make_empty(&prev_input);
494
495 if (statfunc) (*statfunc)(statdata, "saliency", &salmap);
496
497 // We transfer our lowpass5 to the motion channel as unshifted prev:
498 env_pyr_swap(&lowpass5, &motion_chan.unshifted_prev);
499 env_pyr_make_empty(&lowpass5);
500
501 env_img_make_empty(&bwimg);
502 env_img_make_empty(&rgimg);
503 env_img_make_empty(&byimg);
504 /*
505 env_visual_cortex_rescale_ranges(&salmap, &intens, &color, &ori, &flicker, &motion);
506 */
507 itsProfiler.stop();
508}
509
510// ##############################################################################################################
511void Saliency::env_mt_chan_orientation(const char* tagName, const struct env_image* img,
512 env_chan_status_func* status_func, void* status_userdata,
513 struct env_image* result)
514{
515 env_img_make_empty(result);
516
517 if (envp.num_orientations == 0) return;
518
519 struct env_pyr hipass9;
520 env_pyr_init(&hipass9, env_max_pyr_depth(&envp));
521 env_pyr_build_hipass_9(img, envp.cs_lev_min, &imath, &hipass9);
522
523 char buf[17] = {
524 's', 't', 'e', 'e', 'r', 'a', 'b', 'l', 'e', // 0--8
525 '(', '_', '_', // 9--11
526 '/', '_', '_', ')', '\0' // 12--16
527 };
528
529 ENV_ASSERT(envp.num_orientations <= 99);
530
531 buf[13] = '0' + (envp.num_orientations / 10);
532 buf[14] = '0' + (envp.num_orientations % 10);
533
534 std::vector<std::future<void> > fut;
535 std::mutex mtx;
536 for (env_size_t i = 0; i < envp.num_orientations; ++i)
537 fut.push_back(jevois::async([&](env_size_t ii) {
538 struct env_image chanOut; env_img_init_empty(&chanOut);
539
540 char tagname[17]; memcpy(tagname, buf, 17);
541 tagname[10] = '0' + ((ii+1) / 10);
542 tagname[11] = '0' + ((ii+1) % 10);
543
544 // theta = (180.0 * i) / envp.num_orientations + 90.0, where ENV_TRIG_TABSIZ is equivalent to 360.0 or 2*pi
545 const env_size_t thetaidx = (ENV_TRIG_TABSIZ * ii) / (2 * envp.num_orientations) + (ENV_TRIG_TABSIZ / 4);
546 ENV_ASSERT(thetaidx < ENV_TRIG_TABSIZ);
547
548 env_chan_steerable(tagname, &envp, &imath, img->dims, &hipass9, thetaidx,
549 status_func, status_userdata, &chanOut);
550
551 // Access result image one thread at a time:
552 std::lock_guard<std::mutex> _(mtx);
553 if (!env_img_initialized(result))
554 {
555 env_img_resize_dims(result, chanOut.dims);
556 env_c_image_div_scalar(env_img_pixels(&chanOut), env_img_size(&chanOut), (intg32)envp.num_orientations,
557 env_img_pixelsw(result));
558 }
559 else
560 {
561 ENV_ASSERT(env_dims_equal(chanOut.dims, result->dims));
562 env_c_image_div_scalar_accum(env_img_pixels(&chanOut), env_img_size(&chanOut),
563 (intg32)envp.num_orientations, env_img_pixelsw(result));
564 }
565 env_img_make_empty(&chanOut);
566 }, i));
567
568 // Wait for all the jobs to complete:
569 for (std::future<void> & f : fut) f.get();
570
571 env_pyr_make_empty(&hipass9);
572
573 if (env_img_initialized(result))
575
576 if (status_func) (*status_func)(status_userdata, tagName, result);
577}
578
579// ##############################################################################################################
580void Saliency::env_mt_motion_channel_input(struct env_motion_channel* chan, const char* tagName,
581 const struct env_dims inputdims, struct env_pyr* unshiftedCur,
582 env_chan_status_func* status_func, void* status_userdata,
583 struct env_image* result)
584{
585 env_img_make_empty(result);
586
587 if (chan->num_directions != envp.num_motion_directions)
588 {
590 env_motion_channel_init(chan, &envp);
591 }
592
593 if (chan->num_directions == 0) return;
594
595 char buf[17] =
596 {
597 'r', 'e', 'i', 'c', 'h', 'a', 'r', 'd', 't', // 0--8
598 '(', '_', '_', // 9--11
599 '/', '_', '_', ')', '\0' // 12--16
600 };
601
602 ENV_ASSERT(chan->num_directions <= 99);
603
604 buf[13] = '0' + (chan->num_directions / 10);
605 buf[14] = '0' + (chan->num_directions % 10);
606
607 // compute Reichardt motion detection into several directions
608 std::vector<std::future<void> > fut;
609 std::mutex mtx;
610 for (env_size_t dir = 0; dir < chan->num_directions; ++dir)
611 fut.push_back(jevois::async([&](env_size_t d) {
612 struct env_image chanOut; env_img_init_empty(&chanOut);
613
614 char tagname[17]; memcpy(tagname, buf, 17);
615 tagname[10] = '0' + ((d+1) / 10);
616 tagname[11] = '0' + ((d+1) % 10);
617
618 const env_size_t firstlevel = envp.cs_lev_min;
619 const env_size_t depth = env_max_pyr_depth(&envp);
620
621 // theta = (360.0 * i) / chan->num_directions;
622 const env_size_t thetaidx = (d * ENV_TRIG_TABSIZ) / chan->num_directions;
623 ENV_ASSERT(thetaidx < ENV_TRIG_TABSIZ);
624
625 // create an empty pyramid:
626 struct env_pyr shiftedCur; env_pyr_init(&shiftedCur, depth);
627
628 // fill the empty pyramid with the shifted version
629 for (env_size_t i = firstlevel; i < depth; ++i)
630 {
631 env_img_resize_dims(env_pyr_imgw(&shiftedCur, i), env_pyr_img(unshiftedCur, i)->dims);
632 env_shift_image(env_pyr_img(unshiftedCur, i), imath.costab[thetaidx], -imath.sintab[thetaidx],
633 ENV_TRIG_NBITS, env_pyr_imgw(&shiftedCur, i));
634 }
635
636 env_chan_direction(tagname, &envp, &imath, inputdims, &chan->unshifted_prev, unshiftedCur,
637 &chan->shifted_prev[d], &shiftedCur, status_func, status_userdata, &chanOut);
638
639 env_pyr_swap(&chan->shifted_prev[d], &shiftedCur);
640 env_pyr_make_empty(&shiftedCur);
641
642 // Access result image one thread at a time:
643 std::lock_guard<std::mutex> _(mtx);
644 if (env_img_initialized(&chanOut))
645 {
646 if (!env_img_initialized(result))
647 {
648 env_img_resize_dims(result, chanOut.dims);
649 env_c_image_div_scalar(env_img_pixels(&chanOut), env_img_size(&chanOut), (intg32)chan->num_directions,
650 env_img_pixelsw(result));
651 }
652 else
653 {
654 ENV_ASSERT(env_dims_equal(chanOut.dims, result->dims));
655 env_c_image_div_scalar_accum(env_img_pixels(&chanOut), env_img_size(&chanOut),
656 (intg32)chan->num_directions, env_img_pixelsw(result));
657 }
658 }
659 env_img_make_empty(&chanOut);
660 }, dir));
661
662 // Wait for all the jobs to complete:
663 for (std::future<void> & f : fut) f.get();
664
665 if (env_img_initialized(result))
667
668 if (status_func) (*status_func)(status_userdata, tagName, result);
669}
670
671// ##############################################################################################################
672void Saliency::getSaliencyMax(int & x, int & y, intg32 & value)
673{
674 if (env_img_initialized(&salmap) == false) LFATAL("Saliency map has not yet been computed");
675
676 intg32 *sm = salmap.pixels; int const smw = int(salmap.dims.w), smh = int(salmap.dims.h);
677
678 value = *sm;
679
680 for (int j = 0; j < smh; ++j)
681 for (int i = 0; i < smw; ++i)
682 if (*sm > value) { value = *sm++; x = i; y = j; } else ++sm;
683}
684
685// ##############################################################################################################
686void Saliency::inhibitionOfReturn(int const x, int const y, float const sigma)
687{
688 if (env_img_initialized(&salmap) == false) LFATAL("Saliency map has not yet been computed");
689
690 intg32 *sm = salmap.pixels; int const smw = int(salmap.dims.w), smh = int(salmap.dims.h);
691 float const sigsq = sigma * sigma;
692
693 for (int j = 0; j < smh; ++j)
694 for (int i = 0; i < smw; ++i)
695 {
696 float const distsq = (i-x)*(i-x) + (j-y)*(j-y);
697 if (distsq < sigsq)
698 *sm++ = 0; // hard kill in a disk up to sigma
699 else
700 {
701 float val = *sm;
702 val *= 1.0F - expf( -0.5F * (distsq - sigsq ) / sigsq); // smooth decay
703 *sm++ = static_cast<intg32>(val + 0.4999F);
704 }
705 }
706}
707
708// ####################################################################################################
709void drawMap(jevois::RawImage & img, env_image const * fmap, unsigned int xoff, unsigned int yoff,
710 unsigned int scale)
711{
712 unsigned int const imgw = img.width;
713 unsigned short * d = img.pixelsw<unsigned short>() + xoff + yoff * imgw;
714 intg32 *s = fmap->pixels;
715 const env_size_t w = fmap->dims.w, h = fmap->dims.h;
716 const env_size_t ws = w * scale;
717
718 for (env_size_t jj = 0; jj < h; ++jj)
719 {
720 unsigned short const * dd = d;
721
722 // Copy and scale the first row one pixel at a time:
723 for (env_size_t ii = 0; ii < w; ++ii)
724 {
725 intg32 v = *s++;
726 unsigned short const val = 0x8000 | v;
727 for (env_size_t k = 0; k < scale; ++k) *d++ = val;
728 }
729 d += imgw - ws;
730
731 // Then just use memcpy to duplicate it to achieve the scaling factor vertically:
732 for (env_size_t k = 1; k < scale; ++k) { memcpy(d, dd, ws * 2); d += imgw; }
733 }
734
735 // Draw a rectangle to delinate the map:
736 jevois::rawimage::drawRect(img, xoff, yoff, scale * w, scale * h, 0x80a0);
737}
738
739// ####################################################################################################
740void drawMap(jevois::RawImage & img, env_image const * fmap, unsigned int xoff, unsigned int yoff,
741 unsigned int scale, unsigned int bitshift)
742{
743 unsigned int const imgw = img.width;
744 unsigned short * d = img.pixelsw<unsigned short>() + xoff + yoff * imgw;
745 intg32 *s = fmap->pixels;
746 const env_size_t w = fmap->dims.w, h = fmap->dims.h;
747 const env_size_t ws = w * scale;
748
749 for (env_size_t jj = 0; jj < h; ++jj)
750 {
751 unsigned short const * dd = d;
752
753 // Copy and scale the first row one pixel at a time:
754 for (env_size_t ii = 0; ii < w; ++ii)
755 {
756 intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
757 unsigned short const val = 0x8000 | v;
758 for (env_size_t k = 0; k < scale; ++k) *d++ = val;
759 }
760 d += imgw - ws;
761
762 // Then just use memcpy to duplicate it to achieve the scaling factor vertically:
763 for (env_size_t k = 1; k < scale; ++k) { memcpy(d, dd, ws * 2); d += imgw; }
764 }
765
766 // Draw a rectangle to delinate the map:
767 jevois::rawimage::drawRect(img, xoff, yoff, scale * w, scale * h, 0x80a0);
768}
769
770// ####################################################################################################
771void drawGist(jevois::RawImage & img, unsigned char const * gist, size_t gistsize, unsigned int xoff,
772 unsigned int yoff, unsigned int width, unsigned int scale)
773{
774 unsigned int const height = gistsize / width;
775 unsigned int const imgw = img.width;
776 unsigned short * d = img.pixelsw<unsigned short>() + xoff + yoff * imgw;
777 unsigned char const * const dataend = gist + gistsize;
778 unsigned int const ws = width * scale;
779
780 for (env_size_t jj = 0; jj < height; ++jj)
781 {
782 unsigned short const * dd = d;
783
784 // Copy and scale the first row one pixel at a time:
785 for (env_size_t ii = 0; ii < width; ++ii)
786 {
787 intg32 v = gist >= dataend ? 0 : *gist++;
788 unsigned short const val = 0x8000 | v;
789 for (env_size_t k = 0; k < scale; ++k) *d++ = val;
790 }
791 d += imgw - ws;
792
793 // Then just use memcpy to duplicate it to achieve the scaling factor vertically:
794 for (env_size_t k = 1; k < scale; ++k) { memcpy(d, dd, ws * 2); d += imgw; }
795 }
796
797 // Draw a rectangle to delinate the map:
798 //jevois::rawimage::drawRect(img, xoff, yoff, scale * width, scale * height, 0x80a0);
799}
800
#define JEVOIS_UNUSED_PARAM(x)
int h
#define JEVOIS_WAIT_GET_FUTURE(f)
#define WEIGHT_SCALEBITS
Definition Saliency.C:37
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
Definition Saliency.C:771
#define SALUPDATE(envval, param)
Definition Saliency.C:148
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Definition Saliency.C:709
struct env_image ori
Definition Saliency.H:128
unsigned char * gist
Gist vector has 1152 entries, 72 feature maps * 16 values/map.
Definition Saliency.H:151
void inhibitionOfReturn(int const x, int const y, float const sigma)
Inhibit the saliency map around a point, sigma is in pixels at the sacle of the map.
Definition Saliency.C:686
struct env_image motion
Definition Saliency.H:130
size_t const gist_size
Definition Saliency.H:152
struct env_image salmap
The saliency map.
Definition Saliency.H:118
void process(jevois::RawImage const &input, bool do_gist)
Process a raw YUYV image. Results are stored in the Saliency class.
Definition Saliency.C:339
struct env_image color
Definition Saliency.H:127
void getSaliencyMax(int &x, int &y, intg32 &value)
Get location and value of max point in the saliency map.
Definition Saliency.C:672
Saliency(std::string const &instance)
Constructor.
Definition Saliency.C:81
struct env_image flicker
Definition Saliency.H:129
struct env_image intens
Definition Saliency.H:126
void waitUntilDoneWithInput() const
Wait until process() is done using the input image.
Definition Saliency.C:217
virtual ~Saliency()
Destructor.
Definition Saliency.C:106
void checkpoint(char const *description)
T const * pixels() const
unsigned int width
unsigned int height
void env_c_luminance_from_byte(const struct env_rgb_pixel *const src, const env_size_t sz, const env_size_t nbits, intg32 *const dst)
get the luminance with nbits of precision of the input image
void env_c_image_div_scalar_accum(const intg32 *const a, const env_size_t sz, intg32 val, intg32 *const dst)
result += a / val
void env_c_image_div_scalar(const intg32 *const a, const env_size_t sz, intg32 val, intg32 *const dst)
result = a / val
void env_chan_intensity(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_dims inputdims, const struct env_pyr *lowpass5, const int normalizeOutput, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
An intensity channel.
void env_chan_direction(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_dims inputdims, const struct env_pyr *unshiftedPrev, const struct env_pyr *unshiftedCur, const struct env_pyr *shiftedPrev, const struct env_pyr *shiftedCur, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A motion sensitive channel with direction selectivity.
void env_chan_flicker(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_image *prev, const struct env_image *cur, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A temporal flicker channel.
void env_chan_color(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_rgb_pixel *const colimg, const struct env_dims dims, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A double opponent color channel that combines r/g, b/y subchannels.
void env_chan_steerable(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_dims inputdims, const struct env_pyr *hipass9, const env_size_t thetaidx, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
An orientation filtering channel.
void env_chan_msflicker(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_dims inputDims, const struct env_pyr *prev_lowpass5, const struct env_pyr *cur_lowpass5, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A true multi-scale temporal flicker channel.
void() env_chan_status_func(void *userdata, const char *tagName, const struct env_image *img)
Definition env_channel.h:55
void env_img_swap(struct env_image *img1, struct env_image *img2)
Definition env_image.c:48
void env_img_resize_dims(struct env_image *img, const struct env_dims d)
Definition env_image.c:64
#define env_img_initializer
Definition env_image.h:49
void env_img_make_empty(struct env_image *img)
Definition env_image.c:56
void env_img_init(struct env_image *img, const struct env_dims d)
Definition env_image.c:41
#define INTMAXNORMMAX
void env_grid_average(const struct env_image *src, unsigned char *dest, unsigned int bitshift, env_size_t nx, env_size_t ny)
Compute average values in each tile of a grid.
void env_pyr_build_hipass_9(const struct env_image *image, env_size_t firstlevel, const struct env_math *imath, struct env_pyr *result)
#define INTMAXNORMMIN
void env_pyr_build_lowpass_5(const struct env_image *image, env_size_t firstlevel, const struct env_math *imath, struct env_pyr *result)
Wrapper for _cpu or _cuda version.
void env_max_normalize_inplace(struct env_image *src, intg32 min, intg32 max, enum env_maxnorm_type typ, const intg32 rangeThresh)
void env_shift_image(const struct env_image *srcImg, const env_ssize_t dxnumer, const env_ssize_t dynumer, const env_size_t denombits, struct env_image *result)
#define ENV_ASSERT(expr)
Definition env_log.h:63
#define ENV_TRIG_TABSIZ
Definition env_math.h:44
#define ENV_TRIG_NBITS
Definition env_math.h:45
void env_init_integer_math(struct env_math *imath, const struct env_params *envp)
Definition env_math.c:44
void env_motion_channel_init(struct env_motion_channel *chan, const struct env_params *envp)
void env_motion_channel_destroy(struct env_motion_channel *chan)
intg32 env_total_weight(const struct env_params *envp)
Definition env_params.c:85
void env_params_set_defaults(struct env_params *envp)
Definition env_params.c:43
void env_params_validate(const struct env_params *envp)
Definition env_params.c:97
env_size_t env_max_pyr_depth(const struct env_params *envp)
Definition env_params.c:79
void env_pyr_init(struct env_pyr *pyr, const env_size_t n)
Construct with a given number of empty images.
Definition env_pyr.c:41
void env_pyr_swap(struct env_pyr *pyr1, struct env_pyr *pyr2)
Swap contents with another env_pyr.
Definition env_pyr.c:60
void env_pyr_copy_src_dst(const struct env_pyr *src, struct env_pyr *dst)
Definition env_pyr.c:68
void env_pyr_make_empty(struct env_pyr *dst)
Definition env_pyr.c:51
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition env_types.h:52
unsigned long env_size_t
Definition env_types.h:71
#define LFATAL(msg)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
void convertYUYVtoRGBYL(unsigned int w, unsigned int h, unsigned char const *src, int *dstrg, int *dstby, int *dstlum, int thresh, int inputbits)
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
result
Definition demo.py:74
_
Definition demo.py:85
unsigned char * gist
Definition Saliency.H:155
env_params * envp
Definition Saliency.H:155
A simple struct to hold a pair of width/height dimensions.
Definition env_types.h:81
env_size_t w
The width.
Definition env_types.h:82
env_size_t h
The height.
Definition env_types.h:83
Basic image class.
Definition env_image.h:44
struct env_dims dims
Definition env_image.h:45
intg32 * pixels
Definition env_image.h:46
const intg16 * costab
Definition env_math.h:52
const intg16 * sintab
Definition env_math.h:51
env_size_t nbits
Definition env_math.h:49
A composite channel containing a set of direction channels.
struct env_pyr unshifted_prev
struct env_pyr * shifted_prev
byte chan_f_weight
Definition env_params.h:62
byte multiscale_flicker
Definition env_params.h:52
byte chan_o_weight
Definition env_params.h:61
env_size_t cs_del_max
Definition env_params.h:57
byte chan_c_weight
Definition env_params.h:60
void * user_data_preproc
Definition env_params.h:80
int(* submapPreProc)(const char *tagName, env_size_t clev, env_size_t slev, struct env_image *submap, const struct env_image *center, const struct env_image *surround, void *user_data)
Definition env_params.h:65
env_size_t cs_del_min
Definition env_params.h:56
intg32 range_thresh
Definition env_params.h:47
byte chan_m_weight
Definition env_params.h:63
env_size_t num_motion_directions
Definition env_params.h:49
enum env_maxnorm_type maxnorm_type
Definition env_params.h:46
env_size_t cs_lev_min
Definition env_params.h:54
env_size_t cs_lev_max
Definition env_params.h:55
byte chan_i_weight
Definition env_params.h:59
byte flicker_thresh
Definition env_params.h:51
byte motion_thresh
Definition env_params.h:50
env_size_t num_orientations
number of Gabor subchannels
Definition env_params.h:53
This class implements a set of images, often used as a dyadic pyramid.
Definition env_pyr.h:46
RGB pixel class.
Definition env_types.h:75