JeVoisBase  1.21
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Loading...
Searching...
No Matches
SaliencyGist.C
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
18#include <jevois/Core/Module.H>
19
20#include <jevois/Debug/Log.H>
26
27#include <opencv2/core/core.hpp>
28#include <opencv2/imgproc/imgproc.hpp>
29
30#include <future>
31#include <linux/videodev2.h> // for v4l2 pixel types
32
33// icon by Freepik in other at flaticon
34
35//! Simple saliency map and gist computation module
36/*! Computes a saliency map and gist, intended for use by machines.
37
38 See the \jvmod{DemoSaliency} modute for more explanations about saliency and gist algorithms and for a demo output
39 intended for human viewing.
40
41 What is returned depends on the selected output image resolution; it should always be grayscale, and can contain any
42 of:
43
44 - saliency map only
45 - saliency + gist
46 - saliency + feature maps
47 - saliency + feature maps + gist
48
49 See the example video mappings provided with this module for sample resolutions and the associated elements that are
50 returned.
51
52 Serial Messages
53 ---------------
54
55 This module can send standardized serial messages as described in \ref UserSerialStyle. One message is issued on
56 every video frame at the temporally filtered attended location. The \p id field in the messages simply is \b salient
57 for all messages.
58
59 See \ref UserSerialStyle for more on standardized serial messages, and \ref coordhelpers for more info on
60 standardized coordinates.
61
62
63 @author Laurent Itti
64
65 @videomapping GREY 120 25 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps + gist
66 @videomapping GREY 120 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps
67 @videomapping GREY 20 73 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + gist
68 @videomapping GREY 20 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency only
69 @videomapping GREY 72 16 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # gist only
70 @email itti\@usc.edu
71 @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
72 @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
73 @mainurl http://jevois.org
74 @supporturl http://jevois.org/doc
75 @otherurl http://iLab.usc.edu
76 @license GPL v3
77 @distribution Unrestricted
78 @restrictions None
79 \ingroup modules */
81{
82 public:
83 //! Constructor
84 SaliencyGist(std::string const & instance) : jevois::StdModule(instance)
85 {
86 itsSaliency = addSubComponent<Saliency>("saliency");
87 itsKF = addSubComponent<Kalman2D>("kalman");
88 }
89
90 //! Virtual destructor for safe inheritance
91 virtual ~SaliencyGist() { }
92
93 //! Processing function with video output
94 virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
95 {
96 // Wait for next available camera image:
97 jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
98 inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
99
100 // Launch the saliency computation in a thread:
101 auto sal_fut = jevois::async([&](){ itsSaliency->process(inimg, true); });
102
103 // While computing, wait for an image from our gadget driver into which we will put our results:
104 jevois::RawImage outimg = outframe.get();
105
106 // Enforce the correct output image pixel format (will enforce size later):
107 unsigned int const ow = outimg.width, oh = outimg.height;
108 outimg.require("output", ow, oh, V4L2_PIX_FMT_GREY);
109
110 // Once saliency is done using the input image, let camera know we are done with it:
111 itsSaliency->waitUntilDoneWithInput();
112 inframe.done();
113
114 // Wait until saliency computation is complete:
115 sal_fut.get();
116
117 // Find most salient point:
118 int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
119
120 // Compute attended location in original frame coordinates:
121 int const smlev = itsSaliency->smscale::get();
122 int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0; // half a saliency map pixel adjustment
123 unsigned int const dmx = (mx << smlev) + smadj;
124 unsigned int const dmy = (my << smlev) + smadj;
125
126 // Filter these locations:
127 itsKF->set(dmx, dmy, w, h);
128 float kfxraw, kfyraw; itsKF->get(kfxraw, kfyraw, 1.0F); // round to int for serial
129
130 // Send kalman-filtered most-salient-point info to serial port (for arduino, etc):
131 sendSerialStd2D(kfxraw, kfyraw, 0.0F, 0.0F, "salient");
132
133 // Paste results into the output image, first check for valid output dims:
134 unsigned int const mapw = itsSaliency->salmap.dims.w, maph = itsSaliency->salmap.dims.h;
135 unsigned int const gistonlyw = 72; unsigned int const gistsize = itsSaliency->gist_size;
136 unsigned int gisth = (gistsize + ow - 1) / ow; // divide gist_size by ow and round up
137
138 if (false == ( (ow == mapw && oh == maph) ||
139 (ow == mapw * 6 && oh == maph) ||
140 (ow == gistonlyw && oh == gistsize / gistonlyw) ||
141 (ow == mapw && oh == maph + (gistsize + mapw - 1) / mapw) ||
142 (ow == mapw * 6 && oh == maph + (gistsize + mapw*6 - 1) / (mapw*6)) ) )
143 LFATAL("Incorrect output size. With current saliency parameters, valid sizes are: " <<
144 mapw << 'x' << maph << " (saliency map only), " <<
145 mapw * 6 << 'x' << maph << " (saliency map + feature maps), " <<
146 gistonlyw << 'x' << gistsize / gistonlyw << " (gist only), " <<
147 mapw << 'x' << maph + (gistsize + mapw - 1) / mapw << " (saliency map + gist), " <<
148 mapw * 6 << 'x' << maph + (gistsize + mapw*6 - 1) / (mapw*6) << " (saliency + feature maps + gist).");
149
150 // Paste saliency and feature maps if desired:
151 unsigned int offset = 0;
152 if (oh == maph || oh == maph + gisth)
153 {
154 pasteGrayMap(outimg, itsSaliency->salmap, offset, 20);
155 if (ow == mapw * 6)
156 {
157 pasteGrayMap(outimg, itsSaliency->color, offset, 18);
158 pasteGrayMap(outimg, itsSaliency->intens, offset, 18);
159 pasteGrayMap(outimg, itsSaliency->ori, offset, 18);
160 pasteGrayMap(outimg, itsSaliency->flicker, offset, 18);
161 pasteGrayMap(outimg, itsSaliency->motion, offset, 18);
162 }
163 }
164
165 // Paste gist if desired:
166 if (oh == gisth || oh == maph + gisth)
167 {
168 unsigned char * d = outimg.pixelsw<unsigned char>(); if (oh == maph + gisth) d += ow * maph;
169 memcpy(d, itsSaliency->gist, gistsize);
170
171 // Possibly blank out the remainder of the last line of gist:
172 int const rem = ow * gisth - gistsize;
173 if (rem > 0) memset(d + gistsize, 0, rem);
174 }
175
176 // Send the output image with our processing results to the host over USB:
177 outframe.send();
178 }
179
180 // ####################################################################################################
181 //! Paste a map and add its width to the dx offset
182 /*! Beware this is for a gray outimg only. */
183 void pasteGrayMap(jevois::RawImage & outimg, env_image const & fmap, unsigned int & dx, unsigned int bitshift)
184 {
185 env_size_t const fw = fmap.dims.w, fh = fmap.dims.h;
186 unsigned int const ow = outimg.width, oh = outimg.height;
187
188 if (fh > oh) LFATAL("Map would extend past output image bottom");
189 if (fw + dx > ow) LFATAL("Map would extend past output image width");
190
191 unsigned int const stride = ow - fw;
192
193 intg32 * s = fmap.pixels; unsigned char * d = outimg.pixelsw<unsigned char>() + dx;
194
195 for (unsigned int j = 0; j < fh; ++j)
196 {
197 for (unsigned int i = 0; i < fw; ++i)
198 {
199 intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
200 *d++ = (unsigned char)(v);
201 }
202 d += stride;
203 }
204
205 // Add the map width to the dx offset:
206 dx += fw;
207 }
208
209 protected:
210 std::shared_ptr<Saliency> itsSaliency;
211 std::shared_ptr<Kalman2D> itsKF;
212};
213
214// Allow the module to be loaded as a shared object (.so) file:
JEVOIS_REGISTER_MODULE(ArUcoBlob)
int h
Simple saliency map and gist computation module.
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function with video output.
std::shared_ptr< Kalman2D > itsKF
SaliencyGist(std::string const &instance)
Constructor.
void pasteGrayMap(jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int bitshift)
Paste a map and add its width to the dx offset.
virtual ~SaliencyGist()
Virtual destructor for safe inheritance.
std::shared_ptr< Saliency > itsSaliency
unsigned int width
unsigned int height
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
StdModule(std::string const &instance)
void sendSerialStd2D(float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition env_types.h:52
unsigned long env_size_t
Definition env_types.h:71
#define LFATAL(msg)
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
env_size_t w
The width.
Definition env_types.h:82
env_size_t h
The height.
Definition env_types.h:83
Basic image class.
Definition env_image.h:44
struct env_dims dims
Definition env_image.h:45
intg32 * pixels
Definition env_image.h:46