JeVois  1.21
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Loading...
Searching...
No Matches
Network.C
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
18#include <jevois/DNN/Network.H>
19#include <jevois/DNN/Utils.H>
20#include <jevois/Util/Async.H>
21
22// ####################################################################################################
25
26// ####################################################################################################
28{
29 comment::freeze(doit);
30 url::freeze(doit);
31 extraintensors::freeze(doit);
32}
33
34// ####################################################################################################
35void jevois::dnn::Network::onParamChange(network::outreshape const &, std::string const & val)
36{
37 itsReshape.clear();
38 if (val.empty()) return;
39
40 itsReshape = jevois::dnn::parseTensorSpecs(val);
41}
42
43// ####################################################################################################
45{
46 // Do not destroy a network that is loading, and do not throw...
47 size_t count = 0;
48 while (itsLoading.load())
49 {
50 std::this_thread::sleep_for(std::chrono::milliseconds(5));
51 try { if (ready()) break; } catch (...) { }
52 if (count++ == 200) { LINFO("Waiting for network load to complete..."); count = 0; }
53 }
54}
55
56// ####################################################################################################
58{
59 // If we are loaded, we are ready to process:
60 if (itsLoaded.load()) return true;
61
62 // If we are loading, check whether loading is complete or threw, otherwise return false as we keep loading:
63 if (itsLoading.load())
64 {
65 if (itsLoadFut.valid() && itsLoadFut.wait_for(std::chrono::milliseconds(2)) == std::future_status::ready)
66 {
67 try { itsLoadFut.get(); itsLoaded.store(true); itsLoading.store(false); LINFO("Network loaded."); return true; }
68 catch (...) { itsLoading.store(false); jevois::warnAndRethrowException(); }
69 }
70 return false;
71 }
72
73 // Otherwise, trigger an async load:
74 itsLoading.store(true);
75 itsLoadFut = jevois::async(std::bind(&jevois::dnn::Network::load, this));
76 LINFO("Loading network...");
77
78 return false;
79}
80
81// ####################################################################################################
82std::vector<cv::Mat> jevois::dnn::Network::process(std::vector<cv::Mat> const & blobs,
83 std::vector<std::string> & info)
84{
85 if (ready() == false) LFATAL("Network is not ready");
86
87 std::vector<cv::Mat> outs;
88 std::string const c = comment::get();
89
90 // Add any extra input tensors?
91 std::string const extra = extraintensors::get();
92 if (extra.empty() == false)
93 {
94 std::vector<cv::Mat> newblobs = blobs;
95
96 std::vector<std::string> ins = jevois::split(extra, ",\\s*");
97 for (std::string const & in : ins)
98 {
99 vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
100
101 std::vector<std::string> tok = jevois::split(in, ":");
102 if (tok.size() != 3)
103 LFATAL("Malformed extra tensor, need <type>:<shape>:val1 val2 ... valN (separate multiple tensors by comma)");
104
105 // Decode type and convert to vsi, only those types that OpenCV can support:
106 if (tok[0] == "8U") attr.dtype.vx_type = VSI_NN_TYPE_UINT8;
107 else if (tok[0] == "8S") attr.dtype.vx_type = VSI_NN_TYPE_INT8;
108 else if (tok[0] == "16U") attr.dtype.vx_type = VSI_NN_TYPE_UINT16;
109 else if (tok[0] == "16S") attr.dtype.vx_type = VSI_NN_TYPE_INT16;
110 else if (tok[0] == "16F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT16;
111 else if (tok[0] == "32S") attr.dtype.vx_type = VSI_NN_TYPE_INT32;
112 else if (tok[0] == "32F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT32;
113 else if (tok[0] == "64F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT64;
114 else throw std::range_error("Unsupported extra input tensor type [" + tok[0] + "] in " + extra);
115
116 // Decode the dims:
117 std::vector<size_t> dims = jevois::dnn::strshape(tok[1]);
118 attr.dim_num = dims.size();
119 for (size_t i = 0; i < attr.dim_num; ++i) attr.size[attr.dim_num - 1 - i] = dims[i];
120
121 // Allocate the tensor:
122 attr.dtype.qnt_type = VSI_NN_QNT_TYPE_NONE;
123 attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO;
124 cv::Mat b = jevois::dnn::attrmat(attr);
125
126 // Populate the values:
127 std::vector<std::string> vals = jevois::split(tok[2], "\\s+");
128 size_t const nvals = vals.size();
129 if (nvals != b.total())
130 LFATAL("Extra in tensor needs " << b.total() << " values, but " << nvals << " given in [" << in << ']');
131 switch (attr.dtype.vx_type)
132 {
133 case VSI_NN_TYPE_UINT8:
134 {
135 uint8_t * ptr = reinterpret_cast<uint8_t *>(b.data);
136 for (std::string const & v : vals) *ptr++ = std::stoi(v);
137 }
138 break;
139
140 case VSI_NN_TYPE_INT8:
141 {
142 int8_t * ptr = reinterpret_cast<int8_t *>(b.data);
143 for (std::string const & v : vals) *ptr++ = std::stoi(v);
144 }
145 break;
146
147 case VSI_NN_TYPE_UINT16:
148 {
149 uint16_t * ptr = reinterpret_cast<uint16_t *>(b.data);
150 for (std::string const & v : vals) *ptr++ = std::stoi(v);
151 }
152 break;
153
154 case VSI_NN_TYPE_INT16:
155 {
156 int16_t * ptr = reinterpret_cast<int16_t *>(b.data);
157 for (std::string const & v : vals) *ptr++ = std::stoi(v);
158 }
159 break;
160
161 case VSI_NN_TYPE_FLOAT16:
162 {
163 cv::hfloat * ptr = reinterpret_cast<cv::hfloat *>(b.data);
164 for (std::string const & v : vals) *ptr++ = cv::hfloat(std::stof(v));
165 }
166 break;
167
168 case VSI_NN_TYPE_INT32:
169 {
170 int32_t * ptr = reinterpret_cast<int32_t *>(b.data);
171 for (std::string const & v : vals) *ptr++ = std::stoi(v);
172 }
173 break;
174
175 case VSI_NN_TYPE_FLOAT32:
176 {
177 float * ptr = reinterpret_cast<float *>(b.data);
178 for (std::string const & v : vals) *ptr++ = std::stof(v);
179 }
180 break;
181
182 case VSI_NN_TYPE_FLOAT64:
183 {
184 double * ptr = reinterpret_cast<double *>(b.data);
185 for (std::string const & v : vals) *ptr++ = std::stod(v);
186 }
187 break;
188
189 default: LFATAL("internal inconsistency");
190 }
191
192 newblobs.emplace_back(std::move(b));
193 }
194
195 // NOTE: Keep the code below in sync with the default case (no extra inputs). Both branches are duplicated to avoid
196 // having to make a copy of blobs into newblobs in the standard case when we do not have any extra inputs:
197
198 // Show info about input tensors:
199 info.emplace_back("* Input Tensors");
200 for (cv::Mat const & b : newblobs) info.emplace_back("- " + jevois::dnn::shapestr(b));
201
202 // Run processing on the derived class:
203 info.emplace_back("* Network");
204 if (c.empty() == false) info.emplace_back(c);
205
206 outs = std::move(doprocess(newblobs, info));
207 }
208 else
209 {
210 // Show info about input tensors:
211 info.emplace_back("* Input Tensors");
212 for (cv::Mat const & b : blobs) info.emplace_back("- " + jevois::dnn::shapestr(b));
213
214 // Run processing on the derived class:
215 info.emplace_back("* Network");
216 if (c.empty() == false) info.emplace_back(c);
217
218 outs = std::move(doprocess(blobs, info));
219 }
220
221 // Show info about output tensors:
222 info.emplace_back("* Output Tensors");
223 for (size_t i = 0; i < outs.size(); ++i) info.emplace_back("- " + jevois::dnn::shapestr(outs[i]));
224
225 // Possibly reshape the tensors:
226 if (itsReshape.empty() == false)
227 {
228 if (itsReshape.size() != outs.size())
229 LFATAL("Received " << outs.size() << " but outreshape has " << itsReshape.size() << " entries");
230
231 info.emplace_back("* Reshaped Output Tensors");
232 for (size_t i = 0; i < outs.size(); ++i)
233 {
234 outs[i] = outs[i].reshape(1, jevois::dnn::attrdims(itsReshape[i]));
235 info.emplace_back("- " + jevois::dnn::shapestr(outs[i]));
236 }
237 }
238
239 return outs;
240}
virtual void load()=0
Load from disk.
bool ready()
Returns true when network is ready to run (loaded and initialized)
Definition Network.C:57
std::vector< cv::Mat > process(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)
Process input blobs and obtain output blobs.
Definition Network.C:82
void waitBeforeDestroy()
If network is currently loading, wait until that is done before destroying.
Definition Network.C:44
virtual ~Network()
Destructor.
Definition Network.C:23
void onParamChange(network::outreshape const &param, std::string const &val) override
Definition Network.C:35
virtual void freeze(bool doit)
Freeze/unfreeze parameters that users should not change while running.
Definition Network.C:27
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition Log.H:230
void warnAndRethrowException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition Log.C:203
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition Log.H:194
std::vector< vsi_nn_tensor_attr_t > parseTensorSpecs(std::string const &specs)
Parse tensor specification.
Definition Utils.C:387
cv::Mat attrmat(vsi_nn_tensor_attr_t const &attr, void *dataptr=nullptr)
Construct a cv::Mat from attr and possibly data pointer.
Definition Utils.C:471
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition Utils.C:105
std::vector< int > attrdims(vsi_nn_tensor_attr_t const &attr)
Get a tensor dims as a vector of int, useful to construct a matching cv::Mat.
Definition Utils.C:478
std::vector< size_t > strshape(std::string const &str)
Get a vector of size_t from a string containing AxBxC...
Definition Utils.C:277
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Async execution using a thread pool.
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition Utils.C:270