JeVoisBase  1.21
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Loading...
Searching...
No Matches
ObjectMatcher.C
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
19#include <jevois/Debug/Log.H>
21#include <sys/types.h>
22#include <fcntl.h>
23#include <dirent.h>
24#include <sys/stat.h>
25#include <unistd.h>
26
27#include <opencv2/core/utility.hpp>
28#include <opencv2/imgcodecs.hpp>
29#include <opencv2/features2d/features2d.hpp>
30#include <opencv2/calib3d.hpp>
31#include <opencv2/imgproc.hpp>
32#include <opencv2/xfeatures2d/nonfree.hpp>
33
34// ####################################################################################################
37
38// ####################################################################################################
40{
41 // Initialize our feature computer and matcher:
42 itsFeatureDetector = cv::xfeatures2d::SURF::create(hessian::get());
43 size_t const ncores = std::min(4U, std::thread::hardware_concurrency());
44 for (size_t i = 0; i < ncores; ++i)
45 itsMatcher.push_back(cv::Ptr<cv::DescriptorMatcher>(new cv::BFMatcher(cv::NORM_L2)));
46
47 //itsFeatureDetector = cv::ORB::create();
48 //itsMatcher.reset(new cv::BFMatcher(cv::NORM_HAMMING));
49
50 // Load training images and compute keypoints and descriptors:
51 itsTrainData.clear();
52 std::string const dirname = absolutePath(traindir::get());
53
54 LINFO("Training from " << dirname);
55
56 DIR * dir = opendir(dirname.c_str());
57 if (dir == nullptr) PLFATAL("Cound not scan directory " << dirname);
58
59 struct dirent * ent;
60 while ((ent = readdir(dir)) != nullptr)
61 {
62 std::string const fname = ent->d_name;
63 std::string const fullfname = dirname + "/" + fname;
64 if (fname[0] == '.') continue; // skip files that start with a period
65 struct stat st;
66 if (stat(fullfname.c_str(), &st) == -1) continue; // ignore any stat error
67 if ((st.st_mode & S_IFDIR) != 0) continue; // skip sub-directories
68
69 // Create a training data entry:
70 itsTrainData.push_back(TrainData());
71 TrainData & td = itsTrainData.back();
72 td.name = fname;
73 td.image = cv::imread(fullfname, cv::IMREAD_GRAYSCALE);
74
75 // Compute keypoints and descriptors:
76 itsFeatureDetector->detectAndCompute(td.image, cv::Mat(), td.keypoints, td.descriptors);
77
78 LDEBUG(fname << ": " << td.keypoints.size() << " keypoints.");
79 }
80
81 closedir(dir);
82
83 LINFO("Training complete for " << itsTrainData.size() << " images.");
84}
85
86// ####################################################################################################
87double ObjectMatcher::process(cv::Mat const & img, size_t & trainidx, std::vector<cv::Point2f> & corners)
88{
89 std::vector<cv::KeyPoint> keypoints;
90 this->detect(img, keypoints);
91
92 cv::Mat descriptors;
93 this->compute(img, keypoints, descriptors);
94
95 return this->match(keypoints, descriptors, trainidx, corners);
96}
97
98// ####################################################################################################
99double ObjectMatcher::process(cv::Mat const & img, size_t & trainidx)
100{
101 std::vector<cv::KeyPoint> keypoints;
102 this->detect(img, keypoints);
103
104 cv::Mat descriptors;
105 this->compute(img, keypoints, descriptors);
106
107 return this->match(keypoints, descriptors, trainidx);
108}
109
110// ####################################################################################################
111void ObjectMatcher::detect(cv::Mat const & img, std::vector<cv::KeyPoint> & keypoints)
112{
113 itsFeatureDetector->detect(img, keypoints);
114}
115
116// ####################################################################################################
117void ObjectMatcher::compute(cv::Mat const & img, std::vector<cv::KeyPoint> & keypoints, cv::Mat & descriptors)
118{
119 itsFeatureDetector->compute(img, keypoints, descriptors);
120}
121
122// ####################################################################################################
123double ObjectMatcher::match(std::vector<cv::KeyPoint> const & keypoints, cv::Mat const & descriptors,
124 size_t & trainidx, std::vector<cv::Point2f> & corners)
125{
126 if (itsTrainData.empty()) LFATAL("No training data loaded");
127
128 // Parallelize the matching over our cores:
129 size_t const ncores = std::min(4U, std::thread::hardware_concurrency());
130 size_t const ntrain = itsTrainData.size();
131 size_t const r = ntrain % ncores;
132 size_t const q = (ntrain - r) / ncores;
133 size_t percore = q + 1; // up to r cores will process q+1 elements, and up to ncores-r will process q
134 std::vector<std::future<ObjectMatcher::MatchData> > fut;
135
136 size_t startidx = 0;
137 for (size_t i = 0; i < ncores; ++i)
138 {
139 if (i == r) percore = q;
140 if (percore == 0) break;
141
142 size_t const endidx = std::min(startidx + percore, ntrain);
143 fut.push_back(jevois::async([&](size_t cn, size_t mi, size_t ma)
144 { return this->matchcore(cn, keypoints, descriptors, mi, ma, true); },
145 i, startidx, endidx));
146 startidx += percore;
147 }
148
149 // Wait for all jobs to complete, ignore any exception (which may seldom occur due to ill conditioned homography or
150 // such), and pick the best result:
151 double bestdist = 1.0e30;
152 for (auto & f : fut)
153 try
154 {
155 MatchData md = f.get();
156 if (md.avgdist < bestdist) { bestdist = md.avgdist; trainidx = md.trainidx; corners = md.corners; }
157 }
158 catch (...) { }
159
160 return bestdist;
161}
162
163// ####################################################################################################
164double ObjectMatcher::match(std::vector<cv::KeyPoint> const & keypoints, cv::Mat const & descriptors,
165 size_t & trainidx)
166{
167 if (itsTrainData.empty()) LFATAL("No training data loaded");
168
169 // Parallelize the matching over our cores:
170 size_t const ncores = std::min(4U, std::thread::hardware_concurrency());
171 size_t const ntrain = itsTrainData.size();
172 size_t const r = ntrain % ncores;
173 size_t const q = (ntrain - r) / ncores;
174 size_t percore = q + 1; // up to r cores will process q+1 elements, and up to ncores-r will process q
175 std::vector<std::future<ObjectMatcher::MatchData> > fut;
176
177 size_t startidx = 0;
178 for (size_t i = 0; i < ncores; ++i)
179 {
180 if (i == r) percore = q;
181 if (percore == 0) break;
182
183 size_t const endidx = std::min(startidx + percore, ntrain);
184 fut.push_back(jevois::async([&](size_t cn, size_t mi, size_t ma)
185 { return this->matchcore(cn, keypoints, descriptors, mi, ma, false); },
186 i, startidx, endidx));
187 startidx += percore;
188 }
189
190 // Wait for all jobs to complete, ignore any exception (which may seldom occur due to ill conditioned homography or
191 // such), and pick the best result:
192 double bestdist = 1.0e30;
193 for (auto & f : fut)
194 try
195 {
196 MatchData md = f.get();
197 if (md.avgdist < bestdist) { bestdist = md.avgdist; trainidx = md.trainidx; }
198 }
199 catch (...) { }
200
201 return bestdist;
202}
203
204// ####################################################################################################
205ObjectMatcher::MatchData ObjectMatcher::matchcore(size_t corenum, std::vector<cv::KeyPoint> const & keypoints,
206 cv::Mat const & descriptors, size_t minidx, size_t maxidx,
207 bool do_corners)
208{
209 // Compute matches between query and training images:
210 double bestsofar = 1.0e30;
211 MatchData mdata { bestsofar, 0, { } };
212
213 for (size_t idx = minidx; idx < maxidx; ++idx)
214 {
215 TrainData const & td = itsTrainData[idx];
216
217 std::vector<cv::DMatch> matches;
218 itsMatcher[corenum]->match(descriptors, td.descriptors, matches);
219 if (matches.empty()) continue;
220
221 // Sort the matches by distance:
222 std::sort(matches.begin(), matches.end());
223 std::vector<cv::DMatch> good_matches;
224 double const minDist = matches.front().distance, maxDist = matches.back().distance;
225
226 // Keep only the good matches:
227 size_t const ptsPairs = std::min(goodpts::get().max(), matches.size());
228 double const dthresh = distthresh::get();
229 for (size_t i = 0; i < ptsPairs; ++i)
230 if (matches[i].distance <= dthresh) good_matches.push_back(matches[i]); else break;
231
232 LDEBUG(td.name << ": Match distances: " << minDist << " .. " << maxDist << ", " <<
233 matches.size() << " matches, " << good_matches.size() << " good ones.");
234
235 // Abort here if we did not get enough good matches:
236 if (good_matches.size() < goodpts::get().min()) continue;
237
238 // Compute the average match distance:
239 double avgdist = 0.0;
240 for (cv::DMatch const & gm : good_matches) avgdist += gm.distance;
241 avgdist /= good_matches.size();
242 if (avgdist >= bestsofar) continue;
243
244 LDEBUG("Object match: found " << td.name << " distance " << avgdist);
245
246 // Localize the object:
247 std::vector<cv::Point2f> obj, scene;
248 for (size_t i = 0; i < good_matches.size(); ++i)
249 {
250 obj.push_back(td.keypoints[good_matches[i].trainIdx].pt);
251 scene.push_back(keypoints[good_matches[i].queryIdx].pt);
252 }
253
254 if (do_corners)
255 {
256 // Get the corners from the training image:
257 std::vector<cv::Point2f> obj_corners(4);
258 obj_corners[0] = cv::Point(0, 0);
259 obj_corners[1] = cv::Point(td.image.cols, 0);
260 obj_corners[2] = cv::Point(td.image.cols, td.image.rows);
261 obj_corners[3] = cv::Point(0, td.image.rows);
262
263 // Project the corners into the scene image. This may throw if the homography is too messed up:
264 try
265 {
266 // Compute the homography:
267 cv::Mat H = cv::findHomography(obj, scene, cv::RANSAC, 5.0);
268
269 // Check that the homography is not garbage:
270 std::vector<double> sv; cv::SVD::compute(H, sv, cv::SVD::NO_UV);
271 LDEBUG("Homography sv " << sv.front() << " ... " << sv.back());
272
273 if (sv.empty() == false && sv.back() >= 0.001 && sv.front() / sv.back() >= 100.0)
274 {
275 // The homography looks good, use it to map the object corners to the scene image:
276 cv::perspectiveTransform(obj_corners, mdata.corners, H);
277
278 // If all went well, this is our current best object match:
279 mdata.trainidx = idx;
280 mdata.avgdist = avgdist;
281 bestsofar = avgdist;
282 }
283 } catch (...) { }
284 }
285 else
286 {
287 mdata.trainidx = idx;
288 mdata.avgdist = avgdist;
289 bestsofar = avgdist;
290 }
291 }
292
293 return mdata;
294}
295
296// ####################################################################################################
298{
299 if (idx >= itsTrainData.size()) LFATAL("Index too large");
300 return itsTrainData[idx];
301}
302
303// ####################################################################################################
305{ return itsTrainData.size(); }
306
#define H(p, w)
size_t numtrain() const
Get number of training images.
double process(cv::Mat const &img, size_t &trainidx, std::vector< cv::Point2f > &corners)
Process a greyscale image, returns match score, object index, and bounding box corners for best match...
~ObjectMatcher()
Destructor.
TrainData const & traindata(size_t idx) const
Get the training data for a given index.
void postInit() override
Load training images and compute keypoints and descriptors.
void compute(cv::Mat const &img, std::vector< cv::KeyPoint > &keypoints, cv::Mat &descriptors)
Compute descriptors for given keypoints.
double match(std::vector< cv::KeyPoint > const &keypoints, cv::Mat const &descriptors, size_t &trainidx, std::vector< cv::Point2f > &corners)
Match given descriptors against those of our training images, return best match distance.
void detect(cv::Mat const &img, std::vector< cv::KeyPoint > &keypoints)
Detect keypoints.
std::filesystem::path absolutePath(std::filesystem::path const &path="")
#define LFATAL(msg)
#define LDEBUG(msg)
#define PLFATAL(msg)
#define LINFO(msg)
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Training data structure for ObjectMatcher.
std::vector< cv::KeyPoint > keypoints