JeVois  1.21
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Loading...
Searching...
No Matches
Gadget.C
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
18#include <jevois/Core/Gadget.H>
19#include <jevois/Debug/Log.H>
21#include <jevois/Util/Utils.H>
22#include <jevois/Util/Async.H>
24#include <jevois/Core/Engine.H>
25
26#include <sys/types.h>
27#include <sys/stat.h>
28#include <fcntl.h>
29#include <unistd.h>
30#include <sys/time.h> // for gettimeofday()
31
32namespace
33{
34 inline void debugCtrlReq(struct usb_ctrlrequest const & ctrl)
35 {
36 (void)ctrl; // avoid compiler warning about unused param if LDEBUG is turned off
37
38 LDEBUG(std::showbase << std::hex << "bRequestType " << ctrl.bRequestType << " bRequest " << ctrl.bRequest
39 << " wValue " << ctrl.wValue << " wIndex " << ctrl.wIndex << " wLength " << ctrl.wLength);
40 }
41
42 inline void debugStreamingCtrl(std::string const & msg, struct uvc_streaming_control const & ctrl)
43 {
44 (void)ctrl; // avoid compiler warning about unused param if LDEBUG is turned off
45 (void)msg; // avoid compiler warning about unused param if LDEBUG is turned off
46 LDEBUG(msg << ": " << std::showbase << std::hex << "bmHint=" << ctrl.bmHint << ", bFormatIndex=" <<
47 ctrl.bFormatIndex << ", bFrameIndex=" << ctrl.bFrameIndex << ", dwFrameInterval=" << ctrl.dwFrameInterval <<
48 ", wKeyFrameRate=" << ctrl.wKeyFrameRate << ", wPFrameRate=" << ctrl.wPFrameRate <<
49 ", wCompQuality=" << ctrl.wCompQuality << ", wCompWindowSize=" << ctrl.wCompWindowSize <<
50 ", wDelay=" << ctrl.wDelay << ", dwMaxVideoFrameSize=" << ctrl.dwMaxVideoFrameSize <<
51 ", dwMaxPayloadTransferSize=" << ctrl.dwMaxPayloadTransferSize << ", dwClockFrequency=" <<
52 ctrl.dwClockFrequency << ", bmFramingInfo=" << ctrl.bmFramingInfo << ", bPreferedVersion=" <<
53 ctrl.bPreferedVersion << ", bMinVersion=" << ctrl.bMinVersion << ", bMaxVersion=" << ctrl.bMaxVersion);
54 }
55
56 unsigned int uvcToV4Lcontrol(unsigned int entity, unsigned int cs)
57 {
58 switch (entity)
59 {
60 case 1: // Our camera unit
61 // -- #define UVC_CT_SCANNING_MODE_CONTROL 0x01
62 // OK #define UVC_CT_AE_MODE_CONTROL 0x02
63 // OK #define UVC_CT_AE_PRIORITY_CONTROL 0x03
64 // OK #define UVC_CT_EXPOSURE_TIME_ABSOLUTE_CONTROL 0x04
65 // -- #define UVC_CT_EXPOSURE_TIME_RELATIVE_CONTROL 0x05
66 // -- #define UVC_CT_FOCUS_ABSOLUTE_CONTROL 0x06
67 // -- #define UVC_CT_FOCUS_RELATIVE_CONTROL 0x07
68 // -- #define UVC_CT_FOCUS_AUTO_CONTROL 0x08
69 // -- #define UVC_CT_IRIS_ABSOLUTE_CONTROL 0x09
70 // -- #define UVC_CT_IRIS_RELATIVE_CONTROL 0x0a
71 // -- #define UVC_CT_ZOOM_ABSOLUTE_CONTROL 0x0b
72 // -- #define UVC_CT_ZOOM_RELATIVE_CONTROL 0x0c
73 // -- #define UVC_CT_PANTILT_ABSOLUTE_CONTROL 0x0d
74 // -- #define UVC_CT_PANTILT_RELATIVE_CONTROL 0x0e
75 // -- #define UVC_CT_ROLL_ABSOLUTE_CONTROL 0x0f
76 // -- #define UVC_CT_ROLL_RELATIVE_CONTROL 0x10
77 // -- #define UVC_CT_PRIVACY_CONTROL 0x11
78 // note: UVC 1.5 has a few more...
79 //
80 // Windows 10 insists on doing a GET_DEF on this 10-byte control even though we never said we support it:
81 // -- #define UVC_CT_REGION_OF_INTEREST_CONTROL 0x14
82 // This is now handled by sending a default blank reply to all unsupported controls.
83 switch (cs)
84 {
85 case UVC_CT_AE_MODE_CONTROL: return V4L2_CID_EXPOSURE_AUTO;
86 case UVC_CT_AE_PRIORITY_CONTROL: return V4L2_CID_EXPOSURE_AUTO_PRIORITY;
87 case UVC_CT_EXPOSURE_TIME_ABSOLUTE_CONTROL: return V4L2_CID_EXPOSURE_ABSOLUTE;
88 }
89 break;
90
91 case 2: // Our processing unit
92 // From uvcvideo.h and the UVC specs, here are the available processing unit controls:
93 // A.9.5. Processing Unit Control Selectors
94
95 // Note one trick here: UVC_PU_WHITE_BALANCE_COMPONENT_CONTROL contains both the V4L2_CID_RED_BALANCE and
96 // V4L2_CID_BLUE_BALANCE; we handle that in the ioctl processing section, here we just return V4L2_CID_RED_BALANCE
97
98 // OK #define UVC_PU_BACKLIGHT_COMPENSATION_CONTROL 0x01
99 // OK #define UVC_PU_BRIGHTNESS_CONTROL 0x02
100 // OK #define UVC_PU_CONTRAST_CONTROL 0x03
101 // OK #define UVC_PU_GAIN_CONTROL 0x04
102 // OK #define UVC_PU_POWER_LINE_FREQUENCY_CONTROL 0x05
103 // OK #define UVC_PU_HUE_CONTROL 0x06
104 // OK #define UVC_PU_SATURATION_CONTROL 0x07
105 // OK #define UVC_PU_SHARPNESS_CONTROL 0x08
106 // -- #define UVC_PU_GAMMA_CONTROL 0x09
107 // -- #define UVC_PU_WHITE_BALANCE_TEMPERATURE_CONTROL 0x0a
108 // -- #define UVC_PU_WHITE_BALANCE_TEMPERATURE_AUTO_CONTROL 0x0b
109 // OK #define UVC_PU_WHITE_BALANCE_COMPONENT_CONTROL 0x0c
110 // OK #define UVC_PU_WHITE_BALANCE_COMPONENT_AUTO_CONTROL 0x0d
111 // TODO #define UVC_PU_DIGITAL_MULTIPLIER_CONTROL 0x0e
112 // TODO #define UVC_PU_DIGITAL_MULTIPLIER_LIMIT_CONTROL 0x0f
113 // TODO #define UVC_PU_HUE_AUTO_CONTROL 0x10
114 // TODO #define UVC_PU_ANALOG_VIDEO_STANDARD_CONTROL 0x11
115 // TODO #define UVC_PU_ANALOG_LOCK_STATUS_CONTROL 0x12
116 switch (cs)
117 {
118 case UVC_PU_BACKLIGHT_COMPENSATION_CONTROL: return V4L2_CID_BACKLIGHT_COMPENSATION;
119 case UVC_PU_BRIGHTNESS_CONTROL: return V4L2_CID_BRIGHTNESS;
120 case UVC_PU_CONTRAST_CONTROL: return V4L2_CID_CONTRAST;
121 case UVC_PU_GAIN_CONTROL: return V4L2_CID_GAIN;
122 case UVC_PU_POWER_LINE_FREQUENCY_CONTROL: return V4L2_CID_POWER_LINE_FREQUENCY;
123 case UVC_PU_HUE_CONTROL: return V4L2_CID_HUE;
124 case UVC_PU_SATURATION_CONTROL: return V4L2_CID_SATURATION;
125 case UVC_PU_SHARPNESS_CONTROL: return V4L2_CID_SHARPNESS;
126 case UVC_PU_WHITE_BALANCE_COMPONENT_CONTROL: return V4L2_CID_RED_BALANCE;
127 case UVC_PU_WHITE_BALANCE_COMPONENT_AUTO_CONTROL: return V4L2_CID_AUTO_WHITE_BALANCE;
128 //case UVC_PU_GAMMA_CONTROL: return V4L2_CID_GAMMA;
129 }
130 break;
131 }
132
133 LFATAL("Request to access unsupported control " << cs << " on entity " << entity);
134 }
135
136} // namespace
137
138// ##############################################################################################################
139jevois::Gadget::Gadget(std::string const & devname, jevois::VideoInput * camera, jevois::Engine * engine,
140 size_t const nbufs, bool multicam) :
141 itsFd(-1), itsMulticam(multicam), itsNbufs(nbufs), itsBuffers(nullptr), itsCamera(camera), itsEngine(engine),
142 itsRunning(false), itsFormat(), itsFps(0.0F), itsStreaming(false), itsErrorCode(0), itsControl(0), itsEntity(0)
143{
144 JEVOIS_TRACE(1);
145
146 if (itsCamera == nullptr) LFATAL("Gadget requires a valid camera to work");
147
148 jevois::VideoMapping const & m = itsEngine->getDefaultVideoMapping();
149 fillStreamingControl(&itsProbe, m);
150 fillStreamingControl(&itsCommit, m);
151
152 // Get our run() thread going and wait until it is cranking, it will flip itsRunning to true as it starts:
153 itsRunFuture = jevois::async_little(std::bind(&jevois::Gadget::run, this));
154 while (itsRunning.load() == false) std::this_thread::sleep_for(std::chrono::milliseconds(5));
155
156 // Open the device:
157 itsFd = open(devname.c_str(), O_RDWR | O_NONBLOCK);
158 if (itsFd == -1) PLFATAL("Gadget device open failed for " << devname);
159
160 // Get ready to handle UVC events:
161 struct v4l2_event_subscription sub = { };
162
163 sub.type = UVC_EVENT_SETUP;
164 XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
165
166 sub.type = UVC_EVENT_DATA;
167 XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
168
169 sub.type = UVC_EVENT_STREAMON;
170 XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
171
172 sub.type = UVC_EVENT_STREAMOFF;
173 XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
174
175 // Find out what the driver can do:
176 struct v4l2_capability cap = { };
177 XIOCTL(itsFd, VIDIOC_QUERYCAP, &cap);
178
179 LINFO('[' << itsFd << "] UVC gadget " << devname << " card " << cap.card << " bus " << cap.bus_info);
180 if ((cap.capabilities & V4L2_CAP_VIDEO_OUTPUT) == 0) LFATAL(devname << " is not a video output device");
181 if ((cap.capabilities & V4L2_CAP_STREAMING) == 0) LFATAL(devname << " does not support streaming");
182}
183
184// ##############################################################################################################
186{
187 JEVOIS_TRACE(1);
188
189 streamOff();
190
191 // Tell run() thread to finish up:
192 itsRunning.store(false);
193
194 // Will block until the run() thread completes:
195 if (itsRunFuture.valid()) try { itsRunFuture.get(); } catch (...) { jevois::warnAndIgnoreException(); }
196
197 if (close(itsFd) == -1) PLERROR("Error closing UVC gadget -- IGNORED");
198}
199
200// ##############################################################################################################
202{
203 JEVOIS_TRACE(2);
204
205 JEVOIS_TIMED_LOCK(itsMtx);
206
207 // Set the format:
208 memset(&itsFormat, 0, sizeof(struct v4l2_format));
209
210 itsFormat.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
211 itsFormat.fmt.pix.width = m.ow;
212 itsFormat.fmt.pix.height = m.oh;
213 itsFormat.fmt.pix.pixelformat = m.ofmt;
214 itsFormat.fmt.pix.field = V4L2_FIELD_NONE;
215 itsFormat.fmt.pix.sizeimage = m.osize();
216 itsFps = m.ofps;
217
218 // Do not do anything if ofmt is NONE:
219 if (m.ofmt == 0) { LINFO("USB Gadget set video format to NONE"); return; }
220
221 // First try to set our own format, will throw if phony:
222 XIOCTL(itsFd, VIDIOC_S_FMT, &itsFormat);
223
224 // Note that the format does not include fps, this is done with VIDIOC_S_PARM:
225 try
226 {
227 // The gadget driver may not support this ioctl...
228 struct v4l2_streamparm sparm = { };
229 sparm.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
230 sparm.parm.output.outputmode = 2; // V4L2_MODE_VIDEO not defined in our headers? its value is 2.
231 sparm.parm.output.timeperframe = jevois::VideoMapping::fpsToV4l2(m.ofps);
232 XIOCTL_QUIET(itsFd, VIDIOC_S_PARM, &sparm);
233 } catch (...) { }
234
235 LINFO("USB Gadget set video format to " << itsFormat.fmt.pix.width << 'x' << itsFormat.fmt.pix.height << ' ' <<
236 jevois::fccstr(itsFormat.fmt.pix.pixelformat));
237}
238
239// ##############################################################################################################
240void jevois::Gadget::run()
241{
242 JEVOIS_TRACE(1);
243
244 fd_set wfds; // For UVC video streaming
245 fd_set efds; // For UVC events
246 struct timeval tv;
247
248 // Switch to running state:
249 itsRunning.store(true);
250
251 // We may have to wait until the device is opened:
252 while (itsFd == -1) std::this_thread::sleep_for(std::chrono::milliseconds(1));
253
254 // Wait for event from the gadget kernel driver and process them:
255 while (itsRunning.load())
256 {
257 // Wait until we either receive an event or we are ready to send the next buffer over:
258 FD_ZERO(&wfds); FD_ZERO(&efds); FD_SET(itsFd, &wfds); FD_SET(itsFd, &efds);
259 tv.tv_sec = 0; tv.tv_usec = 10000;
260
261 int ret = select(itsFd + 1, nullptr, &wfds, &efds, &tv);
262
263 if (ret == -1) { PLERROR("Select error"); if (errno == EINTR) continue; else break; }
264 else if (ret > 0) // We have some events, handle them right away:
265 {
266 // Note: we may have more than one event, so here we try processEvents() several times to be sure:
267 if (FD_ISSET(itsFd, &efds))
268 {
269 // First event, we will report error if any:
270 try { processEvents(); } catch (...) { jevois::warnAndIgnoreException(); }
271
272 // Let's try to dequeue one more, in most cases it should throw:
273 while (true) try { processEvents(); } catch (...) { break; }
274 }
275
276 if (FD_ISSET(itsFd, &wfds)) try { processVideo(); } catch (...) { jevois::warnAndIgnoreException(); }
277 }
278
279 // We timed out
280
281 // Sometimes we miss events in the main loop, likely because more events come while we are unlocked in the USB UDC
282 // driver and processing here. So let's try to dequeue one more, in most cases it should throw:
283 while (true) try { processEvents(); } catch (...) { break; }
284
285 // While the driver is not busy in select(), queue at most one buffer that is ready to send off:
286 try
287 {
288 JEVOIS_TIMED_LOCK(itsMtx);
289 if (itsDoneImgs.size())
290 {
291 LDEBUG("Queuing image " << itsDoneImgs.front() << " for sending over USB");
292
293 // We need to prepare a legit v4l2_buffer, including bytesused:
294 struct v4l2_buffer buf = { };
295
296 buf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
297 buf.memory = V4L2_MEMORY_MMAP;
298 buf.index = itsDoneImgs.front();
299 buf.length = itsBuffers->get(buf.index)->length();
300
301 if (itsFormat.fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
302 buf.bytesused = itsBuffers->get(buf.index)->bytesUsed();
303 else
304 buf.bytesused = buf.length;
305
306 buf.field = V4L2_FIELD_NONE;
307 buf.flags = 0;
308 gettimeofday(&buf.timestamp, nullptr);
309
310 // Queue it up so it can be sent to the host:
311 itsBuffers->qbuf(buf);
312
313 // This one is done:
314 itsDoneImgs.pop_front();
315 }
316 } catch (...) { jevois::warnAndIgnoreException(); std::this_thread::sleep_for(std::chrono::milliseconds(10)); }
317 }
318
319 // Switch out of running state in case we did interrupt the loop here by a break statement:
320 itsRunning.store(false);
321}
322
323// ##############################################################################################################
324void jevois::Gadget::processEvents()
325{
326 JEVOIS_TRACE(3);
327
328 // Get the event from the driver:
329 struct v4l2_event v4l2ev = { };
330 XIOCTL_QUIET(itsFd, VIDIOC_DQEVENT, &v4l2ev);
331 struct uvc_event * uvcev = reinterpret_cast<struct uvc_event *>(&v4l2ev.u.data);
332
333 // Prepare our response, if any will be sent:
334 struct uvc_request_data resp = { };
335 resp.length = -EL2HLT;
336
337 // Act according to the event type:
338 try
339 {
340 switch (v4l2ev.type)
341 {
342 case UVC_EVENT_CONNECT: return;
343 case UVC_EVENT_DISCONNECT: LDEBUG("EVENT DISCONNECT"); itsEngine->streamOff(); return;
344 case UVC_EVENT_SETUP: LDEBUG("EVENT SETUP"); processEventSetup(uvcev->req, resp); return;
345 case UVC_EVENT_DATA: LDEBUG("EVENT DATA"); processEventData(uvcev->data); return;
346 case UVC_EVENT_STREAMON: LDEBUG("EVENT STREAMON"); itsEngine->streamOn(); return;
347 case UVC_EVENT_STREAMOFF: LDEBUG("EVENT STREAMOFF"); itsEngine->streamOff(); return;
348 }
349 } catch (...) { }
350}
351
352// ##############################################################################################################
353void jevois::Gadget::processVideo()
354{
355 JEVOIS_TRACE(3);
356
358 JEVOIS_TIMED_LOCK(itsMtx);
359
360 // If we are not streaming anymore, abort:
361 if (itsStreaming.load() == false) LFATAL("Aborted while not streaming");
362
363 // Dequeue a buffer from the gadget driver, this is an image that has been sent to the host and hence the buffer is
364 // now available to be filled up with image data and later queued again to the gadget driver:
365 struct v4l2_buffer buf;
366 itsBuffers->dqbuf(buf);
367
368 // Create a RawImage from that buffer:
369 img.width = itsFormat.fmt.pix.width;
370 img.height = itsFormat.fmt.pix.height;
371 img.fmt = itsFormat.fmt.pix.pixelformat;
372 img.fps = itsFps;
373 img.buf = itsBuffers->get(buf.index);
374 img.bufindex = buf.index;
375
376 // Push the RawImage to outside consumers:
377 itsImageQueue.push_back(img);
378 LDEBUG("Empty image " << img.bufindex << " ready for filling in by application code");
379}
380
381// ##############################################################################################################
382void jevois::Gadget::processEventSetup(struct usb_ctrlrequest const & ctrl, struct uvc_request_data & resp)
383{
384 JEVOIS_TRACE(3);
385
386 itsControl = 0; itsEntity = 0;
387
388 debugCtrlReq(ctrl);
389
390 switch (ctrl.bRequestType & USB_TYPE_MASK)
391 {
392 case USB_TYPE_STANDARD: processEventStandard(ctrl, resp); break;
393 case USB_TYPE_CLASS: processEventClass(ctrl, resp); break;
394 default: LERROR("Unsupported setup event type " << std::showbase << std::hex <<
395 (ctrl.bRequestType & USB_TYPE_MASK) << " -- IGNORED");
396 }
397 if (ctrl.bRequestType != 0x21) XIOCTL(itsFd, UVCIOC_SEND_RESPONSE, &resp);
398}
399
400// ##############################################################################################################
401void jevois::Gadget::processEventStandard(struct usb_ctrlrequest const & ctrl, struct uvc_request_data &)
402{
403 JEVOIS_TRACE(3);
404
405 LDEBUG("UVC standard setup event ignored:");
406 debugCtrlReq(ctrl);
407}
408
409// ##############################################################################################################
410void jevois::Gadget::processEventClass(struct usb_ctrlrequest const & ctrl, struct uvc_request_data & resp)
411{
412 JEVOIS_TRACE(3);
413
414 if ((ctrl.bRequestType & USB_RECIP_MASK) != USB_RECIP_INTERFACE) return;
415
416 switch (ctrl.wIndex & 0xff)
417 {
418 case UVC_INTF_CONTROL:
419 processEventControl(ctrl.bRequest, ctrl.wValue >> 8, ctrl.wIndex >> 8, ctrl.wLength, resp);
420 break;
421
423 processEventStreaming(ctrl.bRequest, ctrl.wValue >> 8, resp);
424 break;
425
426 default:
427 LERROR("Unsupported setup event class " << std::showbase << std::hex << (ctrl.wIndex & 0xff) << " -- IGNORED");
428 }
429}
430
431// ##############################################################################################################
432void jevois::Gadget::processEventControl(uint8_t req, uint8_t cs, uint8_t entity_id, uint8_t len,
433 struct uvc_request_data & resp)
434{
435 JEVOIS_TRACE(3);
436
437 // Local function we run on successful processing of an event: we just reset our internal error code
438#define success() { itsErrorCode = 0; }
439
440 // Local function we run on failed processing of an event: stall the request, set our internal error code
441#define failure(code) { resp.length = -EL2HLT; itsErrorCode = code; }
442
443 // Shortcurt to successfully send a 1-byte response:
444#define byteresponse(val) { resp.data[0] = val; resp.length = 1; itsErrorCode = 0; }
445
446 // Shortcurt to successfully send a 2-byte response:
447#define wordresponse(val) { resp.data[0] = val & 0xff; resp.data[1] = (val >> 8) & 0xff; \
448 resp.length = 2; itsErrorCode = 0; }
449
450 // Shortcurt to successfully send a 4-byte response:
451#define intresponse(val) { resp.data[0] = val & 0xff; resp.data[1] = (val >> 8) & 0xff; \
452 resp.data[2] = (val >> 16) & 0xff; resp.data[3] = (val >> 24) & 0xff; \
453 resp.length = 4; itsErrorCode = 0; }
454
455 // Shortcurt to successfully send a blank N-byte response:
456#define arrayblankresponse(len) { memset(resp.data, 0, len); resp.length = len; itsErrorCode = 0; }
457
458 // If anything throws here, we will return failure:
459 try
460 {
461 // First handle any request that is directed to entity 0:
462 if (entity_id == 0)
463 {
464 switch (cs)
465 {
466 case UVC_VC_REQUEST_ERROR_CODE_CONTROL: byteresponse(itsErrorCode); return; // Send error code last prepared
467 default: failure(0x06); return;
468 }
469 }
470
471 // Process according to the entity that this event is directed to and the control that is requested:
472 if (req == UVC_SET_CUR)
473 {
474 // We need to wait for the data phase, so for now just remember the control and return success:
475 itsEntity = entity_id; itsControl = cs;
476 resp.data[0] = 0x0; resp.length = len; success();
477 LDEBUG("SET_CUR ent " << itsEntity <<" ctrl "<< itsControl <<" len "<< len);
478 }
479 else if (req == UVC_GET_INFO)
480 {
481 // FIXME: controls could also be disabled, autoupdate, or asynchronous:
482 byteresponse(UVC_CONTROL_CAP_GET | UVC_CONTROL_CAP_SET);
483 }
484 else if (req == UVC_GET_CUR)
485 {
486 // Fetch the current value from the camera. Note: both Windows and Android insist on querying some controls, like
487 // IRIS and GAMMA, which we did not declare as supported in the kernel driver. We need to handle those requests
488 // and to send some phony data:
489 struct v4l2_control ctrl = { };
490 try { ctrl.id = uvcToV4Lcontrol(entity_id, cs); itsCamera->getControl(ctrl); } catch (...) { ctrl.id = 0; }
491
492 // We need a special handling of white balance here:
493 if (ctrl.id == V4L2_CID_RED_BALANCE)
494 {
495 unsigned int redval = (ctrl.value & 0xffff) << 16; // red is at offset 2 in PU_WHITE_BALANCE_COMPONENT_CONTROL
496
497 // Also get the blue balance value:
498 ctrl.id = V4L2_CID_BLUE_BALANCE;
499 itsCamera->getControl(ctrl);
500
501 // Combine both red and blue values:
502 ctrl.value = (ctrl.value & 0xffff) | redval;
503 }
504 // We also need to remap auto exposure values:
505 else if (ctrl.id == V4L2_CID_EXPOSURE_AUTO)
506 {
507 if (ctrl.value == V4L2_EXPOSURE_MANUAL) ctrl.value = 0x01; // manual mode, set UVC bit D0
508 else if (ctrl.value == V4L2_EXPOSURE_AUTO) ctrl.value = 0x02; // auto mode, set UVC bit D1
509 else ctrl.value = 0x03;
510 // Note, there are 2 more bits under CT_AE_MODE_CONTROL
511 }
512 // Handle the unknown controls:
513 else if (ctrl.id == 0) ctrl.value = 0;
514
515 switch (len)
516 {
517 case 1: byteresponse(ctrl.value); break;
518 case 2: wordresponse(ctrl.value); break;
519 case 4: intresponse(ctrl.value); break;
520 default: LERROR("Unsupported control with length " << len << " -- SENDING BLANK RESPONSE");
521 arrayblankresponse(len); break;
522 }
523 }
524 else
525 {
526 // It's a GET_DEF/RES/MIN/MAX let's first get the data from the camera: Note: both Windows and Android insist on
527 // querying some controls, like IRIS and GAMMA, which we did not declare as supported in the kernel driver. We
528 // need to handle those requests and to send some phony data:
529 struct v4l2_queryctrl qc = { };
530 try { qc.id = uvcToV4Lcontrol(entity_id, cs); itsCamera->queryControl(qc); } catch (...) { qc.id = 0; }
531
532 // We need a special handling of white balance here:
533 if (qc.id == V4L2_CID_RED_BALANCE)
534 {
535 // Also get the blue balance values:
536 struct v4l2_queryctrl qc2 = { };
537 qc2.id = V4L2_CID_BLUE_BALANCE;
538 itsCamera->queryControl(qc2);
539
540 // Combine red and blue values into qc:
541 qc.default_value = (qc.default_value << 16) | qc2.default_value;
542 qc.step = (qc.step << 16) | qc2.step;
543 qc.minimum = (qc.minimum << 16) | qc2.minimum;
544 qc.maximum = (qc.maximum << 16) | qc2.maximum;
545 }
546 // We also need to remap auto exposure values:
547 else if (qc.id == V4L2_CID_EXPOSURE_AUTO)
548 {
549 // Tricky: in the 'step' field, we are supposed to provide a bitmap of the modes that are supported, see UVC
550 // specs. D0=manual, D1=auto, D2=shutter priority, D3=aperture priority. Min and max are ignored for this
551 // control, default is handled.
552 qc.minimum = 0; qc.step = 3; qc.maximum = 3; qc.default_value = 1;
553 }
554 // Also handle the unknown controls here:
555 else if (qc.id == 0)
556 { qc.minimum = 0; qc.step = 1; qc.maximum = 1; qc.default_value = 0; }
557
558 int val = 0;
559 switch (req)
560 {
561 case UVC_GET_DEF: val = qc.default_value; break;
562 case UVC_GET_RES: val = qc.step; break;
563 case UVC_GET_MIN: val = qc.minimum; break;
564 case UVC_GET_MAX: val = qc.maximum; break;
565 default: failure(0x07); return;
566 }
567
568 switch (len)
569 {
570 case 1: byteresponse(val); break;
571 case 2: wordresponse(val); break;
572 case 4: intresponse(val); break;
573 default: LERROR("Unsupported control with length " << len << " -- SENDING BLANK RESPONSE");
574 arrayblankresponse(len); break;
575 }
576 }
577 }
578 catch (...)
579 {
580 LERROR("FAILED entity " << entity_id << " cs " << cs << " len " << len);
581 failure(0x06);
582 }
583}
584
585// ##############################################################################################################
586void jevois::Gadget::fillStreamingControl(struct uvc_streaming_control * ctrl, jevois::VideoMapping const & m)
587{
588 JEVOIS_TRACE(3);
589
590 memset(ctrl, 0, sizeof(struct uvc_streaming_control));
591
592 ctrl->bFormatIndex = m.uvcformat;
593 ctrl->bFrameIndex = m.uvcframe;
594 ctrl->dwFrameInterval = jevois::VideoMapping::fpsToUvc(m.ofps);
595 ctrl->dwMaxVideoFrameSize = m.osize();
596 ctrl->dwMaxPayloadTransferSize = itsMulticam ? 1024 : 3072;
597 ctrl->bmFramingInfo = 3;
598 ctrl->bPreferedVersion = 1;
599 ctrl->bMaxVersion = 1;
600}
601
602// ##############################################################################################################
603void jevois::Gadget::processEventStreaming(uint8_t req, uint8_t cs, struct uvc_request_data & resp)
604{
605 JEVOIS_TRACE(3);
606
607 int const datalen = 26; // uvc 1.0 as reported by our kernel driver
608 if (cs != UVC_VS_PROBE_CONTROL && cs != UVC_VS_COMMIT_CONTROL) return;
609
610 struct uvc_streaming_control * ctrl = reinterpret_cast<struct uvc_streaming_control *>(&resp.data);
611 struct uvc_streaming_control * target = (cs == UVC_VS_PROBE_CONTROL) ? &itsProbe : &itsCommit;
612 resp.length = datalen;
613
614 switch (req)
615 {
616 case UVC_SET_CUR: itsControl = cs; resp.length = datalen; break; // will finish up in data stage
617
618 case UVC_GET_CUR:
619 case UVC_GET_MIN: // we have nothing to negotiate
620 case UVC_GET_MAX: // we have nothing to negotiate
621 memcpy(ctrl, target, datalen);
622 break;
623
624 case UVC_GET_DEF:
625 {
626 // If requested format index, frame index, or interval is bogus (including zero), initialize to our default mapping,
627 // otherwise pass down the selected mapping:
628 size_t idx = itsEngine->getDefaultVideoMappingIdx();
629 try { idx = itsEngine->getVideoMappingIdx(ctrl->bFormatIndex, ctrl->bFrameIndex, ctrl->dwFrameInterval); }
630 catch (...) { }
631 fillStreamingControl(target, itsEngine->getVideoMapping(idx));
632 memcpy(ctrl, target, datalen);
633 }
634 break;
635
636 case UVC_GET_RES: memset(ctrl, 0, datalen); break;
637
638 case UVC_GET_LEN: resp.data[0] = 0x00; resp.data[1] = datalen; resp.length = 2; break;
639
640 case UVC_GET_INFO: resp.data[0] = 0x03; resp.length = 1; break;
641 }
642}
643
644// ##############################################################################################################
645void jevois::Gadget::processEventData(struct uvc_request_data & data)
646{
647 JEVOIS_TRACE(3);
648
649 struct uvc_streaming_control * target;
650
651 // If entity is 1 or 2, this is to set a control:
652 if (itsEntity == 2 || itsEntity == 1) { processEventControlData(data); return; }
653
654 switch (itsControl)
655 {
656 case UVC_VS_PROBE_CONTROL: target = &itsProbe; break;
657 case UVC_VS_COMMIT_CONTROL: target = &itsCommit; break;
658 default: processEventControlData(data); return;
659 }
660
661 // Find the selected format and frame info and fill-in the control data:
662 struct uvc_streaming_control * ctrl = reinterpret_cast<struct uvc_streaming_control *>(&data.data);
663
664 size_t idx = itsEngine->getVideoMappingIdx(ctrl->bFormatIndex, ctrl->bFrameIndex, ctrl->dwFrameInterval);
665
666 fillStreamingControl(target, itsEngine->getVideoMapping(idx));
667
668 LDEBUG("Host requested " << ctrl->bFormatIndex << '/' << ctrl->bFrameIndex << '/' << ctrl->dwFrameInterval <<
669 ", " << ((itsControl == UVC_VS_COMMIT_CONTROL) ? "setting " : "returning ") <<
670 itsEngine->getVideoMapping(idx).str());
671
672 // Set the format if we are doing a commit control:
673 if (itsControl == UVC_VS_COMMIT_CONTROL) itsEngine->setFormat(idx);
674}
675
676// ##############################################################################################################
677void jevois::Gadget::processEventControlData(struct uvc_request_data & data)
678{
679 JEVOIS_TRACE(3);
680
681 struct v4l2_control ctrl;
682
683 // Get the control ID for V4L or throw if unsupported:
684 ctrl.id = uvcToV4Lcontrol(itsEntity, itsControl);
685
686 // Copy the data we received into the control's value:
687 switch (data.length)
688 {
689 case 1: ctrl.value = static_cast<int>(data.data[0]); break;
690 case 2: ctrl.value = static_cast<int>(__s16(data.data[0] | (static_cast<short>(data.data[1]) << 8))); break;
691 case 4: ctrl.value = data.data[0] | (data.data[1] << 8) | (data.data[2] << 16) | (data.data[3] << 24); break;
692 default: LFATAL("Ooops data len is " << data.length);
693 }
694
695 // Tell the camera to set the control. We do not have enough time here to do it as otherwise our USB transaction will
696 // time out while we transfer a bunch of bytes to the camera over the 400kHz serial control link, so we just push it
697 // to a queue and our run() thread will do the work. First, handle special cases:
698 switch (ctrl.id)
699 {
700 case V4L2_CID_RED_BALANCE:
701 {
702 // We need to set both the red and the blue:
703 int blue = ctrl.value & 0xffff;
704 ctrl.value >>= 16; itsCamera->setControl(ctrl);
705 ctrl.id = V4L2_CID_BLUE_BALANCE; ctrl.value = blue; itsCamera->setControl(ctrl);
706 }
707 break;
708
709 case V4L2_CID_EXPOSURE_AUTO:
710 if (ctrl.value & 0x01) ctrl.value = V4L2_EXPOSURE_MANUAL; // UVC bit D0 set for manual mode
711 else if (ctrl.value & 0x02) ctrl.value = V4L2_EXPOSURE_AUTO; // auto mode
712 // Note, there are 2 more bits under CT_AE_MODE_CONTROL
713 itsCamera->setControl(ctrl);
714 break;
715
716 default: itsCamera->setControl(ctrl);
717 }
718}
719
720// ##############################################################################################################
722{
723 JEVOIS_TRACE(2);
724
725 LDEBUG("Turning on UVC stream");
726
727 JEVOIS_TIMED_LOCK(itsMtx);
728
729 if (itsStreaming.load() || itsBuffers) { LERROR("Stream is already on -- IGNORED"); return; }
730 if (itsFormat.fmt.pix.pixelformat == 0) { LINFO("Gadget output format is NONE"); return; }
731
732 // If number of buffers is zero, adjust it depending on frame size:
733 unsigned int nbuf = itsNbufs;
734 if (nbuf == 0)
735 {
736 unsigned int framesize = jevois::v4l2ImageSize(itsFormat.fmt.pix.pixelformat, itsFormat.fmt.pix.width,
737 itsFormat.fmt.pix.height);
738
739 // Aim for about 4 mbyte when using small images, and no more than 4 buffers in any case:
740 nbuf = (4U * 1024U * 1024U) / framesize;
741 if (nbuf > 4) nbuf = 4;
742 }
743
744 // Force number of buffers to a sane value:
745 if (nbuf < 3) nbuf = 3; else if (nbuf > 16) nbuf = 16;
746
747 // Allocate our buffers for the currently selected resolution, format, etc:
748 itsBuffers = new jevois::VideoBuffers("gadget", itsFd, V4L2_BUF_TYPE_VIDEO_OUTPUT, nbuf);
749 LINFO(itsBuffers->size() << " buffers of " << itsBuffers->get(0)->length() << " bytes allocated");
750
751 // Fill itsImageQueue with blank frames that can be given off to application code:
752 for (size_t i = 0; i < nbuf; ++i)
753 {
755 img.width = itsFormat.fmt.pix.width;
756 img.height = itsFormat.fmt.pix.height;
757 img.fmt = itsFormat.fmt.pix.pixelformat;
758 img.fps = itsFps;
759 img.buf = itsBuffers->get(i);
760 img.bufindex = i;
761
762 // Push the RawImage to outside consumers:
763 itsImageQueue.push_back(img);
764 LDEBUG("Empty image " << img.bufindex << " ready for filling in by application code");
765 }
766
767 // Start streaming over the USB link:
768 int type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
769 XIOCTL(itsFd, VIDIOC_STREAMON, &type);
770 LDEBUG("Device stream on");
771
772 itsStreaming.store(true);
773 LDEBUG("Stream is on");
774}
775
776// ##############################################################################################################
778{
779 JEVOIS_TRACE(2);
780
781 itsStreaming.store(false);
782}
783
784// ##############################################################################################################
786{
787 JEVOIS_TRACE(2);
788
789 // Note: we allow for several streamOff() without complaining, this happens, e.g., when destroying a Gadget that is
790 // not currently streaming.
791
792 LDEBUG("Turning off gadget stream");
793
794 // Abort stream in case it was not already done, which will introduce some sleeping in our run() thread, thereby
795 // helping us acquire our needed double lock:
796 abortStream();
797
798 JEVOIS_TIMED_LOCK(itsMtx);
799
800 // Stop streaming over the USB link:
801 int type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
802 try { XIOCTL_QUIET(itsFd, VIDIOC_STREAMOFF, &type); } catch (...) { }
803
804 // Nuke all our buffers:
805 if (itsBuffers) { delete itsBuffers; itsBuffers = nullptr; }
806 itsImageQueue.clear();
807 itsDoneImgs.clear();
808
809 LDEBUG("Gadget stream is off");
810}
811
812// ##############################################################################################################
814{
815 JEVOIS_TRACE(4);
816 int retry = 2000;
817
818 while (--retry >= 0)
819 {
820 if (itsStreaming.load() == false)
821 { LDEBUG("Not streaming"); throw std::runtime_error("Gadget get() rejected while not streaming"); }
822
823 if (itsMtx.try_lock_for(std::chrono::milliseconds(100)))
824 {
825 if (itsStreaming.load() == false)
826 {
827 LDEBUG("Not streaming");
828 itsMtx.unlock();
829 throw std::runtime_error("Gadget get() rejected while not streaming");
830 }
831
832 if (itsImageQueue.size())
833 {
834 img = itsImageQueue.front();
835 itsImageQueue.pop_front();
836 itsMtx.unlock();
837 LDEBUG("Empty image " << img.bufindex << " handed over to application code for filling");
838 return;
839 }
840
841 // No image in the queue, unlock and wait for one:
842 itsMtx.unlock();
843 LDEBUG("Waiting for blank UVC image...");
844 std::this_thread::sleep_for(std::chrono::milliseconds(5));
845 }
846 else
847 {
848 LDEBUG("Waiting for lock");
849 std::this_thread::sleep_for(std::chrono::milliseconds(5));
850 }
851 }
852 LFATAL("Giving up waiting for blank UVC image");
853}
854
855// ##############################################################################################################
857{
858 JEVOIS_TRACE(4);
859 int retry = 2000;
860
861 while (--retry >= 0)
862 {
863 if (itsStreaming.load() == false)
864 { LDEBUG("Not streaming"); throw std::runtime_error("Gadget send() rejected while not streaming"); }
865
866 if (itsMtx.try_lock_for(std::chrono::milliseconds(100)))
867 {
868 if (itsStreaming.load() == false)
869 {
870 LDEBUG("Not streaming");
871 itsMtx.unlock();
872 throw std::runtime_error("Gadget send() rejected while not streaming");
873 }
874
875 // Check that the format matches, this may not be the case if we changed format while the buffer was out for
876 // processing. IF so, we just drop this image since it cannot be sent to the host anymore:
877 if (img.width != itsFormat.fmt.pix.width ||
878 img.height != itsFormat.fmt.pix.height ||
879 img.fmt != itsFormat.fmt.pix.pixelformat)
880 {
881 LDEBUG("Dropping image to send out as format just changed");
882 itsMtx.unlock();
883 return;
884 }
885
886 // We cannot just qbuf() here as our run() thread is likely in select() and the driver will bomb the qbuf as
887 // resource unavailable. So we just enqueue the buffer index and the run() thread will handle the qbuf later:
888 itsDoneImgs.push_back(img.bufindex);
889 itsMtx.unlock();
890 LDEBUG("Filled image " << img.bufindex << " received from application code");
891 return;
892 }
893 else
894 {
895 LDEBUG("Waiting for lock");
896 std::this_thread::sleep_for(std::chrono::milliseconds(5));
897 }
898 }
899 LFATAL("Giving up waiting for lock");
900}
901
#define arrayblankresponse(len)
#define success()
#define intresponse(val)
#define byteresponse(val)
#define failure(code)
#define wordresponse(val)
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition Engine.H:414
VideoMapping const & getDefaultVideoMapping() const
Allow access to the default video mapping.
Definition Engine.C:1570
void streamOff() override
Stop streaming.
Definition Gadget.C:785
void abortStream() override
Abort streaming.
Definition Gadget.C:777
void get(RawImage &img) override
Get a pre-allocated image so that we can fill the pixel data and later send out over USB using send()
Definition Gadget.C:813
Gadget(std::string const &devname, VideoInput *camera, Engine *engine, size_t const nbufs=0, bool multicam=false)
Construct and open the device.
Definition Gadget.C:139
virtual ~Gadget()
Close the device and free all resources.
Definition Gadget.C:185
void setFormat(jevois::VideoMapping const &m) override
Set the video format and frame rate.
Definition Gadget.C:201
void streamOn() override
Start streaming.
Definition Gadget.C:721
void send(RawImage const &img) override
Send an image out over USB to the host computer.
Definition Gadget.C:856
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition RawImage.H:111
float fps
Programmed frames/s as given by current video mapping, may not be actual.
Definition RawImage.H:148
unsigned int fmt
Pixel format as a V4L2_PIX_FMT_XXX.
Definition RawImage.H:147
size_t bufindex
The index of the data buffer in the kernel driver.
Definition RawImage.H:150
unsigned int width
Image width in pixels.
Definition RawImage.H:145
unsigned int height
Image height in pixels.
Definition RawImage.H:146
std::shared_ptr< VideoBuf > buf
The pixel data buffer.
Definition RawImage.H:149
Collection of buffers for V4L2 video frames (Camera or Gadget) with hooks to the MMAP'd areas.
Base class for video input, which will get derived into Camera and MovieInput.
Definition VideoInput.H:32
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition Log.H:230
#define LDEBUG(msg)
Convenience macro for users to print out console or syslog messages, DEBUG level.
Definition Log.H:173
#define JEVOIS_TIMED_LOCK(mtx)
Helper macro to create a timed_lock_guard object.
Definition Log.H:328
#define PLERROR(msg)
Like LERROR but appends errno and strerror(errno), to be used when some system call fails.
Definition Log.H:219
std::string warnAndIgnoreException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition Log.C:236
#define PLFATAL(msg)
Like LDEBUG but appends errno and strerror(errno), to be used when some system call fails.
Definition Log.H:239
#define JEVOIS_TRACE(level)
Trace object.
Definition Log.H:296
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition Log.H:211
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition Log.H:194
#define XIOCTL_QUIET(dev, req, mem)
Helper macro to execute an ioctl, ignore interruptions, and, if error throw quietly.
Definition Utils.H:216
std::string fccstr(unsigned int fcc)
Convert a V4L2 four-cc code (V4L2_PIX_FMT_...) to a 4-char string.
Definition Utils.C:45
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async_little(Function &&f, Args &&... args)
Async execution using a thread pool.
#define XIOCTL(dev, req, mem)
Helper macro to execute an ioctl, ignore interruptions, and, if error, issue a fatal message and thro...
Definition Utils.H:205
unsigned int v4l2ImageSize(unsigned int fcc, unsigned int width, unsigned int height)
Return the image size in bytes for a given V4L2_PIX_FMT_..., width, height.
Definition Utils.C:168
Simple struct to hold video mapping definitions for the processing Engine.
static struct v4l2_fract fpsToV4l2(float fps)
Convert from fps to V4L2 interval.
unsigned int ow
output width
static unsigned int fpsToUvc(float fps)
Convert from fps to USB/UVC interval.
unsigned int osize() const
Return the size in bytes of an output image.
float ofps
output frame rate in frames/sec
unsigned int oh
output height
unsigned int uvcformat
USB-UVC format number (1-based)
unsigned int uvcframe
USB UVC frame number (1-based)
unsigned int ofmt
output pixel format, or 0 for no output over USB
struct usb_ctrlrequest req
Definition uvc.h:46
struct uvc_request_data data
Definition uvc.h:47
__u8 data[UVC_MAX_REQUEST_SIZE - sizeof(__s32)]
Definition uvc.h:39
__s32 length
Definition uvc.h:38
#define UVC_INTF_CONTROL
Definition uvc.h:53
#define UVCIOC_SEND_RESPONSE
Definition uvc.h:51
#define UVC_EVENT_SETUP
Definition uvc.h:30
#define UVC_EVENT_STREAMOFF
Definition uvc.h:29
#define UVC_EVENT_DATA
Definition uvc.h:31
#define UVC_EVENT_CONNECT
Definition uvc.h:26
#define UVC_EVENT_STREAMON
Definition uvc.h:28
#define UVC_INTF_STREAMING
Definition uvc.h:54
#define UVC_EVENT_DISCONNECT
Definition uvc.h:27