PostProcessor.H

Go to the documentation of this file.

// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//
// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
//
// This file is part of the JeVois Smart Embedded Machine Vision Toolkit.  This program is free software; you can
// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
// Foundation, version 2.  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
// License for more details.  You should have received a copy of the GNU General Public License along with this program;
// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//
// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*! \file */
 
#pragma once
 
#include <jevois/Component/Component.H>
#include <opencv2/core/core.hpp>
#include <jevois/GPU/GUIhelper.H>
#include <jevois/Types/Enum.H>
#include <jevois/Types/PoseSkeleton.H>
 
namespace jevois
{
  class StdModule;
  class RawImage;
 
  namespace dnn
  {
    class PreProcessor;
    
    namespace postprocessor
    {
      // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
      // will use different subsets of all available parameters:
      static jevois::ParameterCategory const ParamCateg("DNN Post-Processing Options");
      
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER(classoffset, int, "Offset added to model output when looking up class name. Useful if "
                               "your model uses a background class but your class file does not (use -1), or if your "
                               "model does not use a background class but your class file has one (use 1). If unsure, "
                               "use 0 and check whether reported class names are off.",
                               0, ParamCateg);
      
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(classes, std::string, "Path to text file with names of object classes, or "
                                             "comma-separated list of class names. If path is relative, that is to "
                                             JEVOIS_SHARE_PATH,
                                             "", ParamCateg);
      
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER(top, unsigned int, "Max number of top-scoring predictions that score above "
                               "threshold to report",
                               5, ParamCateg);
      
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(maxnbox, unsigned int, "Max number of top-scoring boxes to report (for YOLO flavors, "
                               "this is the max for each scale), or 0 for no limit",
                               500, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(namedonly, bool, "When true, only report (on display and to serial ports) results "
                               "for which we have a class name defined in the class file. This can allow one "
                               "to skip some of the results by simply commenting out their name in the class file "
                               "(make sure your class file then contains class-ID numbers for each named class).",
                               false, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER(cthresh, float, "Classification threshold (in percent confidence) above which "
                               "predictions will be reported",
                               20.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(perclassthresh, std::string, "Per-class thresholds (in percent "
                                             "confidence) above which predictions will be reported, one value for "
                                             "each class, separated by spaces and/or commas. This is used to override "
                                             "'cthresh' when different thresholds are needed for different classes, "
                                             "e.g., for YOLO-World. Only supported by YOLOv8 and YOLOv8t "
                                             "post-processors. All values must be strictly >0",
                                             "", ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(dthresh, float, "Detection box threshold (in percent confidence) above which "
                               "predictions will be reported. Not all networks use a separate box threshold, "
                               "many only use one threshold confidence threshold (cthresh parameter). The YOLO "
                               "family is an example that uses both box and classification confidences",
                               15.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorPose
      JEVOIS_DECLARE_PARAMETER(jthresh, float, "Joint confidence threshold for pose detection",
                               50.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
  
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER(softmax, bool, "Apply a softmax to classification outputs",
                               false, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER(scorescale, float, "Scaling factors applied to recognition scores. Mainly "
                               "for debugging if your scores seem too high or too low. If too high, usually "
                               "that means that you should turn on parameter softmax instead.",
                               1.0F, ParamCateg);
      
      //! Enum \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DEFINE_ENUM_CLASS(DetectType, (FasterRCNN) (YOLO) (SSD) (TPUSSD) (RAWYOLO) (YOLOv10) (YOLOv10pp)
                               (YOLOX) (YOLOv8) (YOLOv8t) (YOLOv8seg) (YOLOv8segt) (YOLOjevois) (YOLOjevoist) );
      
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(detecttype, DetectType, "Type of detection output format",
                                             DetectType::YOLO, DetectType_Values, ParamCateg);
      
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(nms, float, "Non-maximum suppression intersection-over-union threshold in percent",
                               45.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(nmsperclass, bool, "When true, apply non-maximum suppression (NMS) separately for each "
                               "object class, rather than globally. This will allow overlapping boxes as long as "
                               "they belong to different classes",
                               false, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(anchors, std::string, "For YOLO-type detection models with raw outputs, "
                               "list of anchors. Should be formatted as: w1, h1, w2, h2, ... ; ww1, hh1, ww2, hh2, "
                               "... ; ... where individual entries for a given YOLO layer are separated by commas, "
                               "and successive YOLO layers (from large to small, e.g., first the anchors for 52x52, "
                               "then for 26x26, then for 13x13) are separated by semicolons. Leave empty "
                               "for other models.",
                               "", ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(masksmooth, bool, "When true, extract smoother instance segmentation masks, "
                               "which may be marginally slower.",
                               true, ParamCateg);
      
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(alpha, unsigned char, "Alpha channel value for drawn results",
                               64, ParamCateg);
 
      //! Enum \relates jevois::dnn::PostProcessorDetectOBB
      JEVOIS_DEFINE_ENUM_CLASS(DetectTypeOBB, (YOLOv8) (YOLOv8t) );
      
      //! Parameter \relates jevois::dnn::PostProcessorDetectOBB
      JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(detecttypeobb, DetectTypeOBB, "Type of oriented bounding box detection "
                                             "output format",
                                             DetectTypeOBB::YOLOv8, DetectTypeOBB_Values, ParamCateg);
      
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER(boxclamp, bool, "Clamp detection boxes to fit within the input image. Good to avoid "
                               "having boxes that extend beyond the display and to be able to see object labels "
                               "even on boxes that start at negative coordinates; but will artificially reduce "
                               "the size of detections that extend beyond the display",
                               true, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorSegment
      JEVOIS_DECLARE_PARAMETER(bgid, unsigned char, "Class ID for the background, will show as fully transparent in "
                               "semantic segmentation overlays",
                               0, ParamCateg);
 
      //! Enum \relates jevois::dnn::PostProcessorSegment
      JEVOIS_DEFINE_ENUM_CLASS(SegType, (ClassesHWC) (ClassesCHW) (ArgMax) );
 
      //! Parameter \relates jevois::dnn::PostProcessorSegment
      JEVOIS_DECLARE_PARAMETER(segtype, SegType, "Type of segmentation network output. ClassesHWC: output is 1xHxWxC "
                               "for C classes and we get one score per class; we will show "
                               "the top scoring class for each pixel (e.g., UNet-MobileNet on TPU). ClassesCHW: "
                               "output is 1xCxHxW and the rest is as for ClassesHWC (e.g., DeepLabV3 OpenCV). ArgMax: "
                               "output is HxW, 1xHxW, or 1xHxWx1 and contains the class ID for each pixel "
                               "(e.g., DeepLabV3 on TPU).",
                               SegType::ClassesHWC, SegType_Values, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorPython
      JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pypost, std::string, "Path below " JEVOIS_SHARE_PATH "/ of the python "
                                             "post-processor file. Name of class defined in the file must match "
                                             "the file name without the trailing '.py'",
                                             "", ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetectYOLO
      JEVOIS_DECLARE_PARAMETER(scalexy, float, "If 0, use old-style YOLO boxes (YOLOv2/v3/v4); otherwise, this is "
                               "the scale_xy factor for new-style YOLO coordinates (YOLOv5/v7; value is usually 2.0 "
                               "but check the yolo layer in the model's .cfg file)",
                               0.0F, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorDetect
      JEVOIS_DECLARE_PARAMETER(sigmoid, bool, "Apply sigmoid to raw YOLO outputs, use when the last conv layers "
                               "just before yolo/detection/region layers have linear activation (most "
                               "YOLOv2/v3/v4 models, but not YOLOv5/v7 which have logistic activation on their "
                               "last conv)",
                               true, ParamCateg);
 
      //! Enum \relates jevois::dnn::PostProcessorPose
      JEVOIS_DEFINE_ENUM_CLASS(PoseType, (YOLOv8HAILO) (YOLOv8) (YOLOv8t) );
      
      //! Parameter \relates jevois::dnn::PostProcessorPose
      JEVOIS_DECLARE_PARAMETER(posetype, PoseType, "Type of pose output format",
                               PoseType::YOLOv8HAILO, PoseType_Values, ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorPose
      JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(skeleton, std::string, "YAML file name for definition of pose skeleton. "
                                             "Defines number of joints and links, their names, and their "
                                             "colors for drawing. If file name is relative, it will be with respect "
                                             "to " JEVOIS_SHARE_PATH,
                                             "dnn/skeletons/Coco17.yml", ParamCateg);
 
      //! Parameter \relates jevois::dnn::PostProcessorClassify
      JEVOIS_DECLARE_PARAMETER(serialreport, bool, "Send classification or detection results to serial port",
                               true, ParamCateg);
    }
 
    //! Post-Processor for neural network pipeline
    /*! This is the last step in a deep neural network processing Pipeline. \ingroup dnn */
    class PostProcessor : public jevois::Component
    {
      public:
        
        //! Inherited constructor ok
        using jevois::Component::Component;
 
        //! Destructor
        virtual ~PostProcessor();
 
        //! Freeze/unfreeze parameters that users should not change while running
        virtual void freeze(bool doit) = 0;
 
        //! Process outputs
        virtual void process(std::vector<cv::Mat> const & outs, PreProcessor * preproc) = 0;
 
        //! Report what happened in last process() to console/output video/GUI
        virtual void report(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
                            jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false) = 0;
   };
    
  } // namespace dnn
} // namespace jevois