dnn.hpp 56 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099
  1. /*M///////////////////////////////////////////////////////////////////////////////////////
  2. //
  3. // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  4. //
  5. // By downloading, copying, installing or using the software you agree to this license.
  6. // If you do not agree to this license, do not download, install,
  7. // copy or use the software.
  8. //
  9. //
  10. // License Agreement
  11. // For Open Source Computer Vision Library
  12. //
  13. // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14. // Third party copyrights are property of their respective owners.
  15. //
  16. // Redistribution and use in source and binary forms, with or without modification,
  17. // are permitted provided that the following conditions are met:
  18. //
  19. // * Redistribution's of source code must retain the above copyright notice,
  20. // this list of conditions and the following disclaimer.
  21. //
  22. // * Redistribution's in binary form must reproduce the above copyright notice,
  23. // this list of conditions and the following disclaimer in the documentation
  24. // and/or other materials provided with the distribution.
  25. //
  26. // * The name of the copyright holders may not be used to endorse or promote products
  27. // derived from this software without specific prior written permission.
  28. //
  29. // This software is provided by the copyright holders and contributors "as is" and
  30. // any express or implied warranties, including, but not limited to, the implied
  31. // warranties of merchantability and fitness for a particular purpose are disclaimed.
  32. // In no event shall the Intel Corporation or contributors be liable for any direct,
  33. // indirect, incidental, special, exemplary, or consequential damages
  34. // (including, but not limited to, procurement of substitute goods or services;
  35. // loss of use, data, or profits; or business interruption) however caused
  36. // and on any theory of liability, whether in contract, strict liability,
  37. // or tort (including negligence or otherwise) arising in any way out of
  38. // the use of this software, even if advised of the possibility of such damage.
  39. //
  40. //M*/
  41. #ifndef OPENCV_DNN_DNN_HPP
  42. #define OPENCV_DNN_DNN_HPP
  43. #include <vector>
  44. #include <opencv2/core.hpp>
  45. #include "opencv2/core/async.hpp"
  46. #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_EXPERIMENTAL_NS
  47. #define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v26 {
  48. #define CV__DNN_EXPERIMENTAL_NS_END }
  49. namespace cv { namespace dnn { namespace experimental_dnn_34_v26 { } using namespace experimental_dnn_34_v26; }}
  50. #else
  51. #define CV__DNN_EXPERIMENTAL_NS_BEGIN
  52. #define CV__DNN_EXPERIMENTAL_NS_END
  53. #endif
  54. #include <opencv2/dnn/dict.hpp>
  55. namespace cv {
  56. namespace dnn {
  57. CV__DNN_EXPERIMENTAL_NS_BEGIN
  58. //! @addtogroup dnn
  59. //! @{
  60. typedef std::vector<int> MatShape;
  61. /**
  62. * @brief Enum of computation backends supported by layers.
  63. * @see Net::setPreferableBackend
  64. */
  65. enum Backend
  66. {
  67. //! DNN_BACKEND_DEFAULT equals to DNN_BACKEND_INFERENCE_ENGINE if
  68. //! OpenCV is built with Intel's Inference Engine library or
  69. //! DNN_BACKEND_OPENCV otherwise.
  70. DNN_BACKEND_DEFAULT = 0,
  71. DNN_BACKEND_HALIDE,
  72. DNN_BACKEND_INFERENCE_ENGINE, //!< Intel's Inference Engine computational backend
  73. //!< @sa setInferenceEngineBackendType
  74. DNN_BACKEND_OPENCV,
  75. // OpenCV 4.x: DNN_BACKEND_VKCOM,
  76. // OpenCV 4.x: DNN_BACKEND_CUDA,
  77. #ifdef __OPENCV_BUILD
  78. DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
  79. DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
  80. #endif
  81. };
  82. /**
  83. * @brief Enum of target devices for computations.
  84. * @see Net::setPreferableTarget
  85. */
  86. enum Target
  87. {
  88. DNN_TARGET_CPU = 0,
  89. DNN_TARGET_OPENCL,
  90. DNN_TARGET_OPENCL_FP16,
  91. DNN_TARGET_MYRIAD,
  92. DNN_TARGET_FPGA //!< FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin.
  93. };
  94. CV_EXPORTS std::vector< std::pair<Backend, Target> > getAvailableBackends();
  95. CV_EXPORTS_W std::vector<Target> getAvailableTargets(dnn::Backend be);
  96. /** @brief This class provides all data needed to initialize layer.
  97. *
  98. * It includes dictionary with scalar params (which can be read by using Dict interface),
  99. * blob params #blobs and optional meta information: #name and #type of layer instance.
  100. */
  101. class CV_EXPORTS LayerParams : public Dict
  102. {
  103. public:
  104. //TODO: Add ability to name blob params
  105. std::vector<Mat> blobs; //!< List of learned parameters stored as blobs.
  106. String name; //!< Name of the layer instance (optional, can be used internal purposes).
  107. String type; //!< Type name which was used for creating layer by layer factory (optional).
  108. };
  109. /**
  110. * @brief Derivatives of this class encapsulates functions of certain backends.
  111. */
  112. class BackendNode
  113. {
  114. public:
  115. explicit BackendNode(int backendId);
  116. virtual ~BackendNode(); //!< Virtual destructor to make polymorphism.
  117. int backendId; //!< Backend identifier.
  118. };
  119. /**
  120. * @brief Derivatives of this class wraps cv::Mat for different backends and targets.
  121. */
  122. class BackendWrapper
  123. {
  124. public:
  125. BackendWrapper(int backendId, int targetId);
  126. /**
  127. * @brief Wrap cv::Mat for specific backend and target.
  128. * @param[in] targetId Target identifier.
  129. * @param[in] m cv::Mat for wrapping.
  130. *
  131. * Make CPU->GPU data transfer if it's require for the target.
  132. */
  133. BackendWrapper(int targetId, const cv::Mat& m);
  134. /**
  135. * @brief Make wrapper for reused cv::Mat.
  136. * @param[in] base Wrapper of cv::Mat that will be reused.
  137. * @param[in] shape Specific shape.
  138. *
  139. * Initialize wrapper from another one. It'll wrap the same host CPU
  140. * memory and mustn't allocate memory on device(i.e. GPU). It might
  141. * has different shape. Use in case of CPU memory reusing for reuse
  142. * associated memory on device too.
  143. */
  144. BackendWrapper(const Ptr<BackendWrapper>& base, const MatShape& shape);
  145. virtual ~BackendWrapper(); //!< Virtual destructor to make polymorphism.
  146. /**
  147. * @brief Transfer data to CPU host memory.
  148. */
  149. virtual void copyToHost() = 0;
  150. /**
  151. * @brief Indicate that an actual data is on CPU.
  152. */
  153. virtual void setHostDirty() = 0;
  154. int backendId; //!< Backend identifier.
  155. int targetId; //!< Target identifier.
  156. };
  157. class CV_EXPORTS ActivationLayer;
  158. /** @brief This interface class allows to build new Layers - are building blocks of networks.
  159. *
  160. * Each class, derived from Layer, must implement allocate() methods to declare own outputs and forward() to compute outputs.
  161. * Also before using the new layer into networks you must register your layer by using one of @ref dnnLayerFactory "LayerFactory" macros.
  162. */
  163. class CV_EXPORTS_W Layer : public Algorithm
  164. {
  165. public:
  166. //! List of learned parameters must be stored here to allow read them by using Net::getParam().
  167. CV_PROP_RW std::vector<Mat> blobs;
  168. /** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
  169. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  170. * @param[in] input vector of already allocated input blobs
  171. * @param[out] output vector of already allocated output blobs
  172. *
  173. * If this method is called after network has allocated all memory for input and output blobs
  174. * and before inferencing.
  175. */
  176. CV_DEPRECATED_EXTERNAL
  177. virtual void finalize(const std::vector<Mat*> &input, std::vector<Mat> &output);
  178. /** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
  179. * @param[in] inputs vector of already allocated input blobs
  180. * @param[out] outputs vector of already allocated output blobs
  181. *
  182. * If this method is called after network has allocated all memory for input and output blobs
  183. * and before inferencing.
  184. */
  185. CV_WRAP virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs);
  186. /** @brief Given the @p input blobs, computes the output @p blobs.
  187. * @deprecated Use Layer::forward(InputArrayOfArrays, OutputArrayOfArrays, OutputArrayOfArrays) instead
  188. * @param[in] input the input blobs.
  189. * @param[out] output allocated output blobs, which will store results of the computation.
  190. * @param[out] internals allocated internal blobs
  191. */
  192. CV_DEPRECATED_EXTERNAL
  193. virtual void forward(std::vector<Mat*> &input, std::vector<Mat> &output, std::vector<Mat> &internals);
  194. /** @brief Given the @p input blobs, computes the output @p blobs.
  195. * @param[in] inputs the input blobs.
  196. * @param[out] outputs allocated output blobs, which will store results of the computation.
  197. * @param[out] internals allocated internal blobs
  198. */
  199. virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
  200. /** @brief Given the @p input blobs, computes the output @p blobs.
  201. * @param[in] inputs the input blobs.
  202. * @param[out] outputs allocated output blobs, which will store results of the computation.
  203. * @param[out] internals allocated internal blobs
  204. */
  205. void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
  206. /** @brief
  207. * @overload
  208. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  209. */
  210. CV_DEPRECATED_EXTERNAL
  211. void finalize(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs);
  212. /** @brief
  213. * @overload
  214. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  215. */
  216. CV_DEPRECATED std::vector<Mat> finalize(const std::vector<Mat> &inputs);
  217. /** @brief Allocates layer and computes output.
  218. * @deprecated This method will be removed in the future release.
  219. */
  220. CV_DEPRECATED CV_WRAP void run(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs,
  221. CV_IN_OUT std::vector<Mat> &internals);
  222. /** @brief Returns index of input blob into the input array.
  223. * @param inputName label of input blob
  224. *
  225. * Each layer input and output can be labeled to easily identify them using "%<layer_name%>[.output_name]" notation.
  226. * This method maps label of input blob to its index into input vector.
  227. */
  228. virtual int inputNameToIndex(String inputName); // FIXIT const
  229. /** @brief Returns index of output blob in output array.
  230. * @see inputNameToIndex()
  231. */
  232. CV_WRAP virtual int outputNameToIndex(const String& outputName); // FIXIT const
  233. /**
  234. * @brief Ask layer if it support specific backend for doing computations.
  235. * @param[in] backendId computation backend identifier.
  236. * @see Backend
  237. */
  238. virtual bool supportBackend(int backendId); // FIXIT const
  239. /**
  240. * @brief Returns Halide backend node.
  241. * @param[in] inputs Input Halide buffers.
  242. * @see BackendNode, BackendWrapper
  243. *
  244. * Input buffers should be exactly the same that will be used in forward invocations.
  245. * Despite we can use Halide::ImageParam based on input shape only,
  246. * it helps prevent some memory management issues (if something wrong,
  247. * Halide tests will be failed).
  248. */
  249. virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs);
  250. virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs, const std::vector<Ptr<BackendNode> >& nodes);
  251. /**
  252. * @brief Automatic Halide scheduling based on layer hyper-parameters.
  253. * @param[in] node Backend node with Halide functions.
  254. * @param[in] inputs Blobs that will be used in forward invocations.
  255. * @param[in] outputs Blobs that will be used in forward invocations.
  256. * @param[in] targetId Target identifier
  257. * @see BackendNode, Target
  258. *
  259. * Layer don't use own Halide::Func members because we can have applied
  260. * layers fusing. In this way the fused function should be scheduled.
  261. */
  262. virtual void applyHalideScheduler(Ptr<BackendNode>& node,
  263. const std::vector<Mat*> &inputs,
  264. const std::vector<Mat> &outputs,
  265. int targetId) const;
  266. /**
  267. * @brief Implement layers fusing.
  268. * @param[in] node Backend node of bottom layer.
  269. * @see BackendNode
  270. *
  271. * Actual for graph-based backends. If layer attached successfully,
  272. * returns non-empty cv::Ptr to node of the same backend.
  273. * Fuse only over the last function.
  274. */
  275. virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node);
  276. /**
  277. * @brief Tries to attach to the layer the subsequent activation layer, i.e. do the layer fusion in a partial case.
  278. * @param[in] layer The subsequent activation layer.
  279. *
  280. * Returns true if the activation layer has been attached successfully.
  281. */
  282. virtual bool setActivation(const Ptr<ActivationLayer>& layer);
  283. /**
  284. * @brief Try to fuse current layer with a next one
  285. * @param[in] top Next layer to be fused.
  286. * @returns True if fusion was performed.
  287. */
  288. virtual bool tryFuse(Ptr<Layer>& top);
  289. /**
  290. * @brief Returns parameters of layers with channel-wise multiplication and addition.
  291. * @param[out] scale Channel-wise multipliers. Total number of values should
  292. * be equal to number of channels.
  293. * @param[out] shift Channel-wise offsets. Total number of values should
  294. * be equal to number of channels.
  295. *
  296. * Some layers can fuse their transformations with further layers.
  297. * In example, convolution + batch normalization. This way base layer
  298. * use weights from layer after it. Fused layer is skipped.
  299. * By default, @p scale and @p shift are empty that means layer has no
  300. * element-wise multiplications or additions.
  301. */
  302. virtual void getScaleShift(Mat& scale, Mat& shift) const;
  303. /**
  304. * @brief "Deattaches" all the layers, attached to particular layer.
  305. */
  306. virtual void unsetAttached();
  307. virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
  308. const int requiredOutputs,
  309. std::vector<MatShape> &outputs,
  310. std::vector<MatShape> &internals) const;
  311. virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
  312. const std::vector<MatShape> &outputs) const {CV_UNUSED(inputs); CV_UNUSED(outputs); return 0;}
  313. virtual bool updateMemoryShapes(const std::vector<MatShape> &inputs);
  314. CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes.
  315. CV_PROP String type; //!< Type name which was used for creating layer by layer factory.
  316. CV_PROP int preferableTarget; //!< prefer target for layer forwarding
  317. Layer();
  318. explicit Layer(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  319. void setParamsFrom(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  320. virtual ~Layer();
  321. };
  322. /** @brief This class allows to create and manipulate comprehensive artificial neural networks.
  323. *
  324. * Neural network is presented as directed acyclic graph (DAG), where vertices are Layer instances,
  325. * and edges specify relationships between layers inputs and outputs.
  326. *
  327. * Each network layer has unique integer id and unique string name inside its network.
  328. * LayerId can store either layer name or layer id.
  329. *
  330. * This class supports reference counting of its instances, i. e. copies point to the same instance.
  331. */
  332. class CV_EXPORTS_W_SIMPLE Net
  333. {
  334. public:
  335. CV_WRAP Net(); //!< Default constructor.
  336. CV_WRAP ~Net(); //!< Destructor frees the net only if there aren't references to the net anymore.
  337. /** @brief Create a network from Intel's Model Optimizer intermediate representation (IR).
  338. * @param[in] xml XML configuration file with network's topology.
  339. * @param[in] bin Binary file with trained weights.
  340. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  341. * backend.
  342. */
  343. CV_WRAP static Net readFromModelOptimizer(const String& xml, const String& bin);
  344. /** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
  345. * @param[in] bufferModelConfig buffer with model's configuration.
  346. * @param[in] bufferWeights buffer with model's trained weights.
  347. * @returns Net object.
  348. */
  349. CV_WRAP static
  350. Net readFromModelOptimizer(const std::vector<uchar>& bufferModelConfig, const std::vector<uchar>& bufferWeights);
  351. /** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
  352. * @param[in] bufferModelConfigPtr buffer pointer of model's configuration.
  353. * @param[in] bufferModelConfigSize buffer size of model's configuration.
  354. * @param[in] bufferWeightsPtr buffer pointer of model's trained weights.
  355. * @param[in] bufferWeightsSize buffer size of model's trained weights.
  356. * @returns Net object.
  357. */
  358. static
  359. Net readFromModelOptimizer(const uchar* bufferModelConfigPtr, size_t bufferModelConfigSize,
  360. const uchar* bufferWeightsPtr, size_t bufferWeightsSize);
  361. /** Returns true if there are no layers in the network. */
  362. CV_WRAP bool empty() const;
  363. /** @brief Dump net to String
  364. * @returns String with structure, hyperparameters, backend, target and fusion
  365. * Call method after setInput(). To see correct backend, target and fusion run after forward().
  366. */
  367. CV_WRAP String dump();
  368. /** @brief Dump net structure, hyperparameters, backend, target and fusion to dot file
  369. * @param path path to output file with .dot extension
  370. * @see dump()
  371. */
  372. CV_WRAP void dumpToFile(const String& path);
  373. /** @brief Adds new layer to the net.
  374. * @param name unique name of the adding layer.
  375. * @param type typename of the adding layer (type must be registered in LayerRegister).
  376. * @param params parameters which will be used to initialize the creating layer.
  377. * @returns unique identifier of created layer, or -1 if a failure will happen.
  378. */
  379. int addLayer(const String &name, const String &type, LayerParams &params);
  380. /** @brief Adds new layer and connects its first input to the first output of previously added layer.
  381. * @see addLayer()
  382. */
  383. int addLayerToPrev(const String &name, const String &type, LayerParams &params);
  384. /** @brief Converts string name of the layer to the integer identifier.
  385. * @returns id of the layer, or -1 if the layer wasn't found.
  386. */
  387. CV_WRAP int getLayerId(const String &layer) const;
  388. CV_WRAP std::vector<String> getLayerNames() const;
  389. /** @brief Container for strings and integers.
  390. *
  391. * @deprecated Use getLayerId() with int result.
  392. */
  393. typedef DictValue LayerId;
  394. /** @brief Returns pointer to layer with specified id or name which the network use. */
  395. CV_WRAP Ptr<Layer> getLayer(int layerId) const;
  396. /** @overload
  397. * @deprecated Use int getLayerId(const String &layer)
  398. */
  399. CV_WRAP inline Ptr<Layer> getLayer(const String& layerName) const { return getLayer(getLayerId(layerName)); }
  400. /** @overload
  401. * @deprecated to be removed
  402. */
  403. CV_WRAP Ptr<Layer> getLayer(const LayerId& layerId) const;
  404. /** @brief Returns pointers to input layers of specific layer. */
  405. std::vector<Ptr<Layer> > getLayerInputs(int layerId) const; // FIXIT: CV_WRAP
  406. /** @brief Connects output of the first layer to input of the second layer.
  407. * @param outPin descriptor of the first layer output.
  408. * @param inpPin descriptor of the second layer input.
  409. *
  410. * Descriptors have the following template <DFN>&lt;layer_name&gt;[.input_number]</DFN>:
  411. * - the first part of the template <DFN>layer_name</DFN> is string name of the added layer.
  412. * If this part is empty then the network input pseudo layer will be used;
  413. * - the second optional part of the template <DFN>input_number</DFN>
  414. * is either number of the layer input, either label one.
  415. * If this part is omitted then the first layer input will be used.
  416. *
  417. * @see setNetInputs(), Layer::inputNameToIndex(), Layer::outputNameToIndex()
  418. */
  419. CV_WRAP void connect(String outPin, String inpPin);
  420. /** @brief Connects #@p outNum output of the first layer to #@p inNum input of the second layer.
  421. * @param outLayerId identifier of the first layer
  422. * @param outNum number of the first layer output
  423. * @param inpLayerId identifier of the second layer
  424. * @param inpNum number of the second layer input
  425. */
  426. void connect(int outLayerId, int outNum, int inpLayerId, int inpNum);
  427. /** @brief Registers network output with name
  428. *
  429. * Function may create additional 'Identity' layer.
  430. *
  431. * @param outputName identifier of the output
  432. * @param layerId identifier of the second layer
  433. * @param outputPort number of the second layer input
  434. *
  435. * @returns index of bound layer (the same as layerId or newly created)
  436. */
  437. int registerOutput(const std::string& outputName, int layerId, int outputPort);
  438. /** @brief Sets outputs names of the network input pseudo layer.
  439. *
  440. * Each net always has special own the network input pseudo layer with id=0.
  441. * This layer stores the user blobs only and don't make any computations.
  442. * In fact, this layer provides the only way to pass user data into the network.
  443. * As any other layer, this layer can label its outputs and this function provides an easy way to do this.
  444. */
  445. CV_WRAP void setInputsNames(const std::vector<String> &inputBlobNames);
  446. /** @brief Specify shape of network input.
  447. */
  448. CV_WRAP void setInputShape(const String &inputName, const MatShape& shape);
  449. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  450. * @param outputName name for layer which output is needed to get
  451. * @return blob for first output of specified layer.
  452. * @details By default runs forward pass for the whole network.
  453. */
  454. CV_WRAP Mat forward(const String& outputName = String());
  455. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  456. * @param outputName name for layer which output is needed to get
  457. * @details By default runs forward pass for the whole network.
  458. *
  459. * This is an asynchronous version of forward(const String&).
  460. * dnn::DNN_BACKEND_INFERENCE_ENGINE backend is required.
  461. */
  462. CV_WRAP AsyncArray forwardAsync(const String& outputName = String());
  463. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  464. * @param outputBlobs contains all output blobs for specified layer.
  465. * @param outputName name for layer which output is needed to get
  466. * @details If @p outputName is empty, runs forward pass for the whole network.
  467. */
  468. CV_WRAP void forward(OutputArrayOfArrays outputBlobs, const String& outputName = String());
  469. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  470. * @param outputBlobs contains blobs for first outputs of specified layers.
  471. * @param outBlobNames names for layers which outputs are needed to get
  472. */
  473. CV_WRAP void forward(OutputArrayOfArrays outputBlobs,
  474. const std::vector<String>& outBlobNames);
  475. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  476. * @param outputBlobs contains all output blobs for each layer specified in @p outBlobNames.
  477. * @param outBlobNames names for layers which outputs are needed to get
  478. */
  479. CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector<std::vector<Mat> >& outputBlobs,
  480. const std::vector<String>& outBlobNames);
  481. /**
  482. * @brief Compile Halide layers.
  483. * @param[in] scheduler Path to YAML file with scheduling directives.
  484. * @see setPreferableBackend
  485. *
  486. * Schedule layers that support Halide backend. Then compile them for
  487. * specific target. For layers that not represented in scheduling file
  488. * or if no manual scheduling used at all, automatic scheduling will be applied.
  489. */
  490. CV_WRAP void setHalideScheduler(const String& scheduler);
  491. /**
  492. * @brief Ask network to use specific computation backend where it supported.
  493. * @param[in] backendId backend identifier.
  494. * @see Backend
  495. *
  496. * If OpenCV is compiled with Intel's Inference Engine library, DNN_BACKEND_DEFAULT
  497. * means DNN_BACKEND_INFERENCE_ENGINE. Otherwise it equals to DNN_BACKEND_OPENCV.
  498. */
  499. CV_WRAP void setPreferableBackend(int backendId);
  500. /**
  501. * @brief Ask network to make computations on specific target device.
  502. * @param[in] targetId target identifier.
  503. * @see Target
  504. *
  505. * List of supported combinations backend / target:
  506. * | | DNN_BACKEND_OPENCV | DNN_BACKEND_INFERENCE_ENGINE | DNN_BACKEND_HALIDE |
  507. * |------------------------|--------------------|------------------------------|--------------------|
  508. * | DNN_TARGET_CPU | + | + | + |
  509. * | DNN_TARGET_OPENCL | + | + | + |
  510. * | DNN_TARGET_OPENCL_FP16 | + | + | |
  511. * | DNN_TARGET_MYRIAD | | + | |
  512. * | DNN_TARGET_FPGA | | + | |
  513. */
  514. CV_WRAP void setPreferableTarget(int targetId);
  515. /** @brief Sets the new input value for the network
  516. * @param blob A new blob. Should have CV_32F or CV_8U depth.
  517. * @param name A name of input layer.
  518. * @param scalefactor An optional normalization scale.
  519. * @param mean An optional mean subtraction values.
  520. * @see connect(String, String) to know format of the descriptor.
  521. *
  522. * If scale or mean values are specified, a final input blob is computed
  523. * as:
  524. * \f[input(n,c,h,w) = scalefactor \times (blob(n,c,h,w) - mean_c)\f]
  525. */
  526. CV_WRAP void setInput(InputArray blob, const String& name = "",
  527. double scalefactor = 1.0, const Scalar& mean = Scalar());
  528. /** @brief Sets the new value for the learned param of the layer.
  529. * @param layer name or id of the layer.
  530. * @param numParam index of the layer parameter in the Layer::blobs array.
  531. * @param blob the new value.
  532. * @see Layer::blobs
  533. * @note If shape of the new blob differs from the previous shape,
  534. * then the following forward pass may fail.
  535. */
  536. CV_WRAP void setParam(int layer, int numParam, const Mat &blob);
  537. CV_WRAP inline void setParam(const String& layerName, int numParam, const Mat &blob) { return setParam(getLayerId(layerName), numParam, blob); }
  538. /** @brief Returns parameter blob of the layer.
  539. * @param layer name or id of the layer.
  540. * @param numParam index of the layer parameter in the Layer::blobs array.
  541. * @see Layer::blobs
  542. */
  543. CV_WRAP Mat getParam(int layer, int numParam = 0) const;
  544. CV_WRAP inline Mat getParam(const String& layerName, int numParam = 0) const { return getParam(getLayerId(layerName), numParam); }
  545. /** @brief Returns indexes of layers with unconnected outputs.
  546. *
  547. * FIXIT: Rework API to registerOutput() approach, deprecate this call
  548. */
  549. CV_WRAP std::vector<int> getUnconnectedOutLayers() const;
  550. /** @brief Returns names of layers with unconnected outputs.
  551. *
  552. * FIXIT: Rework API to registerOutput() approach, deprecate this call
  553. */
  554. CV_WRAP std::vector<String> getUnconnectedOutLayersNames() const;
  555. /** @brief Returns input and output shapes for all layers in loaded model;
  556. * preliminary inferencing isn't necessary.
  557. * @param netInputShapes shapes for all input blobs in net input layer.
  558. * @param layersIds output parameter for layer IDs.
  559. * @param inLayersShapes output parameter for input layers shapes;
  560. * order is the same as in layersIds
  561. * @param outLayersShapes output parameter for output layers shapes;
  562. * order is the same as in layersIds
  563. */
  564. CV_WRAP void getLayersShapes(const std::vector<MatShape>& netInputShapes,
  565. CV_OUT std::vector<int>& layersIds,
  566. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  567. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  568. /** @overload */
  569. CV_WRAP void getLayersShapes(const MatShape& netInputShape,
  570. CV_OUT std::vector<int>& layersIds,
  571. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  572. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  573. /** @brief Returns input and output shapes for layer with specified
  574. * id in loaded model; preliminary inferencing isn't necessary.
  575. * @param netInputShape shape input blob in net input layer.
  576. * @param layerId id for layer.
  577. * @param inLayerShapes output parameter for input layers shapes;
  578. * order is the same as in layersIds
  579. * @param outLayerShapes output parameter for output layers shapes;
  580. * order is the same as in layersIds
  581. */
  582. void getLayerShapes(const MatShape& netInputShape,
  583. const int layerId,
  584. CV_OUT std::vector<MatShape>& inLayerShapes,
  585. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  586. /** @overload */
  587. void getLayerShapes(const std::vector<MatShape>& netInputShapes,
  588. const int layerId,
  589. CV_OUT std::vector<MatShape>& inLayerShapes,
  590. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  591. /** @brief Computes FLOP for whole loaded model with specified input shapes.
  592. * @param netInputShapes vector of shapes for all net inputs.
  593. * @returns computed FLOP.
  594. */
  595. CV_WRAP int64 getFLOPS(const std::vector<MatShape>& netInputShapes) const;
  596. /** @overload */
  597. CV_WRAP int64 getFLOPS(const MatShape& netInputShape) const;
  598. /** @overload */
  599. CV_WRAP int64 getFLOPS(const int layerId,
  600. const std::vector<MatShape>& netInputShapes) const;
  601. /** @overload */
  602. CV_WRAP int64 getFLOPS(const int layerId,
  603. const MatShape& netInputShape) const;
  604. /** @brief Returns list of types for layer used in model.
  605. * @param layersTypes output parameter for returning types.
  606. */
  607. CV_WRAP void getLayerTypes(CV_OUT std::vector<String>& layersTypes) const;
  608. /** @brief Returns count of layers of specified type.
  609. * @param layerType type.
  610. * @returns count of layers
  611. */
  612. CV_WRAP int getLayersCount(const String& layerType) const;
  613. /** @brief Computes bytes number which are required to store
  614. * all weights and intermediate blobs for model.
  615. * @param netInputShapes vector of shapes for all net inputs.
  616. * @param weights output parameter to store resulting bytes for weights.
  617. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  618. */
  619. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  620. CV_OUT size_t& weights, CV_OUT size_t& blobs) const; // FIXIT: CV_WRAP
  621. /** @overload */
  622. CV_WRAP void getMemoryConsumption(const MatShape& netInputShape,
  623. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  624. /** @overload */
  625. CV_WRAP void getMemoryConsumption(const int layerId,
  626. const std::vector<MatShape>& netInputShapes,
  627. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  628. /** @overload */
  629. CV_WRAP void getMemoryConsumption(const int layerId,
  630. const MatShape& netInputShape,
  631. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  632. /** @brief Computes bytes number which are required to store
  633. * all weights and intermediate blobs for each layer.
  634. * @param netInputShapes vector of shapes for all net inputs.
  635. * @param layerIds output vector to save layer IDs.
  636. * @param weights output parameter to store resulting bytes for weights.
  637. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  638. */
  639. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  640. CV_OUT std::vector<int>& layerIds,
  641. CV_OUT std::vector<size_t>& weights,
  642. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  643. /** @overload */
  644. void getMemoryConsumption(const MatShape& netInputShape,
  645. CV_OUT std::vector<int>& layerIds,
  646. CV_OUT std::vector<size_t>& weights,
  647. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  648. /** @brief Enables or disables layer fusion in the network.
  649. * @param fusion true to enable the fusion, false to disable. The fusion is enabled by default.
  650. */
  651. CV_WRAP void enableFusion(bool fusion);
  652. /** @brief Returns overall time for inference and timings (in ticks) for layers.
  653. *
  654. * Indexes in returned vector correspond to layers ids. Some layers can be fused with others,
  655. * in this case zero ticks count will be return for that skipped layers. Supported by DNN_BACKEND_OPENCV on DNN_TARGET_CPU only.
  656. *
  657. * @param[out] timings vector for tick timings for all layers.
  658. * @return overall ticks for model inference.
  659. */
  660. CV_WRAP int64 getPerfProfile(CV_OUT std::vector<double>& timings);
  661. private:
  662. struct Impl;
  663. Ptr<Impl> impl;
  664. };
  665. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  666. * @param cfgFile path to the .cfg file with text description of the network architecture.
  667. * @param darknetModel path to the .weights file with learned network.
  668. * @returns Network object that ready to do forward, throw an exception in failure cases.
  669. * @returns Net object.
  670. */
  671. CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String());
  672. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  673. * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
  674. * @param bufferModel A buffer contains a content of .weights file with learned network.
  675. * @returns Net object.
  676. */
  677. CV_EXPORTS_W Net readNetFromDarknet(const std::vector<uchar>& bufferCfg,
  678. const std::vector<uchar>& bufferModel = std::vector<uchar>());
  679. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  680. * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
  681. * @param lenCfg Number of bytes to read from bufferCfg
  682. * @param bufferModel A buffer contains a content of .weights file with learned network.
  683. * @param lenModel Number of bytes to read from bufferModel
  684. * @returns Net object.
  685. */
  686. CV_EXPORTS Net readNetFromDarknet(const char *bufferCfg, size_t lenCfg,
  687. const char *bufferModel = NULL, size_t lenModel = 0);
  688. /** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format.
  689. * @param prototxt path to the .prototxt file with text description of the network architecture.
  690. * @param caffeModel path to the .caffemodel file with learned network.
  691. * @returns Net object.
  692. */
  693. CV_EXPORTS_W Net readNetFromCaffe(const String &prototxt, const String &caffeModel = String());
  694. /** @brief Reads a network model stored in Caffe model in memory.
  695. * @param bufferProto buffer containing the content of the .prototxt file
  696. * @param bufferModel buffer containing the content of the .caffemodel file
  697. * @returns Net object.
  698. */
  699. CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto,
  700. const std::vector<uchar>& bufferModel = std::vector<uchar>());
  701. /** @brief Reads a network model stored in Caffe model in memory.
  702. * @details This is an overloaded member function, provided for convenience.
  703. * It differs from the above function only in what argument(s) it accepts.
  704. * @param bufferProto buffer containing the content of the .prototxt file
  705. * @param lenProto length of bufferProto
  706. * @param bufferModel buffer containing the content of the .caffemodel file
  707. * @param lenModel length of bufferModel
  708. * @returns Net object.
  709. */
  710. CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto,
  711. const char *bufferModel = NULL, size_t lenModel = 0);
  712. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  713. * @param model path to the .pb file with binary protobuf description of the network architecture
  714. * @param config path to the .pbtxt file that contains text graph definition in protobuf format.
  715. * Resulting Net object is built by text graph using weights from a binary one that
  716. * let us make it more flexible.
  717. * @returns Net object.
  718. */
  719. CV_EXPORTS_W Net readNetFromTensorflow(const String &model, const String &config = String());
  720. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  721. * @param bufferModel buffer containing the content of the pb file
  722. * @param bufferConfig buffer containing the content of the pbtxt file
  723. * @returns Net object.
  724. */
  725. CV_EXPORTS_W Net readNetFromTensorflow(const std::vector<uchar>& bufferModel,
  726. const std::vector<uchar>& bufferConfig = std::vector<uchar>());
  727. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  728. * @details This is an overloaded member function, provided for convenience.
  729. * It differs from the above function only in what argument(s) it accepts.
  730. * @param bufferModel buffer containing the content of the pb file
  731. * @param lenModel length of bufferModel
  732. * @param bufferConfig buffer containing the content of the pbtxt file
  733. * @param lenConfig length of bufferConfig
  734. */
  735. CV_EXPORTS Net readNetFromTensorflow(const char *bufferModel, size_t lenModel,
  736. const char *bufferConfig = NULL, size_t lenConfig = 0);
  737. /**
  738. * @brief Reads a network model stored in <a href="http://torch.ch">Torch7</a> framework's format.
  739. * @param model path to the file, dumped from Torch by using torch.save() function.
  740. * @param isBinary specifies whether the network was serialized in ascii mode or binary.
  741. * @param evaluate specifies testing phase of network. If true, it's similar to evaluate() method in Torch.
  742. * @returns Net object.
  743. *
  744. * @note Ascii mode of Torch serializer is more preferable, because binary mode extensively use `long` type of C language,
  745. * which has various bit-length on different systems.
  746. *
  747. * The loading file must contain serialized <a href="https://github.com/torch/nn/blob/master/doc/module.md">nn.Module</a> object
  748. * with importing network. Try to eliminate a custom objects from serialazing data to avoid importing errors.
  749. *
  750. * List of supported layers (i.e. object instances derived from Torch nn.Module class):
  751. * - nn.Sequential
  752. * - nn.Parallel
  753. * - nn.Concat
  754. * - nn.Linear
  755. * - nn.SpatialConvolution
  756. * - nn.SpatialMaxPooling, nn.SpatialAveragePooling
  757. * - nn.ReLU, nn.TanH, nn.Sigmoid
  758. * - nn.Reshape
  759. * - nn.SoftMax, nn.LogSoftMax
  760. *
  761. * Also some equivalents of these classes from cunn, cudnn, and fbcunn may be successfully imported.
  762. */
  763. CV_EXPORTS_W Net readNetFromTorch(const String &model, bool isBinary = true, bool evaluate = true);
  764. /**
  765. * @brief Read deep learning network represented in one of the supported formats.
  766. * @param[in] model Binary file contains trained weights. The following file
  767. * extensions are expected for models from different frameworks:
  768. * * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/)
  769. * * `*.pb` (TensorFlow, https://www.tensorflow.org/)
  770. * * `*.t7` | `*.net` (Torch, http://torch.ch/)
  771. * * `*.weights` (Darknet, https://pjreddie.com/darknet/)
  772. * * `*.bin` (DLDT, https://software.intel.com/openvino-toolkit)
  773. * * `*.onnx` (ONNX, https://onnx.ai/)
  774. * @param[in] config Text file contains network configuration. It could be a
  775. * file with the following extensions:
  776. * * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/)
  777. * * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/)
  778. * * `*.cfg` (Darknet, https://pjreddie.com/darknet/)
  779. * * `*.xml` (DLDT, https://software.intel.com/openvino-toolkit)
  780. * @param[in] framework Explicit framework name tag to determine a format.
  781. * @returns Net object.
  782. *
  783. * This function automatically detects an origin framework of trained model
  784. * and calls an appropriate function such @ref readNetFromCaffe, @ref readNetFromTensorflow,
  785. * @ref readNetFromTorch or @ref readNetFromDarknet. An order of @p model and @p config
  786. * arguments does not matter.
  787. */
  788. CV_EXPORTS_W Net readNet(const String& model, const String& config = "", const String& framework = "");
  789. /**
  790. * @brief Read deep learning network represented in one of the supported formats.
  791. * @details This is an overloaded member function, provided for convenience.
  792. * It differs from the above function only in what argument(s) it accepts.
  793. * @param[in] framework Name of origin framework.
  794. * @param[in] bufferModel A buffer with a content of binary file with weights
  795. * @param[in] bufferConfig A buffer with a content of text file contains network configuration.
  796. * @returns Net object.
  797. */
  798. CV_EXPORTS_W Net readNet(const String& framework, const std::vector<uchar>& bufferModel,
  799. const std::vector<uchar>& bufferConfig = std::vector<uchar>());
  800. /** @brief Loads blob which was serialized as torch.Tensor object of Torch7 framework.
  801. * @warning This function has the same limitations as readNetFromTorch().
  802. */
  803. CV_EXPORTS_W Mat readTorchBlob(const String &filename, bool isBinary = true);
  804. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  805. * @param[in] xml XML configuration file with network's topology.
  806. * @param[in] bin Binary file with trained weights.
  807. * @returns Net object.
  808. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  809. * backend.
  810. */
  811. CV_EXPORTS_W
  812. Net readNetFromModelOptimizer(const String &xml, const String &bin);
  813. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  814. * @param[in] bufferModelConfig Buffer contains XML configuration with network's topology.
  815. * @param[in] bufferWeights Buffer contains binary data with trained weights.
  816. * @returns Net object.
  817. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  818. * backend.
  819. */
  820. CV_EXPORTS_W
  821. Net readNetFromModelOptimizer(const std::vector<uchar>& bufferModelConfig, const std::vector<uchar>& bufferWeights);
  822. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  823. * @param[in] bufferModelConfigPtr Pointer to buffer which contains XML configuration with network's topology.
  824. * @param[in] bufferModelConfigSize Binary size of XML configuration data.
  825. * @param[in] bufferWeightsPtr Pointer to buffer which contains binary data with trained weights.
  826. * @param[in] bufferWeightsSize Binary size of trained weights data.
  827. * @returns Net object.
  828. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  829. * backend.
  830. */
  831. CV_EXPORTS
  832. Net readNetFromModelOptimizer(const uchar* bufferModelConfigPtr, size_t bufferModelConfigSize,
  833. const uchar* bufferWeightsPtr, size_t bufferWeightsSize);
  834. /** @brief Reads a network model <a href="https://onnx.ai/">ONNX</a>.
  835. * @param onnxFile path to the .onnx file with text description of the network architecture.
  836. * @returns Network object that ready to do forward, throw an exception in failure cases.
  837. */
  838. CV_EXPORTS_W Net readNetFromONNX(const String &onnxFile);
  839. /** @brief Reads a network model from <a href="https://onnx.ai/">ONNX</a>
  840. * in-memory buffer.
  841. * @param buffer memory address of the first byte of the buffer.
  842. * @param sizeBuffer size of the buffer.
  843. * @returns Network object that ready to do forward, throw an exception
  844. * in failure cases.
  845. */
  846. CV_EXPORTS Net readNetFromONNX(const char* buffer, size_t sizeBuffer);
  847. /** @brief Reads a network model from <a href="https://onnx.ai/">ONNX</a>
  848. * in-memory buffer.
  849. * @param buffer in-memory buffer that stores the ONNX model bytes.
  850. * @returns Network object that ready to do forward, throw an exception
  851. * in failure cases.
  852. */
  853. CV_EXPORTS_W Net readNetFromONNX(const std::vector<uchar>& buffer);
  854. /** @brief Creates blob from .pb file.
  855. * @param path to the .pb file with input tensor.
  856. * @returns Mat.
  857. */
  858. CV_EXPORTS_W Mat readTensorFromONNX(const String& path);
  859. /** @brief Creates 4-dimensional blob from image. Optionally resizes and crops @p image from center,
  860. * subtract @p mean values, scales values by @p scalefactor, swap Blue and Red channels.
  861. * @param image input image (with 1-, 3- or 4-channels).
  862. * @param size spatial size for output image
  863. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  864. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  865. * @param scalefactor multiplier for @p image values.
  866. * @param swapRB flag which indicates that swap first and last channels
  867. * in 3-channel image is necessary.
  868. * @param crop flag which indicates whether image will be cropped after resize or not
  869. * @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
  870. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
  871. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  872. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  873. * @returns 4-dimensional Mat with NCHW dimensions order.
  874. */
  875. CV_EXPORTS_W Mat blobFromImage(InputArray image, double scalefactor=1.0, const Size& size = Size(),
  876. const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  877. int ddepth=CV_32F);
  878. /** @brief Creates 4-dimensional blob from image.
  879. * @details This is an overloaded member function, provided for convenience.
  880. * It differs from the above function only in what argument(s) it accepts.
  881. */
  882. CV_EXPORTS void blobFromImage(InputArray image, OutputArray blob, double scalefactor=1.0,
  883. const Size& size = Size(), const Scalar& mean = Scalar(),
  884. bool swapRB=false, bool crop=false, int ddepth=CV_32F);
  885. /** @brief Creates 4-dimensional blob from series of images. Optionally resizes and
  886. * crops @p images from center, subtract @p mean values, scales values by @p scalefactor,
  887. * swap Blue and Red channels.
  888. * @param images input images (all with 1-, 3- or 4-channels).
  889. * @param size spatial size for output image
  890. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  891. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  892. * @param scalefactor multiplier for @p images values.
  893. * @param swapRB flag which indicates that swap first and last channels
  894. * in 3-channel image is necessary.
  895. * @param crop flag which indicates whether image will be cropped after resize or not
  896. * @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
  897. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
  898. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  899. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  900. * @returns 4-dimensional Mat with NCHW dimensions order.
  901. */
  902. CV_EXPORTS_W Mat blobFromImages(InputArrayOfArrays images, double scalefactor=1.0,
  903. Size size = Size(), const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  904. int ddepth=CV_32F);
  905. /** @brief Creates 4-dimensional blob from series of images.
  906. * @details This is an overloaded member function, provided for convenience.
  907. * It differs from the above function only in what argument(s) it accepts.
  908. */
  909. CV_EXPORTS void blobFromImages(InputArrayOfArrays images, OutputArray blob,
  910. double scalefactor=1.0, Size size = Size(),
  911. const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  912. int ddepth=CV_32F);
  913. /** @brief Parse a 4D blob and output the images it contains as 2D arrays through a simpler data structure
  914. * (std::vector<cv::Mat>).
  915. * @param[in] blob_ 4 dimensional array (images, channels, height, width) in floating point precision (CV_32F) from
  916. * which you would like to extract the images.
  917. * @param[out] images_ array of 2D Mat containing the images extracted from the blob in floating point precision
  918. * (CV_32F). They are non normalized neither mean added. The number of returned images equals the first dimension
  919. * of the blob (batch size). Every image has a number of channels equals to the second dimension of the blob (depth).
  920. */
  921. CV_EXPORTS_W void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_);
  922. /** @brief Convert all weights of Caffe network to half precision floating point.
  923. * @param src Path to origin model from Caffe framework contains single
  924. * precision floating point weights (usually has `.caffemodel` extension).
  925. * @param dst Path to destination model with updated weights.
  926. * @param layersTypes Set of layers types which parameters will be converted.
  927. * By default, converts only Convolutional and Fully-Connected layers'
  928. * weights.
  929. *
  930. * @note Shrinked model has no origin float32 weights so it can't be used
  931. * in origin Caffe framework anymore. However the structure of data
  932. * is taken from NVidia's Caffe fork: https://github.com/NVIDIA/caffe.
  933. * So the resulting model may be used there.
  934. */
  935. CV_EXPORTS_W void shrinkCaffeModel(const String& src, const String& dst,
  936. const std::vector<String>& layersTypes = std::vector<String>());
  937. /** @brief Create a text representation for a binary network stored in protocol buffer format.
  938. * @param[in] model A path to binary network.
  939. * @param[in] output A path to output text file to be created.
  940. *
  941. * @note To reduce output file size, trained weights are not included.
  942. */
  943. CV_EXPORTS_W void writeTextGraph(const String& model, const String& output);
  944. /** @brief Performs non maximum suppression given boxes and corresponding scores.
  945. * @param bboxes a set of bounding boxes to apply NMS.
  946. * @param scores a set of corresponding confidences.
  947. * @param score_threshold a threshold used to filter boxes by score.
  948. * @param nms_threshold a threshold used in non maximum suppression.
  949. * @param indices the kept indices of bboxes after NMS.
  950. * @param eta a coefficient in adaptive threshold formula: \f$nms\_threshold_{i+1}=eta\cdot nms\_threshold_i\f$.
  951. * @param top_k if `>0`, keep at most @p top_k picked indices.
  952. */
  953. CV_EXPORTS void NMSBoxes(const std::vector<Rect>& bboxes, const std::vector<float>& scores,
  954. const float score_threshold, const float nms_threshold,
  955. CV_OUT std::vector<int>& indices,
  956. const float eta = 1.f, const int top_k = 0);
  957. CV_EXPORTS_W void NMSBoxes(const std::vector<Rect2d>& bboxes, const std::vector<float>& scores,
  958. const float score_threshold, const float nms_threshold,
  959. CV_OUT std::vector<int>& indices,
  960. const float eta = 1.f, const int top_k = 0);
  961. CV_EXPORTS_AS(NMSBoxesRotated) void NMSBoxes(const std::vector<RotatedRect>& bboxes, const std::vector<float>& scores,
  962. const float score_threshold, const float nms_threshold,
  963. CV_OUT std::vector<int>& indices,
  964. const float eta = 1.f, const int top_k = 0);
  965. //! @}
  966. CV__DNN_EXPERIMENTAL_NS_END
  967. }
  968. }
  969. #include <opencv2/dnn/layer.hpp>
  970. #include <opencv2/dnn/dnn.inl.hpp>
  971. /// @deprecated Include this header directly from application. Automatic inclusion will be removed
  972. #include <opencv2/dnn/utils/inference_engine.hpp>
  973. #endif /* OPENCV_DNN_DNN_HPP */