Mandatory APIs for NNStreamer Filter sub-plugins (No External Dependencies) More...

#include "tensor_typedef.h"

Include dependency graph for nnstreamer_plugin_api_filter.h:

This graph shows which files directly or indirectly include this file:

Classes
struct	_GstTensorFilterProperties
	GstTensorFilter's properties for NN framework (internal data structure) More...

struct	_GstTensorFilterFrameworkStatistics
	Structure definition for tensor-filter framework statistics. More...

struct	_GstTensorFilterFrameworkInfo
	Tensor_Filter Subplugin framework related information. More...

struct	_GstTensorFilterFrameworkEventData
	User data for the tensor_tilter subplugin related events. More...

struct	_GstTensorFilterFramework
	Tensor_Filter Subplugin definition. More...

struct	parse_accl_args
	Accelerator related arguments for parsing. More...

Macros
#define	ACCL_NONE_STR "none"

#define	ACCL_DEFAULT_STR "default"

#define	ACCL_AUTO_STR "auto"

#define	ACCL_CPU_STR "cpu"

#define	ACCL_CPU_SIMD_STR "cpu.simd"

#define	ACCL_CPU_NEON_STR "cpu.neon"

#define	ACCL_GPU_STR "gpu"

#define	ACCL_NPU_STR "npu"

#define	ACCL_NPU_MOVIDIUS_STR "npu.movidius"

#define	ACCL_NPU_EDGE_TPU_STR "npu.edgetpu"

#define	ACCL_NPU_VIVANTE_STR "npu.vivante"

#define	ACCL_NPU_SRCN_STR "npu.srcn" /** srcn hardware supported by nnfw */

#define	ACCL_NPU_SLSI_STR "npu.slsi"

#define	ACCL_NPU_SR_STR "npu.sr"

#define	GST_TENSOR_FILTER_FRAMEWORK_BASE (0xDEAFDEAD00000000ULL)

#define	GST_TENSOR_FILTER_FRAMEWORK_V0 (GST_TENSOR_FILTER_FRAMEWORK_BASE)

#define	GST_TENSOR_FILTER_FRAMEWORK_V1 (GST_TENSOR_FILTER_FRAMEWORK_BASE \| 0x10000ULL)

#define	GST_TENSOR_FILTER_API_VERSION_DEFINED (1)

#define	GST_TENSOR_FILTER_API_VERSION_MIN (0) /* The minimum API version supported (could be obsolete) */

#define	GST_TENSOR_FILTER_API_VERSION_MAX (1) /* The maximum API version supported (recommended) */

#define	checkGstTensorFilterFrameworkVersion(value, version) ((GST_TENSOR_FILTER_FRAMEWORK_BASE \| ((version) << 16)) == (value & 0xFFFFFFFFFFFF0000ULL))
	Check the value of the version field of GstTensorFilterFramework. More...

#define	parse_accl_hw(...) parse_accl_hw_fill((parse_accl_args){__VA_ARGS__})
	workaround to provide default arguments More...

Typedefs
typedef tensor_layout	tensors_layout[NNS_TENSOR_SIZE_LIMIT]

typedef struct _GstTensorFilterProperties	GstTensorFilterProperties
	GstTensorFilter's properties for NN framework (internal data structure) More...

typedef struct _GstTensorFilterFrameworkStatistics	GstTensorFilterFrameworkStatistics
	Structure definition for tensor-filter framework statistics. More...

typedef struct _GstTensorFilterFrameworkInfo	GstTensorFilterFrameworkInfo
	Tensor_Filter Subplugin framework related information. More...

typedef struct _GstTensorFilterFrameworkEventData	GstTensorFilterFrameworkEventData
	User data for the tensor_tilter subplugin related events. More...

typedef struct _GstTensorFilterFramework	GstTensorFilterFramework

Enumerations
enum	accl_hw { ACCL_NONE = 0, ACCL_AUTO = 0x1, ACCL_DEFAULT = 0x2, ACCL_CPU = 0x1000, ACCL_CPU_SIMD = 0x1100, ACCL_CPU_NEON = 0x1100, ACCL_GPU = 0x2000, ACCL_NPU = 0x4000, ACCL_NPU_MOVIDIUS = 0x4001, ACCL_NPU_EDGE_TPU = 0x4002, ACCL_NPU_VIVANTE = 0x4003, ACCL_NPU_SRCN = 0x4004, ACCL_NPU_SLSI = 0x4005, ACCL_NPU_SR = 0x4100 }
	acceleration hw properties. More...

enum	event_ops { DESTROY_NOTIFY, RELOAD_MODEL, CUSTOM_PROP, SET_INPUT_PROP, SET_OUTPUT_PROP, SET_ACCELERATOR, CHECK_HW_AVAILABILITY }
	Tensor_Filter Subplugin related events. More...

enum	model_info_ops { GET_IN_OUT_INFO, SET_INPUT_INFO }
	Tensor_Filter Subplugin's model related info gathering operations. More...

Functions
int	nnstreamer_filter_probe (GstTensorFilterFramework *tfsp)
	Filter's sub-plugin should call this function to register itself. More...

void	nnstreamer_filter_exit (const char *name)
	Filter's sub-plugin may call this to unregister itself. More...

const GstTensorFilterFramework *	nnstreamer_filter_find (const char *name)
	Find filter sub-plugin with the name. More...

void	nnstreamer_filter_set_custom_property_desc (const char name, const char prop,...)
	set custom property description for tensor filter sub-plugin More...

accl_hw	get_accl_hw_type (const char *str)
	return accl_hw type from string More...

const char *	get_accl_hw_str (const accl_hw key)
	return string based on accl_hw type More...

accl_hw	parse_accl_hw_fill (parse_accl_args accl_args)
	parse user given string to extract accelerator based on given regex filling in optional arguments More...

void *	nnstreamer_filter_shared_model_get (void instance, const char key)
	Get the shared model representation that is already shared and has the same key. More...

void *	nnstreamer_filter_shared_model_insert_and_get (void instance, char key, void *interpreter)
	Insert the new shared model representation and get the value. More...

int	nnstreamer_filter_shared_model_remove (void instance, const char key, void(free_callback)(void ))
	Remove the instance registered at the referred list of shared model table. If referred list is empty, `free_callback` is executed. More...

void	nnstreamer_filter_shared_model_replace (void instance, const char key, void new_interpreter, void(replace_callback)(void , void ), void(free_callback)(void ))
	Helper to reload interpreter for instances that has shared key. `replace_callback` is called iterating instances in referred list. More...

Detailed Description

Mandatory APIs for NNStreamer Filter sub-plugins (No External Dependencies)

Date: 30 Jan 2019

See also: https://github.com/nnstreamer/nnstreamer

Author: MyungJoo Ham myung.nosp@m.joo..nosp@m.ham@s.nosp@m.amsu.nosp@m.ng.co.nosp@m.m

Bug:: No known bugs except for NYI items

Definition in file nnstreamer_plugin_api_filter.h.

Macro Definition Documentation

◆ ACCL_AUTO_STR

#define ACCL_AUTO_STR "auto"

Definition at line 22 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_CPU_NEON_STR

#define ACCL_CPU_NEON_STR "cpu.neon"

Definition at line 25 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_CPU_SIMD_STR

#define ACCL_CPU_SIMD_STR "cpu.simd"

Definition at line 24 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_CPU_STR

#define ACCL_CPU_STR "cpu"

Definition at line 23 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_DEFAULT_STR

#define ACCL_DEFAULT_STR "default"

Definition at line 21 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_GPU_STR

#define ACCL_GPU_STR "gpu"

Definition at line 26 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NONE_STR

#define ACCL_NONE_STR "none"

Macros for accelerator types

Definition at line 20 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NPU_EDGE_TPU_STR

#define ACCL_NPU_EDGE_TPU_STR "npu.edgetpu"

Definition at line 30 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NPU_MOVIDIUS_STR

#define ACCL_NPU_MOVIDIUS_STR "npu.movidius"

Definition at line 29 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NPU_SLSI_STR

#define ACCL_NPU_SLSI_STR "npu.slsi"

Definition at line 33 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NPU_SR_STR

#define ACCL_NPU_SR_STR "npu.sr"

Definition at line 34 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NPU_SRCN_STR

#define ACCL_NPU_SRCN_STR "npu.srcn" /** srcn hardware supported by nnfw */

Definition at line 32 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NPU_STR

#define ACCL_NPU_STR "npu"

Todo:: Define ACCL_DSP_STR

Definition at line 28 of file nnstreamer_plugin_api_filter.h.

◆ ACCL_NPU_VIVANTE_STR

#define ACCL_NPU_VIVANTE_STR "npu.vivante"

Definition at line 31 of file nnstreamer_plugin_api_filter.h.

◆ checkGstTensorFilterFrameworkVersion

#define checkGstTensorFilterFrameworkVersion	(	value,
		version
	)	((GST_TENSOR_FILTER_FRAMEWORK_BASE \| ((version) << 16)) == (value & 0xFFFFFFFFFFFF0000ULL))

Check the value of the version field of GstTensorFilterFramework.

Definition at line 47 of file nnstreamer_plugin_api_filter.h.

◆ GST_TENSOR_FILTER_API_VERSION_DEFINED

#define GST_TENSOR_FILTER_API_VERSION_DEFINED (1)

Definition at line 40 of file nnstreamer_plugin_api_filter.h.

◆ GST_TENSOR_FILTER_API_VERSION_MAX

#define GST_TENSOR_FILTER_API_VERSION_MAX (1) /* The maximum API version supported (recommended) */

Definition at line 42 of file nnstreamer_plugin_api_filter.h.

◆ GST_TENSOR_FILTER_API_VERSION_MIN

#define GST_TENSOR_FILTER_API_VERSION_MIN (0) /* The minimum API version supported (could be obsolete) */

Definition at line 41 of file nnstreamer_plugin_api_filter.h.

◆ GST_TENSOR_FILTER_FRAMEWORK_BASE

#define GST_TENSOR_FILTER_FRAMEWORK_BASE (0xDEAFDEAD00000000ULL)

Definition at line 36 of file nnstreamer_plugin_api_filter.h.

◆ GST_TENSOR_FILTER_FRAMEWORK_V0

#define GST_TENSOR_FILTER_FRAMEWORK_V0 (GST_TENSOR_FILTER_FRAMEWORK_BASE)

Definition at line 37 of file nnstreamer_plugin_api_filter.h.

◆ GST_TENSOR_FILTER_FRAMEWORK_V1

#define GST_TENSOR_FILTER_FRAMEWORK_V1 (GST_TENSOR_FILTER_FRAMEWORK_BASE | 0x10000ULL)

Definition at line 38 of file nnstreamer_plugin_api_filter.h.

◆ parse_accl_hw

#define parse_accl_hw ( ... ) parse_accl_hw_fill((parse_accl_args){__VA_ARGS__})

workaround to provide default arguments

Definition at line 544 of file nnstreamer_plugin_api_filter.h.

Typedef Documentation

◆ GstTensorFilterFramework

typedef struct _GstTensorFilterFramework GstTensorFilterFramework

Definition at line 240 of file nnstreamer_plugin_api_filter.h.

◆ GstTensorFilterFrameworkEventData

typedef struct _GstTensorFilterFrameworkEventData GstTensorFilterFrameworkEventData

User data for the tensor_tilter subplugin related events.

◆ GstTensorFilterFrameworkInfo

typedef struct _GstTensorFilterFrameworkInfo GstTensorFilterFrameworkInfo

Tensor_Filter Subplugin framework related information.

All the information except the supported accelerator is provided statically. Accelerators can be provided based on static or dynamic check dependent on framework support.

◆ GstTensorFilterFrameworkStatistics

typedef struct _GstTensorFilterFrameworkStatistics GstTensorFilterFrameworkStatistics

Structure definition for tensor-filter framework statistics.

◆ GstTensorFilterProperties

typedef struct _GstTensorFilterProperties GstTensorFilterProperties

GstTensorFilter's properties for NN framework (internal data structure)

Because custom filters of tensor_filter may need to access internal data of GstTensorFilter, we define this data structure here.

◆ tensors_layout

typedef tensor_layout tensors_layout[NNS_TENSOR_SIZE_LIMIT]

Definition at line 104 of file nnstreamer_plugin_api_filter.h.

Enumeration Type Documentation

◆ accl_hw

enum accl_hw

acceleration hw properties.

Enabling acceleration (choosing any accelerator hardware other ACCL_NONE) will enable acceleration for the framework dependent on the acceleration supported by the framework.

For example, enabling acceleration with tflite will enable NNAPI. However, with NNFW will enable GPU/NEON etc.

Appropriate acceleration should be used with each framework. For example, ACCL_CPU_NEON is supported with NNFW tensor filter. Using ACCL_NEON with pytorch would result in a warning message, and the accelerator would fallback on ACCL_AUTO.

ACCL_AUTO automatically chooses the accelerator based on the ones supported by the subplugin framework. However, ACCL_DEFAULT would use the accelerator set by the subplugin framework, if any.

Note: Acceleration for a framework can be enabled/disabled in the build. If the acceleration for a framework was disabled in the build, setting the accelerator while use a tensor filter with that framework will have no effect.; Add definition of new accelerators to accl_hw_get_type() in tensor_filter_common.c as well.

Enumerator
ACCL_NONE	no explicit acceleration no acceleration (defaults to CPU)
ACCL_AUTO	If there is no default config, and device needs to be specified, fallback to ACCL_AUTO choose optimized device automatically
ACCL_DEFAULT	use default device configuration by the framework
ACCL_CPU	Enables acceleration, 0xn000 any version of that device, 0xnxxx: device # xxx-1 specify device as CPU, if possible
ACCL_CPU_SIMD	specify device as SIMD in cpu, if possible
ACCL_CPU_NEON	specify device as NEON (alias for SIMD) in cpu, if possible
ACCL_GPU	specify device as GPU, if possible
ACCL_NPU	Todo: Define ACCL_DSP specify device as any NPU, if possible
ACCL_NPU_MOVIDIUS	specify device as movidius, if possible
ACCL_NPU_EDGE_TPU	specify device as edge tpu, if possible
ACCL_NPU_VIVANTE	specify device as vivante, if possible
ACCL_NPU_SRCN	specify device as srcn, if possible
ACCL_NPU_SLSI	specify device as S.LSI, if possible
ACCL_NPU_SR	specify device as any SR npu, if possible

Definition at line 80 of file nnstreamer_plugin_api_filter.h.

◆ event_ops

enum event_ops

Tensor_Filter Subplugin related events.

These are possible set of events that can be supported by the tensor filter subplugin.

Enumerator
DESTROY_NOTIFY	Free the data element allocated in the invoke callback
RELOAD_MODEL	Reloads the subplugin with newly provided model
CUSTOM_PROP	Update the custom properties for the framework
SET_INPUT_PROP	Update input tensor info and layout
SET_OUTPUT_PROP	Update output tensor info and layout
SET_ACCELERATOR	Update accelerator of the subplugin to be used as backend
CHECK_HW_AVAILABILITY	Check the hw availability with custom option

Definition at line 177 of file nnstreamer_plugin_api_filter.h.

◆ model_info_ops

enum model_info_ops

Tensor_Filter Subplugin's model related info gathering operations.

Enumerator
GET_IN_OUT_INFO	Gets the input and output tensor info
SET_INPUT_INFO	Sets the provided input tensor info, and get updated output tensor info

Definition at line 191 of file nnstreamer_plugin_api_filter.h.

Function Documentation

◆ get_accl_hw_str()

const char* get_accl_hw_str ( const accl_hw key )

return string based on accl_hw type

Parameters

key	The key enum value

Returns: Corresponding string. Returns ACCL_NONE_STR if not found.

Note: Do not free the returned char *

Definition at line 2598 of file tensor_filter_common.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ get_accl_hw_type()

accl_hw get_accl_hw_type ( const char * str )

return accl_hw type from string

◆ nnstreamer_filter_exit()

void nnstreamer_filter_exit ( const char * name )

Filter's sub-plugin may call this to unregister itself.

Parameters

[in] name The name of filter sub-plugin.

Definition at line 638 of file tensor_filter_common.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ nnstreamer_filter_find()

const GstTensorFilterFramework* nnstreamer_filter_find ( const char * name )

Find filter sub-plugin with the name.

Parameters

[in] name The name of filter sub-plugin.

Returns: NULL if not found or the sub-plugin object has an error.

Definition at line 697 of file tensor_filter_common.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ nnstreamer_filter_probe()

int nnstreamer_filter_probe ( GstTensorFilterFramework * tfsp )

Filter's sub-plugin should call this function to register itself.

Parameters

[in] tfsp Tensor-Filter Sub-Plugin to be registered.

Returns: TRUE if registered. FALSE is failed or duplicated.

Note: Do not change the subplugins callbacks after probing the filter.

Parameters

[in] tfsp Tensor-Filter Sub-Plugin to be registered.

Returns: TRUE if registered. FALSE is failed or duplicated.

Definition at line 611 of file tensor_filter_common.c.

Here is the call graph for this function:

◆ nnstreamer_filter_set_custom_property_desc()

void nnstreamer_filter_set_custom_property_desc	(	const char *	name,
		const char *	prop,
			...
	)

set custom property description for tensor filter sub-plugin

Definition at line 647 of file tensor_filter_common.c.

Here is the call graph for this function:

◆ nnstreamer_filter_shared_model_get()

void* nnstreamer_filter_shared_model_get	(	void *	instance,
		const char *	key
	)

Get the shared model representation that is already shared and has the same key.

Parameters

[in]	instance	The instance that is sharing the model representation. It will be registered at the referred list.
[in]	key	The key to find the matched shared representation.

Returns: The model interpreter. NULL if it does not exist.

Definition at line 2993 of file tensor_filter_common.c.

◆ nnstreamer_filter_shared_model_insert_and_get()

void* nnstreamer_filter_shared_model_insert_and_get	(	void *	instance,
		char *	key,
		void *	interpreter
	)

Insert the new shared model representation and get the value.

Parameters

[in]	instance	The instance that is sharing the model representation. It will be registered at the referred list.
[in]	key	The key for shared model.
[in]	interpreter	The interpreter to be shared.

Returns: The model interpreter inserted. NULL if it is already inserted.

Internal error case. The interpreter already exists in shared table, do not insert and return null.

Definition at line 3026 of file tensor_filter_common.c.

◆ nnstreamer_filter_shared_model_remove()

int nnstreamer_filter_shared_model_remove	(	void *	instance,
		const char *	key,
		void()(void )	free_callback
	)

Remove the instance registered at the referred list of shared model table. If referred list is empty, free_callback is executed.

Parameters

[in]	instance	The instance that should be removed from the referred list.
[in]	key	The key to find the shared model.
[in]	free_callback	The callback function to destroy the interpreter, which takes the interpreter as arg.

Returns: TRUE if the instance is removed. FALSE if failed to remove it.

Definition at line 3081 of file tensor_filter_common.c.

◆ nnstreamer_filter_shared_model_replace()

void nnstreamer_filter_shared_model_replace	(	void *	instance,
		const char *	key,
		void *	new_interpreter,
		void()(void , void *)	replace_callback,
		void()(void )	free_callback
	)

Helper to reload interpreter for instances that has shared key. replace_callback is called iterating instances in referred list.

Parameters

[in]	instance	The instance that is sharing the model representation.
[in]	key	The key to find the shared model.
[in]	interpreter	The new interpreter to replace.
[in]	replace_callback	The callback function to replace with new interpreter.
[in]	free_callback	The callback function to destroy the old interpreter.

Definition at line 3128 of file tensor_filter_common.c.

◆ parse_accl_hw_fill()

accl_hw parse_accl_hw_fill ( parse_accl_args accl_args )

parse user given string to extract accelerator based on given regex filling in optional arguments

Note

The order of arguments for calling this function is:

in_accl: user provided input accelerator string
sup_accl: list of supported accelerator
auto_accl: auto accelerator (optional)
def_accl: default accelerator (optional)

remove unsupported accelerators from this list based on runtime system

filtered supported accelerators can be empty

update default accelerator if it is not available at runtime

update auto accelerator if it is not available at runtime

Definition at line 2833 of file tensor_filter_common.c.

Here is the call graph for this function:

Classes

Macros

Typedefs

Enumerations

Functions

Detailed Description

Macro Definition Documentation

◆ ACCL_AUTO_STR

◆ ACCL_CPU_NEON_STR

◆ ACCL_CPU_SIMD_STR

◆ ACCL_CPU_STR

◆ ACCL_DEFAULT_STR

◆ ACCL_GPU_STR

◆ ACCL_NONE_STR

◆ ACCL_NPU_EDGE_TPU_STR

◆ ACCL_NPU_MOVIDIUS_STR

◆ ACCL_NPU_SLSI_STR

◆ ACCL_NPU_SR_STR

◆ ACCL_NPU_SRCN_STR

◆ ACCL_NPU_STR

◆ ACCL_NPU_VIVANTE_STR

◆ checkGstTensorFilterFrameworkVersion

◆ GST_TENSOR_FILTER_API_VERSION_DEFINED

◆ GST_TENSOR_FILTER_API_VERSION_MAX

◆ GST_TENSOR_FILTER_API_VERSION_MIN

◆ GST_TENSOR_FILTER_FRAMEWORK_BASE

◆ GST_TENSOR_FILTER_FRAMEWORK_V0

◆ GST_TENSOR_FILTER_FRAMEWORK_V1

◆ parse_accl_hw

Typedef Documentation

◆ GstTensorFilterFramework

◆ GstTensorFilterFrameworkEventData

◆ GstTensorFilterFrameworkInfo

◆ GstTensorFilterFrameworkStatistics

◆ GstTensorFilterProperties

◆ tensors_layout

Enumeration Type Documentation

◆ accl_hw

◆ event_ops

◆ model_info_ops

Function Documentation

◆ get_accl_hw_str()

◆ get_accl_hw_type()

◆ nnstreamer_filter_exit()

◆ nnstreamer_filter_find()

◆ nnstreamer_filter_probe()

◆ nnstreamer_filter_set_custom_property_desc()

◆ nnstreamer_filter_shared_model_get()

◆ nnstreamer_filter_shared_model_insert_and_get()

◆ nnstreamer_filter_shared_model_remove()

◆ nnstreamer_filter_shared_model_replace()

◆ parse_accl_hw_fill()