GStreamer plugin to use general neural network frameworks as filters. More...

#include <string.h>
#include <nnstreamer_util.h>
#include "tensor_filter.h"

Include dependency graph for tensor_filter.c:

Classes
struct	_FilterTransformData
	Internal data structure for tensor_filter transform data. More...

Macros
#define	EVENT_NAME_UPDATE_MODEL "evt_update_model"

#define	GST_CAT_DEFAULT gst_tensor_filter_debug

#define	TF_MODELNAME(prop) ((prop)->model_files ? ((prop)->model_files[0]) : "[No Model File]")

#define	CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_MAKE ("{ static, flexible }")
	Default caps string for both sink and source pad. More...

#define	gst_tensor_filter_parent_class parent_class

#define	LATENCY_REPORT_HEADROOM 0.05
	Headroom (extra duration) added to actual latency estimate reported to LATENCY query, to limit number of updates when tracking the maximum value - arbitrarily set to 5%. More...

#define	LATENCY_REPORT_THRESHOLD 0.25
	Threshold deciding when tracking latency estimate that current value is sufficiently lower than reported value so that a notification update is necessary - arbitrarily set to 25%. More...

#define	THRESHOLD_DROP_OLD (2000)

#define	THRESHOLD_CACHE_OLD (1000)

Typedefs
typedef struct _FilterTransformData	FilterTransformData
	Internal data structure for tensor_filter transform data. More...

Functions
	GST_DEBUG_CATEGORY_STATIC (gst_tensor_filter_debug)

	G_DEFINE_TYPE (GstTensorFilter, gst_tensor_filter, GST_TYPE_BASE_TRANSFORM)

static void	gst_tensor_filter_set_property (GObject object, guint prop_id, const GValue value, GParamSpec *pspec)
	Setter for tensor_filter properties. More...

static void	gst_tensor_filter_get_property (GObject object, guint prop_id, GValue value, GParamSpec *pspec)
	Getter for tensor_filter properties. More...

static void	gst_tensor_filter_finalize (GObject *object)
	Function to finalize instance. More...

static GstFlowReturn	gst_tensor_filter_transform (GstBaseTransform trans, GstBuffer inbuf, GstBuffer *outbuf)
	non-ip transform. required vmethod of GstBaseTransform. More...

static GstCaps *	gst_tensor_filter_transform_caps (GstBaseTransform trans, GstPadDirection direction, GstCaps caps, GstCaps *filter)
	configure tensor-srcpad cap from "proposed" cap. More...

static GstCaps *	gst_tensor_filter_fixate_caps (GstBaseTransform trans, GstPadDirection direction, GstCaps caps, GstCaps *othercaps)
	fixate caps. required vmethod of GstBaseTransform. More...

static gboolean	gst_tensor_filter_set_caps (GstBaseTransform trans, GstCaps incaps, GstCaps *outcaps)
	set caps. required vmethod of GstBaseTransform. More...

static gboolean	gst_tensor_filter_query (GstBaseTransform trans, GstPadDirection direction, GstQuery query)
	query handling, optional vmethod of GstBaseTransform. More...

static gboolean	gst_tensor_filter_transform_size (GstBaseTransform trans, GstPadDirection direction, GstCaps caps, gsize size, GstCaps othercaps, gsize othersize)
	Tell the framework the required size of buffer based on the info of the other side pad. optional vmethod of BaseTransform. More...

static gboolean	gst_tensor_filter_start (GstBaseTransform *trans)
	Called when the element starts processing. optional vmethod of BaseTransform. More...

static gboolean	gst_tensor_filter_stop (GstBaseTransform *trans)
	Called when the element stops processing. optional vmethod of BaseTransform. More...

static gboolean	gst_tensor_filter_sink_event (GstBaseTransform trans, GstEvent event)
	Event handler for sink pad of tensor filter. More...

static gboolean	gst_tensor_filter_src_event (GstBaseTransform trans, GstEvent event)
	Event handler for src pad of tensor filter. More...

static void	gst_tensor_filter_class_init (GstTensorFilterClass *klass)
	initialize the tensor_filter's class More...

static void	gst_tensor_filter_init (GstTensorFilter *self)
	initialize the new element instantiate pads and add them to element set pad callback functions initialize instance structure More...

static gsize	gst_tensor_filter_get_tensor_size (GstTensorFilter *self, guint index, gboolean is_input)
	Calculate tensor buffer size. More...

static void	gst_tensor_filter_destroy_notify (void *data)
	Free the data allocated for tensor transform. More...

static GstMemory *	gst_tensor_filter_get_wrapped_mem (GstTensorFilter *self, gpointer data, gsize size)
	Allocate new memory block from given data. More...

static void	prepare_statistics (GstTensorFilterPrivate *priv)
	Prepare statistics for performance profiling (e.g, latency, throughput) More...

static void	accumulate_latency (void data, void user_data)
	Helper function to accumulate latencies. More...

static void	record_statistics (GstTensorFilterPrivate *priv)
	Record statistics for performance profiling (e.g, latency, throughput) More...

static void	track_latency (GstTensorFilter *self)
	Track estimated latency and notify pipeline when it changes. Latency estimates may be a bit jittery. On the principle we want to inform pipeline with the latency from longest inference. However, first inference may take much longer, or model filter configuration may change. Therefore any change of more than 10% (arbitrary value) to a lower latency is also reported to pipeline. Notification is done sending LATENCY message to bus. Upon receipt, application will initiate a pipeline latency probe via LATENCY query. More...

static gboolean	gst_tensor_filter_check_throttling_delay (GstBaseTransform trans, GstBuffer inbuf)
	Check throttling delay and send qos overflow event to upstream elements. More...

static GstFlowReturn	_gst_tensor_filter_transform_validate (GstBaseTransform trans, GstBuffer inbuf, GstBuffer *outbuf)
	Check input parameters for gst_tensor_filter_transform ();. More...

static void	_gst_tensor_filter_release_mem_until_idx (FilterTransformData *trans_data, guint end_index)
	Internal function to release mem and unmap info with index. More...

static gsize	_gst_tensor_filter_convert_meta (FilterTransformData trans_data, GstTensorsInfo info, guint idx)
	Internal function to convert tensor meta and get header size of flexible tensor. More...

static FilterTransformData *	_gst_tensor_filter_transform_get_all_input_data (GstBaseTransform trans, GstBuffer buf)
	Internal function to get input tensors. More...

static GstTensorMemory *	_gst_tensor_filter_transform_get_invoke_tensors (GstBaseTransform trans, FilterTransformData trans_data)
	Internal function to get invoke tensors. More...

static FilterTransformData *	_gst_tensor_filter_transform_get_output_data (GstBaseTransform *trans)
	Internal function to get output tensors. More...

static GstFlowReturn	_gst_tensor_filter_transform_prepare_output_tensors (GstBaseTransform trans, FilterTransformData trans_data)
	Internal function to get output tensors. More...

static GstFlowReturn	_gst_tensor_filter_transform_check_invoke_result (GstBaseTransform trans, FilterTransformData in_trans_data, FilterTransformData *out_trans_data, gint invoke_res)
	Internal function to check the invoke result. More...

static void	_gst_tensor_filter_transform_update_outbuf (GstBaseTransform trans, FilterTransformData in_trans_data, FilterTransformData out_trans_data, GstBuffer outbuf)
	Internal function to make output buffer. More...

static gboolean	gst_tensor_filter_watchdog_trigger (gpointer ptr)
	Called when there is no input within suspend time specified by the user. More...

static gboolean	gst_tensor_filter_configure_tensor (GstTensorFilter self, const GstCaps incaps)
	Configure input and output tensor info from incaps. More...

Variables
static GstStaticPadTemplate	sink_factory
	The capabilities of the inputs. More...

static GstStaticPadTemplate	src_factory
	The capabilities of the outputs. More...

Detailed Description

GStreamer plugin to use general neural network frameworks as filters.

GStreamer Tensor_Filter Copyright (C) 2005 Thomas Vander Stichele thoma.nosp@m.s@ap.nosp@m.estaa.nosp@m.rt.o.nosp@m.rg Copyright (C) 2005 Ronald S. Bultje rbult.nosp@m.je@r.nosp@m.onald.nosp@m..bit.nosp@m.freak.nosp@m..net Copyright (C) 2018 MyungJoo Ham myung.nosp@m.joo..nosp@m.ham@s.nosp@m.amsu.nosp@m.ng.co.nosp@m.m

This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; version 2.1 of the License.

This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details.

Date: 24 May 2018

See also: http://github.com/nnstreamer/nnstreamer

Author: MyungJoo Ham myung.nosp@m.joo..nosp@m.ham@s.nosp@m.amsu.nosp@m.ng.co.nosp@m.m

Bug:: No known bugs except for NYI items

Todo:

set priority among properties

logic for dynamic properties(like model change)

This is the main plugin for per-NN-framework plugins. Specific implementations for each NN framework must be written in each framework specific files; e.g., tensor_filter_tensorflow_lite.c

Definition in file tensor_filter.c.

Macro Definition Documentation

◆ CAPS_STRING

#define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_MAKE ("{ static, flexible }")

Default caps string for both sink and source pad.

Definition at line 87 of file tensor_filter.c.

◆ EVENT_NAME_UPDATE_MODEL

#define EVENT_NAME_UPDATE_MODEL "evt_update_model"

SECTION:element-tensor_filter

A plugin that invokes neural network models and their framework or an independent shared object implementing tensor_filter_custom.h. The input and output are always in the format of other/tensor or other/tensors.

<refsect2> <title>Example launch line</title> |[ gst-launch -v -m fakesrc ! tensor_filter framework=tensorflow-lite, model=./inception_v3.pb, input=3:224:224, output=1000 ! fakesink silent=TRUE ]| </refsect2>

If input is other/tensor C array input[1][224][224][3] and output is other/tensor C array output[1][1][1][1000]

The current QoS policy: In a nnstreamer pipeline, the QoS is currently satisfied by adjusting a input or output framerate, initiated by 'tensor_rate' element. When 'tensor_filter' receives a throttling QoS event from the 'tensor_rate' element, it compares the average processing latency and throttling delay, and takes the maximum value as the threshold to drop incoming frames by checking a buffer timestamp. In this way, 'tensor filter' can avoid unnecessary calculation and adjust a framerate, effectively reducing resource utilizations. Even in the case of receiving QoS events from multiple downstream pipelines (e.g., tee), 'tensor_filter' takes the minimum value as the throttling delay for downstream pipeline with more tight QoS requirement. Lastly, 'tensor_filter' also sends QoS events to upstream elements (e.g., tensor_converter, tensor_src) to possibly reduce incoming framerates, which is a better solution than dropping framerates.

Todo:: rename & move this to better location

Definition at line 76 of file tensor_filter.c.

◆ GST_CAT_DEFAULT

#define GST_CAT_DEFAULT gst_tensor_filter_debug

Definition at line 79 of file tensor_filter.c.

◆ gst_tensor_filter_parent_class

#define gst_tensor_filter_parent_class parent_class

Definition at line 105 of file tensor_filter.c.

◆ LATENCY_REPORT_HEADROOM

#define LATENCY_REPORT_HEADROOM 0.05

Headroom (extra duration) added to actual latency estimate reported to LATENCY query, to limit number of updates when tracking the maximum value - arbitrarily set to 5%.

Definition at line 113 of file tensor_filter.c.

◆ LATENCY_REPORT_THRESHOLD

#define LATENCY_REPORT_THRESHOLD 0.25

Threshold deciding when tracking latency estimate that current value is sufficiently lower than reported value so that a notification update is necessary - arbitrarily set to 25%.

Definition at line 120 of file tensor_filter.c.

◆ TF_MODELNAME

#define TF_MODELNAME ( prop ) ((prop)->model_files ? ((prop)->model_files[0]) : "[No Model File]")

Definition at line 81 of file tensor_filter.c.

◆ THRESHOLD_CACHE_OLD

#define THRESHOLD_CACHE_OLD (1000)

Definition at line 406 of file tensor_filter.c.

◆ THRESHOLD_DROP_OLD

#define THRESHOLD_DROP_OLD (2000)

Definition at line 405 of file tensor_filter.c.

Typedef Documentation

◆ FilterTransformData

typedef struct _FilterTransformData FilterTransformData

Internal data structure for tensor_filter transform data.

Function Documentation

◆ _gst_tensor_filter_convert_meta()

static gsize _gst_tensor_filter_convert_meta	(	FilterTransformData *	trans_data,
		GstTensorsInfo *	info,
		guint	idx
	)

static

Internal function to convert tensor meta and get header size of flexible tensor.

Definition at line 684 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _gst_tensor_filter_release_mem_until_idx()

static void _gst_tensor_filter_release_mem_until_idx	(	FilterTransformData *	trans_data,
		guint	end_index
	)

static

Internal function to release mem and unmap info with index.

Definition at line 667 of file tensor_filter.c.

Here is the caller graph for this function:

◆ _gst_tensor_filter_transform_check_invoke_result()

static GstFlowReturn _gst_tensor_filter_transform_check_invoke_result	(	GstBaseTransform *	trans,
		FilterTransformData *	in_trans_data,
		FilterTransformData *	out_trans_data,
		gint	invoke_res
	)

static

Internal function to check the invoke result.

Definition at line 936 of file tensor_filter.c.

Here is the caller graph for this function:

◆ _gst_tensor_filter_transform_get_all_input_data()

static FilterTransformData* _gst_tensor_filter_transform_get_all_input_data	(	GstBaseTransform *	trans,
		GstBuffer *	buf
	)

static

Internal function to get input tensors.

Definition at line 707 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _gst_tensor_filter_transform_get_invoke_tensors()

static GstTensorMemory* _gst_tensor_filter_transform_get_invoke_tensors	(	GstBaseTransform *	trans,
		FilterTransformData *	trans_data
	)

static

Internal function to get invoke tensors.

Definition at line 754 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _gst_tensor_filter_transform_get_output_data()

static FilterTransformData* _gst_tensor_filter_transform_get_output_data ( GstBaseTransform * trans )

static

Internal function to get output tensors.

Definition at line 836 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _gst_tensor_filter_transform_prepare_output_tensors()

static GstFlowReturn _gst_tensor_filter_transform_prepare_output_tensors	(	GstBaseTransform *	trans,
		FilterTransformData *	trans_data
	)

static

Internal function to get output tensors.

Definition at line 870 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _gst_tensor_filter_transform_update_outbuf()

static void _gst_tensor_filter_transform_update_outbuf	(	GstBaseTransform *	trans,
		FilterTransformData *	in_trans_data,
		FilterTransformData *	out_trans_data,
		GstBuffer *	outbuf
	)

static

Internal function to make output buffer.

Definition at line 976 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _gst_tensor_filter_transform_validate()

static GstFlowReturn _gst_tensor_filter_transform_validate	(	GstBaseTransform *	trans,
		GstBuffer *	inbuf,
		GstBuffer *	outbuf
	)

static

Check input parameters for gst_tensor_filter_transform ();.

This is fatal; if framework is not configured until this stage, it means that an extension is missing or not configured. We need readable messages for non-developers

Definition at line 593 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ accumulate_latency()

static void accumulate_latency	(	void *	data,
		void *	user_data
	)

static

Helper function to accumulate latencies.

Definition at line 397 of file tensor_filter.c.

Here is the caller graph for this function:

◆ G_DEFINE_TYPE()

G_DEFINE_TYPE	(	GstTensorFilter	,
		gst_tensor_filter	,
		GST_TYPE_BASE_TRANSFORM
	)

◆ GST_DEBUG_CATEGORY_STATIC()

GST_DEBUG_CATEGORY_STATIC ( gst_tensor_filter_debug )

◆ gst_tensor_filter_check_throttling_delay()

static gboolean gst_tensor_filter_check_throttling_delay	(	GstBaseTransform *	trans,
		GstBuffer *	inbuf
	)

static

Check throttling delay and send qos overflow event to upstream elements.

Send qos overflow event to upstream elements. Upstream elements (e.g., tensor_src, tensor_converter) may handle this.

Definition at line 536 of file tensor_filter.c.

Here is the caller graph for this function:

◆ gst_tensor_filter_class_init()

static void gst_tensor_filter_class_init ( GstTensorFilterClass * klass )

static

initialize the tensor_filter's class

Definition at line 169 of file tensor_filter.c.

Here is the call graph for this function:

◆ gst_tensor_filter_configure_tensor()

static gboolean gst_tensor_filter_configure_tensor	(	GstTensorFilter *	self,
		const GstCaps *	incaps
	)

static

Configure input and output tensor info from incaps.

Parameters

self	"this" pointer
incaps	received caps for sink pad

Returns: TRUE if fully configured

GstTensorFilter has to parse the tensor dimension and type from NN model.

Call functions getInputDimension and getOutputDimension to get the dimension and type.
If these functions are not defined, call setInputDimension with parsed info from caps.
If set-prop configured dimension, verify the dimension with fw callbacks.

Check configuration from caps. If true, fully configured tensor info from caps.

if set-property called and already has info, verify it!

If incoming tensor is flexible, we cannot validate tensor info here. Need to compare buffer size in transform().

Todo:: We do not support this (flexible tensor for flexible input model). Cap-negotiation of the current tensor-filter requires either side of "model / set-property" or "incoming gstcaps" to be static/explicit. Ideally, this should support flexible tensor for flexible input model, leaving the negotiation to other elements, but we didn't implement it yet.

call setInputDimension if output tensor is not configured

if set-property called and already has info, verify it!

Todo:: framerate of output tensors How can we update the framerate? GstTensorFilter cannot assure the framerate. Simply set the framerate of out-tensor from incaps.

already configured, compare to old.

Definition at line 1207 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_destroy_notify()

static void gst_tensor_filter_destroy_notify ( void * data )

static

Free the data allocated for tensor transform.

default function for tensor filter framework if not provided by the framework. The data is in GPtrArray - first element is private data of framework and second element is the data to be freed.

Definition at line 357 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_finalize()

static void gst_tensor_filter_finalize ( GObject * object )

static

Function to finalize instance.

Definition at line 249 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_fixate_caps()

static GstCaps * gst_tensor_filter_fixate_caps	(	GstBaseTransform *	trans,
		GstPadDirection	direction,
		GstCaps *	caps,
		GstCaps *	othercaps
	)

static

fixate caps. required vmethod of GstBaseTransform.

Removes no-used-variable warning for priv in when DBG is set

To get the out-caps, GstTensorFilter has to parse tensor info from NN model.

Definition at line 1528 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_get_property()

static void gst_tensor_filter_get_property	(	GObject *	object,
		guint	prop_id,
		GValue *	value,
		GParamSpec *	pspec
	)

static

Getter for tensor_filter properties.

Definition at line 330 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_get_tensor_size()

static gsize gst_tensor_filter_get_tensor_size	(	GstTensorFilter *	self,
		guint	index,
		gboolean	is_input
	)

static

Calculate tensor buffer size.

Parameters

self	"this" pointer
index	index of tensors

Returns: tensor buffer size

Definition at line 277 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_get_wrapped_mem()

static GstMemory* gst_tensor_filter_get_wrapped_mem	(	GstTensorFilter *	self,
		gpointer	data,
		gsize	size
	)

static

Allocate new memory block from given data.

tensor-filter should send event to sub-plugin when memory is freed.

Definition at line 372 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_init()

static void gst_tensor_filter_init ( GstTensorFilter * self )

static

initialize the new element instantiate pads and add them to element set pad callback functions initialize instance structure

Definition at line 234 of file tensor_filter.c.

Here is the call graph for this function:

◆ gst_tensor_filter_query()

static gboolean gst_tensor_filter_query	(	GstBaseTransform *	trans,
		GstPadDirection	direction,
		GstQuery *	query
	)

static

query handling, optional vmethod of GstBaseTransform.

Definition at line 1623 of file tensor_filter.c.

Here is the caller graph for this function:

◆ gst_tensor_filter_set_caps()

static gboolean gst_tensor_filter_set_caps	(	GstBaseTransform *	trans,
		GstCaps *	incaps,
		GstCaps *	outcaps
	)

static

set caps. required vmethod of GstBaseTransform.

compare output tensor

Definition at line 1563 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_set_property()

static void gst_tensor_filter_set_property	(	GObject *	object,
		guint	prop_id,
		const GValue *	value,
		GParamSpec *	pspec
	)

static

Setter for tensor_filter properties.

Definition at line 304 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_sink_event()

static gboolean gst_tensor_filter_sink_event	(	GstBaseTransform *	trans,
		GstEvent *	event
	)

static

Event handler for sink pad of tensor filter.

Parameters

trans	"this" pointer
event	a passed event object

Returns: TRUE if there is no error.

other events are handled in the default event handler

Definition at line 1723 of file tensor_filter.c.

Here is the caller graph for this function:

◆ gst_tensor_filter_src_event()

static gboolean gst_tensor_filter_src_event	(	GstBaseTransform *	trans,
		GstEvent *	event
	)

static

Event handler for src pad of tensor filter.

Parameters

trans	"this" pointer
event	a passed event object

Returns: TRUE if there is no error.

other events are handled in the default event handler

Definition at line 1764 of file tensor_filter.c.

Here is the caller graph for this function:

◆ gst_tensor_filter_start()

static gboolean gst_tensor_filter_start ( GstBaseTransform * trans )

static

Called when the element starts processing. optional vmethod of BaseTransform.

Parameters

trans "this" pointer

Returns: TRUE if there is no error.

Definition at line 1802 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_stop()

static gboolean gst_tensor_filter_stop ( GstBaseTransform * trans )

static

Called when the element stops processing. optional vmethod of BaseTransform.

Parameters

trans "this" pointer

Returns: TRUE if there is no error.

Definition at line 1836 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_transform()

static GstFlowReturn gst_tensor_filter_transform	(	GstBaseTransform *	trans,
		GstBuffer *	inbuf,
		GstBuffer *	outbuf
	)

static

non-ip transform. required vmethod of GstBaseTransform.

Reset suspend timeout

Set suspend timeout

Definition at line 1097 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_transform_caps()

static GstCaps * gst_tensor_filter_transform_caps	(	GstBaseTransform *	trans,
		GstPadDirection	direction,
		GstCaps *	caps,
		GstCaps *	filter
	)

static

configure tensor-srcpad cap from "proposed" cap.

@trans ("this" pointer) @direction (why do we need this?) @caps sinkpad cap (if direction GST_PAD_SINK) @filter this element's cap (don't know specifically.)

Be careful not to fix/set caps at this stage. Negotiation not completed yet.

GstTensorFilter has to parse the tensor dimension and type from NN model. In this stage, in-caps may not be fixed yet. To get the tensor info and generate pad-caps, call getInputDimension and getOutputDimension. If these functions are not defined, we have to call setInputDimension, and then it will fully configure the tensor info.

Todo:: how to set the framerate of output tensors

Todo:: We do not have a testcase hitting here. Thus, we do not ensure the validity here. However, according to gstreamer doxygen entry, if filter is given, that's not to be ignored. For now, we assume that if caps-size is 0, filter is "ANY".

Definition at line 1405 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_transform_size()

static gboolean gst_tensor_filter_transform_size	(	GstBaseTransform *	trans,
		GstPadDirection	direction,
		GstCaps *	caps,
		gsize	size,
		GstCaps *	othercaps,
		gsize *	othersize
	)

static

Tell the framework the required size of buffer based on the info of the other side pad. optional vmethod of BaseTransform.

We cannot directly get the value from size value, we need to review the pad-caps. This is called when non-ip mode is used.

Internal Logic Error. Cannot proceed without configured pipeline

Consider multi-tensors. Set each memory block in transform()

Definition at line 1694 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gst_tensor_filter_watchdog_trigger()

static gboolean gst_tensor_filter_watchdog_trigger ( gpointer ptr )

static

Called when there is no input within suspend time specified by the user.

Definition at line 1083 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ prepare_statistics()

static void prepare_statistics ( GstTensorFilterPrivate * priv )

static

Prepare statistics for performance profiling (e.g, latency, throughput)

Definition at line 388 of file tensor_filter.c.

Here is the caller graph for this function:

◆ record_statistics()

static void record_statistics ( GstTensorFilterPrivate * priv )

static

Record statistics for performance profiling (e.g, latency, throughput)

statistics values are monotonously increasing. to avoid potential overflow, let's cache old values and subtract them from the statistics if some threshold is exceeded.

Definition at line 412 of file tensor_filter.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ track_latency()

static void track_latency ( GstTensorFilter * self )

static

Track estimated latency and notify pipeline when it changes. Latency estimates may be a bit jittery. On the principle we want to inform pipeline with the latency from longest inference. However, first inference may take much longer, or model filter configuration may change. Therefore any change of more than 10% (arbitrary value) to a lower latency is also reported to pipeline. Notification is done sending LATENCY message to bus. Upon receipt, application will initiate a pipeline latency probe via LATENCY query.

Definition at line 503 of file tensor_filter.c.

Here is the caller graph for this function:

Variable Documentation

◆ sink_factory

GstStaticPadTemplate sink_factory

static

Initial value:

= GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS (CAPS_STRING))

The capabilities of the inputs.

Definition at line 92 of file tensor_filter.c.

◆ src_factory

GstStaticPadTemplate src_factory

static

Initial value:

= GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS (CAPS_STRING))

The capabilities of the outputs.

Definition at line 100 of file tensor_filter.c.

Classes

Macros

Typedefs

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ CAPS_STRING

◆ EVENT_NAME_UPDATE_MODEL

◆ GST_CAT_DEFAULT

◆ gst_tensor_filter_parent_class

◆ LATENCY_REPORT_HEADROOM

◆ LATENCY_REPORT_THRESHOLD

◆ TF_MODELNAME

◆ THRESHOLD_CACHE_OLD

◆ THRESHOLD_DROP_OLD

Typedef Documentation

◆ FilterTransformData

Function Documentation

◆ _gst_tensor_filter_convert_meta()

◆ _gst_tensor_filter_release_mem_until_idx()

◆ _gst_tensor_filter_transform_check_invoke_result()

◆ _gst_tensor_filter_transform_get_all_input_data()

◆ _gst_tensor_filter_transform_get_invoke_tensors()

◆ _gst_tensor_filter_transform_get_output_data()

◆ _gst_tensor_filter_transform_prepare_output_tensors()

◆ _gst_tensor_filter_transform_update_outbuf()

◆ _gst_tensor_filter_transform_validate()

◆ accumulate_latency()

◆ G_DEFINE_TYPE()

◆ GST_DEBUG_CATEGORY_STATIC()

◆ gst_tensor_filter_check_throttling_delay()

◆ gst_tensor_filter_class_init()

◆ gst_tensor_filter_configure_tensor()

◆ gst_tensor_filter_destroy_notify()

◆ gst_tensor_filter_finalize()

◆ gst_tensor_filter_fixate_caps()

◆ gst_tensor_filter_get_property()

◆ gst_tensor_filter_get_tensor_size()

◆ gst_tensor_filter_get_wrapped_mem()

◆ gst_tensor_filter_init()

◆ gst_tensor_filter_query()

◆ gst_tensor_filter_set_caps()

◆ gst_tensor_filter_set_property()

◆ gst_tensor_filter_sink_event()

◆ gst_tensor_filter_src_event()

◆ gst_tensor_filter_start()

◆ gst_tensor_filter_stop()

◆ gst_tensor_filter_transform()

◆ gst_tensor_filter_transform_caps()

◆ gst_tensor_filter_transform_size()

◆ gst_tensor_filter_watchdog_trigger()

◆ prepare_statistics()

◆ record_statistics()

◆ track_latency()

Variable Documentation

◆ sink_factory

◆ src_factory