Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * Copyright (C) 2021 Samsung Electronics Co., Ltd.
4 : *
5 : * @file gsttensor_sparseenc.c
6 : * @date 27 Jul 2021
7 : * @brief GStreamer element to encode dense tensors into sparse tensors
8 : * @see https://github.com/nnstreamer/nnstreamer
9 : * @author Yongjoo Ahn <yongjoo1.ahn@samsung.com>
10 : * @bug No known bugs except for NYI items
11 : */
12 :
13 : /**
14 : * SECTION:element-tensor_sparse_enc
15 : *
16 : * tensor_sparse_enc is a GStreamer element to encode incoming tensor into sparse format.
17 : *
18 : * The input is always in the format of other/tensors,format=static.
19 : * The output is always in the format of ohter/tensors,format=sparse.
20 : *
21 : * Please see also tensor_sparse_dec.
22 : *
23 : * <refsect2>
24 : * <title>Example launch line</title>
25 : * |[
26 : * gst-launch-1.0 ... ! other/tensors,format=static ! \
27 : * tensor_sparse_enc ! tensor_sink
28 : * ]|
29 : * </refsect2>
30 : */
31 :
32 : #ifdef HAVE_CONFIG_H
33 : #include <config.h>
34 : #endif
35 :
36 : #include <string.h>
37 : #include <nnstreamer_util.h>
38 : #include "gsttensor_sparseenc.h"
39 :
40 : /**
41 : * @brief Macro for debug mode.
42 : */
43 : #ifndef DBG
44 : #define DBG (!self->silent)
45 : #endif
46 :
47 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_sparse_enc_debug);
48 : #define GST_CAT_DEFAULT gst_tensor_sparse_enc_debug
49 :
50 : /**
51 : * @brief tensor_sparse_enc properties
52 : */
53 : enum
54 : {
55 : PROP_0,
56 : PROP_SILENT
57 : };
58 :
59 : /**
60 : * @brief Flag to print minimized log.
61 : */
62 : #define DEFAULT_SILENT TRUE
63 :
64 : /**
65 : * @brief Template for sink pad.
66 : */
67 : static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
68 : GST_PAD_SINK,
69 : GST_PAD_ALWAYS,
70 : GST_STATIC_CAPS (GST_TENSORS_CAP_DEFAULT));
71 :
72 : /**
73 : * @brief Template for src pad.
74 : */
75 : static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
76 : GST_PAD_SRC,
77 : GST_PAD_ALWAYS,
78 : GST_STATIC_CAPS (GST_TENSORS_SPARSE_CAP_DEFAULT));
79 :
80 : #define gst_tensor_sparse_enc_parent_class parent_class
81 1083 : G_DEFINE_TYPE (GstTensorSparseEnc, gst_tensor_sparse_enc, GST_TYPE_ELEMENT);
82 :
83 : static void gst_tensor_sparse_enc_finalize (GObject * object);
84 : static void gst_tensor_sparse_enc_set_property (GObject * object, guint prop_id,
85 : const GValue * value, GParamSpec * pspec);
86 : static void gst_tensor_sparse_enc_get_property (GObject * object, guint prop_id,
87 : GValue * value, GParamSpec * pspec);
88 : static GstFlowReturn
89 : gst_tensor_sparse_enc_chain (GstPad * pad, GstObject * parent, GstBuffer * buf);
90 : static GstCaps *gst_tensor_sparse_enc_query_caps (GstTensorSparseEnc * self,
91 : GstPad * pad, GstCaps * filter);
92 : static gboolean gst_tensor_sparse_enc_sink_event (GstPad * pad,
93 : GstObject * parent, GstEvent * event);
94 : static gboolean gst_tensor_sparse_enc_sink_query (GstPad * pad,
95 : GstObject * parent, GstQuery * query);
96 :
97 : /**
98 : * @brief Initialize the tensor_sparse's class.
99 : */
100 : static void
101 29 : gst_tensor_sparse_enc_class_init (GstTensorSparseEncClass * klass)
102 : {
103 : GObjectClass *object_class;
104 : GstElementClass *element_class;
105 :
106 29 : GST_DEBUG_CATEGORY_INIT (gst_tensor_sparse_enc_debug, "tensor_sparse_enc", 0,
107 : "Element to encode sparse tensors");
108 :
109 29 : object_class = (GObjectClass *) klass;
110 29 : element_class = (GstElementClass *) klass;
111 :
112 29 : object_class->set_property = gst_tensor_sparse_enc_set_property;
113 29 : object_class->get_property = gst_tensor_sparse_enc_get_property;
114 29 : object_class->finalize = gst_tensor_sparse_enc_finalize;
115 :
116 : /**
117 : * GstTensorSparseEnc::silent:
118 : *
119 : * The flag to enable/disable debugging messages.
120 : */
121 29 : g_object_class_install_property (object_class, PROP_SILENT,
122 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
123 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
124 :
125 29 : gst_element_class_add_pad_template (element_class,
126 : gst_static_pad_template_get (&src_template));
127 :
128 29 : gst_element_class_add_pad_template (element_class,
129 : gst_static_pad_template_get (&sink_template));
130 :
131 29 : gst_element_class_set_static_metadata (element_class,
132 : "TensorSparseEnc",
133 : "Filter/Tensor",
134 : "Element to encode dense tensors into sparse tensors",
135 : "Samsung Electronics Co., Ltd.");
136 29 : }
137 :
138 : /**
139 : * @brief Initialize tensor_sparse_enc element.
140 : */
141 : static void
142 10 : gst_tensor_sparse_enc_init (GstTensorSparseEnc * self)
143 : {
144 : /* setup sink pad */
145 10 : self->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink");
146 10 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
147 :
148 : /* setup src pad */
149 10 : self->srcpad = gst_pad_new_from_static_template (&src_template, "src");
150 10 : gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
151 :
152 : /* setup chain function */
153 10 : gst_pad_set_chain_function (self->sinkpad,
154 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_enc_chain));
155 :
156 : /* setup event function */
157 10 : gst_pad_set_event_function (self->sinkpad,
158 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_enc_sink_event));
159 :
160 10 : gst_pad_set_query_function (self->sinkpad,
161 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_enc_sink_query));
162 :
163 : /* init properties */
164 10 : self->silent = DEFAULT_SILENT;
165 10 : gst_tensors_config_init (&self->in_config);
166 10 : }
167 :
168 : /**
169 : * @brief Function to finalize instance.
170 : */
171 : static void
172 10 : gst_tensor_sparse_enc_finalize (GObject * object)
173 : {
174 : GstTensorSparseEnc *self;
175 10 : self = GST_TENSOR_SPARSE_ENC (object);
176 :
177 10 : gst_tensors_config_free (&self->in_config);
178 :
179 10 : G_OBJECT_CLASS (parent_class)->finalize (object);
180 10 : }
181 :
182 : /**
183 : * @brief Setter for tensor_sparse_enc properties.
184 : */
185 : static void
186 1 : gst_tensor_sparse_enc_set_property (GObject * object, guint prop_id,
187 : const GValue * value, GParamSpec * pspec)
188 : {
189 : GstTensorSparseEnc *self;
190 :
191 1 : self = GST_TENSOR_SPARSE_ENC (object);
192 :
193 1 : switch (prop_id) {
194 1 : case PROP_SILENT:
195 1 : self->silent = g_value_get_boolean (value);
196 1 : break;
197 0 : default:
198 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
199 0 : break;
200 : }
201 1 : }
202 :
203 : /**
204 : * @brief Getter for tensor_sparse_enc properties.
205 : */
206 : static void
207 2 : gst_tensor_sparse_enc_get_property (GObject * object, guint prop_id,
208 : GValue * value, GParamSpec * pspec)
209 : {
210 : GstTensorSparseEnc *self;
211 :
212 2 : self = GST_TENSOR_SPARSE_ENC (object);
213 :
214 2 : switch (prop_id) {
215 2 : case PROP_SILENT:
216 2 : g_value_set_boolean (value, self->silent);
217 2 : break;
218 0 : default:
219 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
220 0 : break;
221 : }
222 2 : }
223 :
224 : /**
225 : * @brief Parse caps and set tensors config
226 : */
227 : static gboolean
228 8 : gst_tensor_sparse_enc_parse_caps (GstTensorSparseEnc * self,
229 : const GstCaps * caps)
230 : {
231 : GstStructure *structure;
232 : GstTensorsConfig config;
233 :
234 16 : g_return_val_if_fail (caps != NULL, FALSE);
235 8 : g_return_val_if_fail (gst_caps_is_fixed (caps), FALSE);
236 :
237 8 : structure = gst_caps_get_structure (caps, 0);
238 :
239 8 : if (!gst_tensors_config_from_structure (&config, structure) ||
240 8 : !gst_tensors_config_validate (&config)) {
241 : /** not fully configured */
242 0 : GST_ERROR_OBJECT (self, "Failed to configure tensors config.\n");
243 0 : return FALSE;
244 : }
245 :
246 8 : self->in_config = config;
247 8 : return TRUE;
248 : }
249 :
250 : /**
251 : * @brief This function handles sink pad event.
252 : */
253 : static gboolean
254 33 : gst_tensor_sparse_enc_sink_event (GstPad * pad, GstObject * parent,
255 : GstEvent * event)
256 : {
257 : GstTensorSparseEnc *self;
258 33 : self = GST_TENSOR_SPARSE_ENC (parent);
259 :
260 33 : g_return_val_if_fail (event != NULL, FALSE);
261 :
262 33 : switch (GST_EVENT_TYPE (event)) {
263 8 : case GST_EVENT_CAPS:
264 : {
265 : GstCaps *caps;
266 : gboolean ret;
267 :
268 8 : gst_event_parse_caps (event, &caps);
269 8 : silent_debug_caps (self, caps, "caps");
270 :
271 8 : ret = gst_tensor_sparse_enc_parse_caps (self, caps);
272 8 : gst_event_unref (event);
273 8 : return ret;
274 : }
275 25 : default:
276 25 : break;
277 : }
278 :
279 25 : return gst_pad_event_default (pad, parent, event);
280 : }
281 :
282 : /**
283 : * @brief Get pad caps for caps negotiation.
284 : */
285 : static GstCaps *
286 48 : gst_tensor_sparse_enc_query_caps (GstTensorSparseEnc * self, GstPad * pad,
287 : GstCaps * filter)
288 : {
289 : GstCaps *caps;
290 :
291 48 : caps = gst_pad_get_current_caps (pad);
292 48 : if (!caps) {
293 : /** pad don't have current caps. use the template caps */
294 48 : caps = gst_pad_get_pad_template_caps (pad);
295 : }
296 :
297 48 : silent_debug_caps (self, caps, "caps");
298 48 : silent_debug_caps (self, filter, "filter");
299 :
300 48 : if (filter) {
301 : GstCaps *intersection;
302 : intersection =
303 7 : gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
304 :
305 7 : gst_caps_unref (caps);
306 7 : caps = intersection;
307 : }
308 :
309 48 : silent_debug_caps (self, caps, "result");
310 48 : return caps;
311 : }
312 :
313 : /**
314 : * @brief This function handles sink pad query.
315 : */
316 : static gboolean
317 69 : gst_tensor_sparse_enc_sink_query (GstPad * pad, GstObject * parent,
318 : GstQuery * query)
319 : {
320 : GstTensorSparseEnc *self;
321 :
322 69 : self = GST_TENSOR_SPARSE_ENC (parent);
323 :
324 69 : GST_DEBUG_OBJECT (self, "Received %s query: %" GST_PTR_FORMAT,
325 : GST_QUERY_TYPE_NAME (query), query);
326 :
327 69 : switch (GST_QUERY_TYPE (query)) {
328 48 : case GST_QUERY_CAPS:
329 : {
330 : GstCaps *caps;
331 : GstCaps *filter;
332 :
333 48 : gst_query_parse_caps (query, &filter);
334 48 : caps = gst_tensor_sparse_enc_query_caps (self, pad, filter);
335 48 : silent_debug_caps (self, filter, "filter");
336 48 : silent_debug_caps (self, caps, "caps");
337 48 : gst_query_set_caps_result (query, caps);
338 48 : gst_caps_unref (caps);
339 48 : return TRUE;
340 : }
341 16 : case GST_QUERY_ACCEPT_CAPS:
342 : {
343 : GstCaps *caps;
344 : GstCaps *template_caps;
345 16 : gboolean res = FALSE;
346 :
347 16 : gst_query_parse_accept_caps (query, &caps);
348 16 : silent_debug_caps (self, caps, "caps");
349 :
350 16 : if (gst_caps_is_fixed (caps)) {
351 16 : template_caps = gst_pad_get_pad_template_caps (pad);
352 :
353 16 : res = gst_caps_can_intersect (template_caps, caps);
354 16 : gst_caps_unref (template_caps);
355 : }
356 :
357 16 : gst_query_set_accept_caps_result (query, res);
358 16 : return TRUE;
359 : }
360 5 : default:
361 5 : break;
362 : }
363 :
364 5 : return gst_pad_query_default (pad, parent, query);
365 : }
366 :
367 :
368 : /**
369 : * @brief Internal function to transform the input buffer.
370 : */
371 : static GstFlowReturn
372 35 : gst_tensor_sparse_enc_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
373 : {
374 35 : GstFlowReturn ret = GST_FLOW_OK;
375 35 : GstTensorSparseEnc *self = GST_TENSOR_SPARSE_ENC (parent);
376 : GstMemory *in_mem, *out_mem;
377 : GstBuffer *outbuf;
378 : GstTensorsInfo *info;
379 : GstTensorInfo *_info;
380 : guint i;
381 :
382 : UNUSED (pad);
383 :
384 35 : info = &self->in_config.info;
385 35 : buf = gst_tensor_buffer_from_config (buf, &self->in_config);
386 35 : outbuf = gst_buffer_new ();
387 :
388 101 : for (i = 0; i < info->num_tensors; ++i) {
389 : GstTensorMetaInfo meta;
390 :
391 66 : _info = gst_tensors_info_get_nth_info (info, i);
392 66 : gst_tensor_info_convert_to_meta (_info, &meta);
393 :
394 66 : meta.format = _NNS_TENSOR_FORMAT_SPARSE;
395 66 : meta.media_type = _NNS_TENSOR;
396 :
397 : /* do real encoding here */
398 66 : in_mem = gst_tensor_buffer_get_nth_memory (buf, i);
399 66 : out_mem = gst_tensor_sparse_from_dense (&meta, in_mem);
400 66 : gst_memory_unref (in_mem);
401 :
402 66 : if (!out_mem) {
403 0 : nns_loge ("failed to convert to sparse tensor");
404 0 : ret = GST_FLOW_ERROR;
405 0 : goto done;
406 : }
407 :
408 66 : gst_tensor_buffer_append_memory (outbuf, out_mem, _info);
409 : }
410 :
411 35 : ret = gst_pad_push (self->srcpad, outbuf);
412 :
413 35 : done:
414 35 : gst_buffer_unref (buf);
415 35 : if (ret != GST_FLOW_OK)
416 0 : gst_buffer_unref (outbuf);
417 :
418 35 : return ret;
419 : }
|
