Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * Copyright (C) 2021 Samsung Electronics Co., Ltd.
4 : *
5 : * @file gsttensor_crop.c
6 : * @date 10 May 2021
7 : * @brief GStreamer element to crop the regions of incoming tensor
8 : * @see https://github.com/nnstreamer/nnstreamer
9 : * @author Jaeyun Jung <jy1210.jung@samsung.com>
10 : * @bug No known bugs except for NYI items
11 : */
12 :
13 : /**
14 : * SECTION:element-tensor_crop
15 : *
16 : * tensor_crop is a GStreamer element to crop the regions of incoming tensor.
17 : *
18 : * tensor_crop has two always sink pads - raw and info.
19 : * The raw pad accepts tensor (other/tensor) which will be cropped with crop info.
20 : * The info pad has capability for flexible tensor stream (other/tensors-flexible), that can have a various buffer size for crop info.
21 : * Incoming buffer on info pad should be an array of crop info.
22 : * Note that NNStreamer supports maximum NNS_TENSOR_SIZE_LIMIT memory blocks in a buffer.
23 : * So, when incoming buffer on info pad has more than NNS_TENSOR_SIZE_LIMIT crop-info array, tensor_crop will ignore the data.
24 : *
25 : * The output is always in the format of other/tensors-flexible.
26 : *
27 : * <refsect2>
28 : * <title>Example launch line</title>
29 : * |[
30 : * gst-launch-1.0 tensor_crop name=crop ! (cropped tensors) ... \
31 : * videotestsrc ! videoconvert ! video/x-raw,format=RGB ! tensor_converter ! tee name=t \
32 : * t. ! queue ! crop.raw \
33 : * t. ! queue ! (process raw video tensor and push buffer which includes crop info) ! crop.info
34 : * ]|
35 : * </refsect2>
36 : */
37 :
38 : #ifdef HAVE_CONFIG_H
39 : #include <config.h>
40 : #endif
41 :
42 : #include <string.h>
43 : #include <nnstreamer_util.h>
44 : #include "gsttensor_crop.h"
45 : #include "tensor_data.h"
46 :
47 : /**
48 : * @brief Internal data structure to describe tensor region.
49 : */
50 : typedef struct
51 : {
52 : guint x;
53 : guint y;
54 : guint w;
55 : guint h;
56 : } tensor_region_s;
57 :
58 : /**
59 : * @brief Internal data structure to describe cropping tensor data.
60 : * @todo Add various mode to crop tensor. Now tensor-crop handles NHWC data format only.
61 : */
62 : typedef struct
63 : {
64 : guint num;
65 : tensor_region_s region[NNS_TENSOR_SIZE_LIMIT];
66 : } tensor_crop_info_s;
67 :
68 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_crop_debug);
69 : #define GST_CAT_DEFAULT gst_tensor_crop_debug
70 :
71 : /**
72 : * @brief tensor_crop properties
73 : */
74 : enum
75 : {
76 : PROP_0,
77 : PROP_LATENESS,
78 : PROP_SILENT
79 : };
80 :
81 : /**
82 : * @brief Flag to print minimized log.
83 : */
84 : #define DEFAULT_SILENT TRUE
85 :
86 : /**
87 : * @brief Default value to compare timestamp of raw and info buffer, in milliseconds (-1 means no synchronization).
88 : */
89 : #define DEFAULT_LATENESS (-1)
90 :
91 : /**
92 : * @brief Template for sink pad (raw data).
93 : */
94 : static GstStaticPadTemplate raw_template = GST_STATIC_PAD_TEMPLATE ("raw",
95 : GST_PAD_SINK,
96 : GST_PAD_ALWAYS,
97 : GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT ";"
98 : GST_TENSORS_CAP_MAKE ("{ static, flexible }")));
99 :
100 : /**
101 : * @brief Template for sink pad (crop info).
102 : */
103 : static GstStaticPadTemplate info_template = GST_STATIC_PAD_TEMPLATE ("info",
104 : GST_PAD_SINK,
105 : GST_PAD_ALWAYS,
106 : GST_STATIC_CAPS (GST_TENSORS_FLEX_CAP_DEFAULT));
107 :
108 : /**
109 : * @brief Template for src pad.
110 : */
111 : static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
112 : GST_PAD_SRC,
113 : GST_PAD_ALWAYS,
114 : GST_STATIC_CAPS (GST_TENSORS_FLEX_CAP_DEFAULT));
115 :
116 : #define gst_tensor_crop_parent_class parent_class
117 1011 : G_DEFINE_TYPE (GstTensorCrop, gst_tensor_crop, GST_TYPE_ELEMENT);
118 :
119 : static void gst_tensor_crop_finalize (GObject * object);
120 : static void gst_tensor_crop_set_property (GObject * object, guint prop_id,
121 : const GValue * value, GParamSpec * pspec);
122 : static void gst_tensor_crop_get_property (GObject * object, guint prop_id,
123 : GValue * value, GParamSpec * pspec);
124 : static GstStateChangeReturn gst_tensor_crop_change_state (GstElement * element,
125 : GstStateChange transition);
126 : static gboolean gst_tensor_crop_src_event (GstPad * pad, GstObject * parent,
127 : GstEvent * event);
128 : static gboolean gst_tensor_crop_sink_event (GstCollectPads * pads,
129 : GstCollectData * data, GstEvent * event, gpointer user_data);
130 : static GstFlowReturn gst_tensor_crop_collected (GstCollectPads * pads,
131 : gpointer user_data);
132 :
133 : /**
134 : * @brief Initialize the tensor_crop's class.
135 : */
136 : static void
137 22 : gst_tensor_crop_class_init (GstTensorCropClass * klass)
138 : {
139 : GObjectClass *object_class;
140 : GstElementClass *element_class;
141 :
142 22 : GST_DEBUG_CATEGORY_INIT (gst_tensor_crop_debug, "tensor_crop", 0,
143 : "Element to crop the regions of incoming tensor");
144 :
145 22 : object_class = (GObjectClass *) klass;
146 22 : element_class = (GstElementClass *) klass;
147 :
148 22 : object_class->set_property = gst_tensor_crop_set_property;
149 22 : object_class->get_property = gst_tensor_crop_get_property;
150 22 : object_class->finalize = gst_tensor_crop_finalize;
151 :
152 : /**
153 : * GstTensorCrop::lateness:
154 : *
155 : * The time difference between raw and info buffer, in milliseconds (-1 means no synchronization).
156 : * If raw and info buffers on the pads have different timestamp and time-diff is larger than 'lateness',
157 : * tensor-crop will drop old buffer and wait for next buffers.
158 : */
159 22 : g_object_class_install_property (object_class, PROP_LATENESS,
160 : g_param_spec_int ("lateness", "Lateness",
161 : "The time difference between raw and info buffer in milliseconds",
162 : -1, G_MAXINT, DEFAULT_LATENESS,
163 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
164 :
165 : /**
166 : * GstTensorCrop::silent:
167 : *
168 : * The flag to enable/disable debugging messages.
169 : */
170 22 : g_object_class_install_property (object_class, PROP_SILENT,
171 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
172 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
173 :
174 22 : element_class->change_state =
175 22 : GST_DEBUG_FUNCPTR (gst_tensor_crop_change_state);
176 :
177 22 : gst_element_class_add_pad_template (element_class,
178 : gst_static_pad_template_get (&src_template));
179 22 : gst_element_class_add_pad_template (element_class,
180 : gst_static_pad_template_get (&raw_template));
181 22 : gst_element_class_add_pad_template (element_class,
182 : gst_static_pad_template_get (&info_template));
183 :
184 22 : gst_element_class_set_static_metadata (element_class,
185 : "TensorCrop",
186 : "Filter/Tensor",
187 : "Element to crop the regions of incoming tensor",
188 : "Samsung Electronics Co., Ltd.");
189 22 : }
190 :
191 : /**
192 : * @brief Clear and reset old pad data.
193 : */
194 : static void
195 32 : gst_tensor_crop_pad_reset (GstTensorCropPadData * cpad)
196 : {
197 32 : gst_tensors_config_free (&cpad->config);
198 32 : gst_tensors_config_init (&cpad->config);
199 32 : }
200 :
201 : /**
202 : * @brief Clear and reset old data in tensor_crop.
203 : */
204 : static void
205 16 : gst_tensor_crop_reset (GstTensorCrop * self)
206 : {
207 : GstTensorCropPadData *cpad;
208 : GSList *walk;
209 :
210 16 : if (self->collect) {
211 16 : walk = self->collect->data;
212 :
213 48 : while (walk) {
214 32 : cpad = (GstTensorCropPadData *) walk->data;
215 :
216 32 : gst_tensor_crop_pad_reset (cpad);
217 32 : walk = g_slist_next (walk);
218 : }
219 : }
220 :
221 16 : self->send_stream_start = TRUE;
222 16 : }
223 :
224 : /**
225 : * @brief Initialize tensor_crop element.
226 : */
227 : static void
228 8 : gst_tensor_crop_init (GstTensorCrop * self)
229 : {
230 : /* setup sink pad */
231 8 : self->sinkpad_raw = gst_pad_new_from_static_template (&raw_template, "raw");
232 8 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad_raw);
233 :
234 8 : self->sinkpad_info =
235 8 : gst_pad_new_from_static_template (&info_template, "info");
236 8 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad_info);
237 :
238 8 : self->collect = gst_collect_pads_new ();
239 8 : gst_collect_pads_set_function (self->collect,
240 8 : GST_DEBUG_FUNCPTR (gst_tensor_crop_collected), self);
241 8 : gst_collect_pads_set_event_function (self->collect,
242 8 : GST_DEBUG_FUNCPTR (gst_tensor_crop_sink_event), self);
243 :
244 8 : gst_collect_pads_add_pad (self->collect, self->sinkpad_raw,
245 : sizeof (GstTensorCropPadData), NULL, TRUE);
246 8 : gst_collect_pads_add_pad (self->collect, self->sinkpad_info,
247 : sizeof (GstTensorCropPadData), NULL, TRUE);
248 :
249 : /* setup src pad */
250 8 : self->srcpad = gst_pad_new_from_static_template (&src_template, "src");
251 8 : gst_pad_set_event_function (self->srcpad,
252 : GST_DEBUG_FUNCPTR (gst_tensor_crop_src_event));
253 8 : gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
254 :
255 : /* init properties */
256 8 : self->lateness = DEFAULT_LATENESS;
257 8 : self->silent = DEFAULT_SILENT;
258 8 : self->send_stream_start = TRUE;
259 8 : }
260 :
261 : /**
262 : * @brief Function to finalize instance.
263 : */
264 : static void
265 8 : gst_tensor_crop_finalize (GObject * object)
266 : {
267 : GstTensorCrop *self;
268 :
269 8 : self = GST_TENSOR_CROP (object);
270 :
271 8 : gst_tensor_crop_reset (self);
272 :
273 8 : if (self->collect) {
274 8 : gst_object_unref (self->collect);
275 8 : self->collect = NULL;
276 : }
277 :
278 8 : G_OBJECT_CLASS (parent_class)->finalize (object);
279 8 : }
280 :
281 : /**
282 : * @brief Setter for tensor_crop properties.
283 : */
284 : static void
285 3 : gst_tensor_crop_set_property (GObject * object, guint prop_id,
286 : const GValue * value, GParamSpec * pspec)
287 : {
288 : GstTensorCrop *self;
289 :
290 3 : self = GST_TENSOR_CROP (object);
291 :
292 3 : switch (prop_id) {
293 2 : case PROP_LATENESS:
294 2 : self->lateness = g_value_get_int (value);
295 2 : break;
296 1 : case PROP_SILENT:
297 1 : self->silent = g_value_get_boolean (value);
298 1 : break;
299 0 : default:
300 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
301 0 : break;
302 : }
303 3 : }
304 :
305 : /**
306 : * @brief Getter for tensor_crop properties.
307 : */
308 : static void
309 4 : gst_tensor_crop_get_property (GObject * object, guint prop_id,
310 : GValue * value, GParamSpec * pspec)
311 : {
312 : GstTensorCrop *self;
313 :
314 4 : self = GST_TENSOR_CROP (object);
315 :
316 4 : switch (prop_id) {
317 2 : case PROP_LATENESS:
318 2 : g_value_set_int (value, self->lateness);
319 2 : break;
320 2 : case PROP_SILENT:
321 2 : g_value_set_boolean (value, self->silent);
322 2 : break;
323 0 : default:
324 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
325 0 : break;
326 : }
327 4 : }
328 :
329 : /**
330 : * @brief Handle state transition.
331 : */
332 : static GstStateChangeReturn
333 62 : gst_tensor_crop_change_state (GstElement * element, GstStateChange transition)
334 : {
335 : GstTensorCrop *self;
336 : GstStateChangeReturn ret;
337 :
338 62 : self = GST_TENSOR_CROP (element);
339 :
340 62 : switch (transition) {
341 8 : case GST_STATE_CHANGE_READY_TO_PAUSED:
342 8 : gst_collect_pads_start (self->collect);
343 8 : break;
344 8 : case GST_STATE_CHANGE_PAUSED_TO_READY:
345 8 : gst_collect_pads_stop (self->collect);
346 8 : break;
347 46 : default:
348 46 : break;
349 : }
350 :
351 62 : ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
352 :
353 62 : switch (transition) {
354 8 : case GST_STATE_CHANGE_PAUSED_TO_READY:
355 8 : gst_tensor_crop_reset (self);
356 8 : break;
357 54 : default:
358 54 : break;
359 : }
360 :
361 62 : return ret;
362 : }
363 :
364 : /**
365 : * @brief Handle event on src pad.
366 : */
367 : static gboolean
368 3 : gst_tensor_crop_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
369 : {
370 3 : g_return_val_if_fail (event != NULL, FALSE);
371 :
372 3 : switch (GST_EVENT_TYPE (event)) {
373 1 : case GST_EVENT_SEEK:
374 : /* disable seeking */
375 1 : gst_event_unref (event);
376 1 : return FALSE;
377 2 : default:
378 2 : break;
379 : }
380 :
381 2 : return gst_pad_event_default (pad, parent, event);
382 : }
383 :
384 : /**
385 : * @brief Handle event on sink pad.
386 : */
387 : static gboolean
388 60 : gst_tensor_crop_sink_event (GstCollectPads * pads, GstCollectData * data,
389 : GstEvent * event, gpointer user_data)
390 : {
391 : GstTensorCropPadData *cpad;
392 : UNUSED (user_data);
393 :
394 60 : g_return_val_if_fail (event != NULL, FALSE);
395 :
396 60 : cpad = (GstTensorCropPadData *) data;
397 :
398 60 : switch (GST_EVENT_TYPE (event)) {
399 15 : case GST_EVENT_CAPS:
400 : {
401 : GstCaps *caps;
402 : GstStructure *structure;
403 :
404 15 : gst_event_parse_caps (event, &caps);
405 15 : structure = gst_caps_get_structure (caps, 0);
406 :
407 15 : gst_tensors_config_from_structure (&cpad->config, structure);
408 :
409 15 : gst_event_unref (event);
410 15 : return gst_tensors_config_validate (&cpad->config);
411 : }
412 45 : default:
413 45 : break;
414 : }
415 :
416 45 : return gst_collect_pads_event_default (pads, data, event, FALSE);
417 : }
418 :
419 : /**
420 : * @brief Set pad caps if not negotiated.
421 : */
422 : static GstFlowReturn
423 10 : gst_tensor_crop_negotiate (GstTensorCrop * self)
424 : {
425 10 : if (!gst_pad_has_current_caps (self->sinkpad_raw)) {
426 0 : GST_ERROR_OBJECT (self,
427 : "The raw pad of tensor_crop '%s' does not have pad caps.",
428 : GST_ELEMENT_NAME (self));
429 0 : return GST_FLOW_NOT_NEGOTIATED;
430 : }
431 :
432 10 : if (!gst_pad_has_current_caps (self->sinkpad_info)) {
433 0 : GST_ERROR_OBJECT (self,
434 : "The info pad of tensor_crop '%s' does not have pad caps.",
435 : GST_ELEMENT_NAME (self));
436 0 : return GST_FLOW_NOT_NEGOTIATED;
437 : }
438 :
439 10 : if (!gst_pad_has_current_caps (self->srcpad)) {
440 : GstCaps *caps;
441 : GstSegment segment;
442 : GstTensorsConfig config;
443 : GstTensorCropPadData *cpad;
444 : GSList *walk;
445 :
446 6 : if (self->send_stream_start) {
447 : /**
448 : * Cannot use gst-pad util to get stream ID (multiple sink pads).
449 : * Create stream ID using first sink pad.
450 : */
451 12 : g_autofree gchar *sink_sid = gst_pad_get_stream_id (self->sinkpad_raw);
452 12 : g_autofree gchar *element_name = gst_element_get_name (self);
453 12 : g_autofree gchar *pad_name = gst_pad_get_name (self->srcpad);
454 18 : g_autofree gchar *sid = g_strdup_printf ("%s-%s-nnscrop-%s-%08x",
455 6 : GST_STR_NULL (sink_sid), element_name, pad_name, g_random_int ());
456 :
457 6 : gst_pad_push_event (self->srcpad, gst_event_new_stream_start (sid));
458 6 : self->send_stream_start = FALSE;
459 : }
460 :
461 : /**
462 : * Get config from collect-pads and set framerate.
463 : * Output is always flexible tensor.
464 : */
465 6 : gst_tensors_config_init (&config);
466 6 : config.info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
467 :
468 6 : walk = self->collect->data;
469 18 : while (walk) {
470 12 : cpad = (GstTensorCropPadData *) walk->data;
471 :
472 12 : if (config.rate_n < 0 ||
473 6 : gst_util_fraction_compare (cpad->config.rate_n, cpad->config.rate_d,
474 : config.rate_n, config.rate_d) < 0) {
475 6 : config.rate_n = cpad->config.rate_n;
476 6 : config.rate_d = cpad->config.rate_d;
477 : }
478 :
479 12 : walk = g_slist_next (walk);
480 : }
481 :
482 6 : caps = gst_tensors_caps_from_config (&config);
483 6 : gst_pad_set_caps (self->srcpad, caps);
484 6 : gst_caps_unref (caps);
485 :
486 6 : gst_segment_init (&segment, GST_FORMAT_TIME);
487 6 : gst_pad_push_event (self->srcpad, gst_event_new_segment (&segment));
488 : }
489 :
490 10 : return GST_FLOW_OK;
491 : }
492 :
493 : /**
494 : * @brief Internal function to prepare output meta info.
495 : */
496 : static gboolean
497 6 : gst_tensor_crop_prepare_out_meta (GstTensorCrop * self, gpointer buffer,
498 : GstTensorMetaInfo * meta, GstTensorInfo * info, gboolean * is_flexible)
499 : {
500 : GstCaps *caps;
501 : GstStructure *structure;
502 : GstTensorsConfig config;
503 : GstTensorInfo *_info;
504 6 : gboolean ret = FALSE;
505 :
506 6 : gst_tensor_meta_info_init (meta);
507 6 : gst_tensor_info_init (info);
508 :
509 6 : caps = gst_pad_get_current_caps (self->sinkpad_raw);
510 6 : structure = gst_caps_get_structure (caps, 0);
511 :
512 6 : if (!gst_tensors_config_from_structure (&config, structure)) {
513 0 : GST_ERROR_OBJECT (self, "Failed to get the config from caps.");
514 0 : goto done;
515 : }
516 :
517 : /**
518 : * @note tensor-crop handles single tensor. Parse first one.
519 : */
520 6 : _info = gst_tensors_info_get_nth_info (&config.info, 0);
521 6 : *is_flexible = gst_tensors_config_is_flexible (&config);
522 :
523 6 : if (*is_flexible) {
524 : /* meta from buffer */
525 1 : if (gst_tensor_meta_info_parse_header (meta, buffer)) {
526 1 : ret = gst_tensor_meta_info_convert (meta, info);
527 : }
528 : } else {
529 : /* meta from caps */
530 5 : ret = gst_tensor_info_convert_to_meta (_info, meta);
531 5 : gst_tensor_info_copy (info, _info);
532 : }
533 :
534 : /* output is flex tensor */
535 6 : meta->format = _NNS_TENSOR_FORMAT_FLEXIBLE;
536 :
537 6 : done:
538 6 : gst_caps_unref (caps);
539 6 : gst_tensors_config_free (&config);
540 6 : return ret;
541 : }
542 :
543 : /**
544 : * @brief Internal function to parse buffer and fill crop info.
545 : */
546 : static gboolean
547 7 : gst_tensor_crop_get_crop_info (GstTensorCrop * self, GstBuffer * info,
548 : tensor_crop_info_s * cinfo)
549 : {
550 : GstMemory *mem;
551 : GstMapInfo map;
552 : GstTensorMetaInfo meta;
553 : gsize hsize, dsize, esize;
554 : guint i, j;
555 : guint8 *pos, *src, *desc;
556 7 : gboolean ret = FALSE;
557 :
558 7 : i = gst_tensor_buffer_get_count (info);
559 7 : g_assert (i > 0);
560 7 : if (i > 1) {
561 0 : GST_WARNING_OBJECT (self,
562 : "Info buffer has %u memories, parse first one.", i);
563 : }
564 :
565 7 : mem = gst_tensor_buffer_get_nth_memory (info, 0);
566 7 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
567 0 : GST_ERROR_OBJECT (self, "Failed to map the info buffer.");
568 0 : goto done;
569 : }
570 :
571 : /* parse crop-info from flex tensor */
572 7 : if (!gst_tensor_meta_info_parse_header (&meta, map.data)) {
573 0 : GST_ERROR_OBJECT (self, "Failed to get the meta from info buffer.");
574 0 : goto done;
575 : }
576 :
577 7 : hsize = gst_tensor_meta_info_get_header_size (&meta);
578 7 : dsize = gst_tensor_meta_info_get_data_size (&meta);
579 7 : esize = gst_tensor_get_element_size (meta.type);
580 :
581 7 : if (hsize + dsize != map.size) {
582 1 : GST_ERROR_OBJECT (self,
583 : "Invalid meta info, info buffer size is incorrect (received %zd, expected %zd).",
584 : map.size, hsize + dsize);
585 1 : goto done;
586 : }
587 :
588 : /**
589 : * @todo Add various mode to crop tensor.
590 : * Now tensor-crop handles NHWC data format only.
591 : */
592 6 : g_assert ((dsize % (esize * 4)) == 0);
593 :
594 6 : memset (cinfo, 0, sizeof (tensor_crop_info_s));
595 :
596 6 : cinfo->num = MIN (dsize / (esize * 4), NNS_TENSOR_SIZE_LIMIT);
597 :
598 17 : for (i = 0; i < cinfo->num; i++) {
599 11 : pos = map.data + hsize + (esize * 4 * i);
600 :
601 55 : for (j = 0; j < 4; j++) {
602 44 : src = pos + (esize * j);
603 44 : desc = (guint8 *) (&cinfo->region[i]) + sizeof (guint) * j;
604 :
605 44 : gst_tensor_data_raw_typecast (src, meta.type, desc, _NNS_UINT32);
606 : }
607 : }
608 :
609 6 : ret = TRUE;
610 :
611 7 : done:
612 7 : if (mem) {
613 7 : gst_memory_unmap (mem, &map);
614 7 : gst_memory_unref (mem);
615 : }
616 :
617 7 : return ret;
618 : }
619 :
620 : /**
621 : * @brief Internal function to crop incoming buffer.
622 : */
623 : static GstBuffer *
624 6 : gst_tensor_crop_do_cropping (GstTensorCrop * self, GstBuffer * raw,
625 : tensor_crop_info_s * cinfo)
626 : {
627 6 : GstBuffer *result = NULL;
628 : GstMemory *mem;
629 : GstMapInfo map;
630 : GstTensorMetaInfo meta;
631 : GstTensorInfo info;
632 : gboolean flexible;
633 : gsize hsize, esize, dsize;
634 : guint8 *cropped, *dpos, *desc, *src;
635 : guint i, j, ch, mw, mh, _x, _y, _w, _h;
636 :
637 6 : i = gst_tensor_buffer_get_count (raw);
638 6 : g_assert (i > 0);
639 6 : if (i > 1) {
640 0 : GST_WARNING_OBJECT (self,
641 : "Raw data buffer has %u memories, parse first one.", i);
642 : }
643 :
644 6 : mem = gst_tensor_buffer_get_nth_memory (raw, 0);
645 6 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
646 0 : GST_ERROR_OBJECT (self, "Failed to map the raw buffer.");
647 0 : goto done;
648 : }
649 :
650 6 : if (!gst_tensor_crop_prepare_out_meta (self, map.data, &meta,
651 : &info, &flexible)) {
652 0 : GST_ERROR_OBJECT (self, "Failed to get the output meta.");
653 0 : goto done;
654 : }
655 :
656 6 : hsize = flexible ? gst_tensor_meta_info_get_header_size (&meta) : 0;
657 6 : dsize = gst_tensor_meta_info_get_data_size (&meta);
658 6 : dpos = map.data + hsize;
659 6 : if ((hsize + dsize) != map.size) {
660 1 : GST_ERROR_OBJECT (self,
661 : "Raw buffer has invalid data size (received %zd, expected %zd).",
662 : map.size, dsize);
663 1 : goto done;
664 : }
665 :
666 5 : result = gst_buffer_new ();
667 :
668 : /** @todo Add various mode to crop tensor. */
669 5 : ch = info.dimension[0];
670 5 : mw = info.dimension[1];
671 5 : mh = info.dimension[2];
672 5 : esize = gst_tensor_get_element_size (info.type);
673 5 : hsize = gst_tensor_meta_info_get_header_size (&meta);
674 :
675 14 : for (i = 0; i < cinfo->num; i++) {
676 : GstTensorInfo crop_info;
677 : GstMemory *crop_mem;
678 :
679 9 : _x = (cinfo->region[i].x < mw) ? cinfo->region[i].x : mw;
680 9 : _y = (cinfo->region[i].y < mh) ? cinfo->region[i].y : mh;
681 9 : _w = (_x + cinfo->region[i].w - 1 < mw) ? cinfo->region[i].w : (mw - _x);
682 9 : _h = (_y + cinfo->region[i].h - 1 < mh) ? cinfo->region[i].h : (mh - _y);
683 :
684 9 : g_assert (_w > 0 && _h > 0);
685 9 : dsize = hsize + (esize * ch * _w * _h);
686 9 : cropped = (guint8 *) g_malloc0 (dsize);
687 :
688 : /* set header for flex tensor */
689 9 : meta.dimension[1] = _w;
690 9 : meta.dimension[2] = _h;
691 9 : meta.dimension[3] = 1;
692 9 : gst_tensor_meta_info_update_header (&meta, cropped);
693 :
694 232 : for (j = 0; j < _h; j++) {
695 223 : src = dpos + esize * ch * (_x + (j + _y) * mw);
696 223 : desc = cropped + hsize + (esize * ch * _w) * j;
697 223 : memcpy (desc, src, (esize * ch * _w));
698 : }
699 :
700 : /* Prepare tensors-info to append memory into gst-buffer. */
701 9 : gst_tensor_meta_info_convert (&meta, &crop_info);
702 : crop_mem =
703 9 : gst_memory_new_wrapped (0, cropped, dsize, 0, dsize, cropped, g_free);
704 :
705 9 : gst_tensor_buffer_append_memory (result, crop_mem, &crop_info);
706 9 : gst_tensor_info_free (&crop_info);
707 : }
708 :
709 : /* set timestamp from raw buffer */
710 5 : gst_buffer_copy_into (result, raw, GST_BUFFER_COPY_METADATA, 0, -1);
711 :
712 6 : done:
713 6 : if (mem) {
714 6 : gst_memory_unmap (mem, &map);
715 6 : gst_memory_unref (mem);
716 : }
717 :
718 6 : return result;
719 : }
720 :
721 : /**
722 : * @brief Internal function to transform the input buffer.
723 : */
724 : static GstFlowReturn
725 9 : gst_tensor_crop_chain (GstTensorCrop * self,
726 : GstCollectData * data_raw, GstCollectData * data_info)
727 : {
728 : GstFlowReturn ret;
729 : GstBuffer *buf_raw, *buf_info, *result;
730 : GstTensorCropPadData *cpad;
731 : tensor_crop_info_s cinfo;
732 : gboolean drop_raw, drop_info;
733 :
734 18 : g_return_val_if_fail (data_raw && data_info, GST_FLOW_ERROR);
735 :
736 9 : buf_raw = gst_collect_pads_peek (self->collect, data_raw);
737 9 : buf_info = gst_collect_pads_peek (self->collect, data_info);
738 9 : drop_raw = (buf_raw != NULL);
739 9 : drop_info = (buf_info != NULL);
740 :
741 9 : if (!buf_raw || !buf_info) {
742 0 : ret = GST_FLOW_EOS;
743 0 : goto done;
744 : }
745 :
746 9 : cpad = (GstTensorCropPadData *) data_raw;
747 9 : buf_raw = gst_tensor_buffer_from_config (buf_raw, &cpad->config);
748 9 : cpad = (GstTensorCropPadData *) data_info;
749 9 : buf_info = gst_tensor_buffer_from_config (buf_info, &cpad->config);
750 :
751 : /**
752 : * The case when raw and info have different timestamp.
753 : * Compare timestamp and if time diff is less than lateness, crop raw buffer.
754 : */
755 9 : if (self->lateness >= 0) {
756 : GstClockTime ts_raw, ts_info, lateness;
757 :
758 4 : ts_raw = GST_BUFFER_TIMESTAMP (buf_raw);
759 4 : ts_info = GST_BUFFER_TIMESTAMP (buf_info);
760 4 : lateness = self->lateness * GST_MSECOND;
761 :
762 4 : if (GST_CLOCK_TIME_IS_VALID (ts_raw) && GST_CLOCK_TIME_IS_VALID (ts_info)) {
763 4 : if (((GstClockTime) ABS (GST_CLOCK_DIFF (ts_raw, ts_info))) > lateness) {
764 2 : GST_DEBUG_OBJECT (self, "Drop old buffer and wait for next.");
765 2 : GST_DEBUG_OBJECT (self, "Raw buffer ts: %" GST_TIME_FORMAT,
766 : GST_TIME_ARGS (ts_raw));
767 2 : GST_DEBUG_OBJECT (self, "Info buffer ts: %" GST_TIME_FORMAT,
768 : GST_TIME_ARGS (ts_info));
769 :
770 : /* clear old buffer and return ok to get next buffer */
771 2 : if (ts_raw > ts_info)
772 1 : drop_raw = FALSE;
773 : else
774 1 : drop_info = FALSE;
775 :
776 2 : ret = GST_FLOW_OK;
777 2 : goto done;
778 : }
779 : } else {
780 0 : GST_WARNING_OBJECT (self,
781 : "Incoming buffer has invalid timestamp, continue cropping data.");
782 : }
783 : }
784 :
785 7 : if (!gst_tensor_crop_get_crop_info (self, buf_info, &cinfo)) {
786 1 : ret = GST_FLOW_ERROR;
787 1 : goto done;
788 : }
789 :
790 6 : result = gst_tensor_crop_do_cropping (self, buf_raw, &cinfo);
791 6 : ret = gst_pad_push (self->srcpad, result);
792 :
793 9 : done:
794 9 : if (buf_raw)
795 9 : gst_buffer_unref (buf_raw);
796 9 : if (buf_info)
797 9 : gst_buffer_unref (buf_info);
798 :
799 : /* clear buffer in collect pads */
800 9 : if (drop_raw)
801 8 : gst_buffer_unref (gst_collect_pads_pop (self->collect, data_raw));
802 9 : if (drop_info)
803 8 : gst_buffer_unref (gst_collect_pads_pop (self->collect, data_info));
804 :
805 9 : return ret;
806 : }
807 :
808 : /**
809 : * @brief Chain function called when the buffer is available on all of the collect pads.
810 : */
811 : static GstFlowReturn
812 10 : gst_tensor_crop_collected (GstCollectPads * pads, gpointer user_data)
813 : {
814 : GstTensorCrop *self;
815 : GstCollectData *data_raw, *data_info;
816 : GSList *walk;
817 : GstFlowReturn ret;
818 :
819 10 : self = GST_TENSOR_CROP (user_data);
820 10 : data_raw = data_info = NULL;
821 :
822 10 : ret = gst_tensor_crop_negotiate (self);
823 10 : if (ret != GST_FLOW_OK)
824 0 : return ret;
825 :
826 28 : for (walk = pads->data; walk; walk = g_slist_next (walk)) {
827 : GstCollectData *data;
828 :
829 19 : data = (GstCollectData *) walk->data;
830 :
831 19 : if (GST_COLLECT_PADS_STATE_IS_SET (data, GST_COLLECT_PADS_STATE_EOS)) {
832 1 : gst_pad_push_event (self->srcpad, gst_event_new_eos ());
833 1 : return GST_FLOW_EOS;
834 : }
835 :
836 18 : if (data->pad == self->sinkpad_raw) {
837 9 : data_raw = data;
838 9 : } else if (data->pad == self->sinkpad_info) {
839 9 : data_info = data;
840 : }
841 : }
842 :
843 9 : return gst_tensor_crop_chain (self, data_raw, data_info);
844 : }
|
