Line data Source code
1 : // SPDX-License-Identifier: Apache-2.0
2 : /**
3 : * Copyright (C) 2023 Jijoong Moon <jijoong.moon@samsung.com>
4 : *
5 : * @file dir_data_producers.h
6 : * @date 24 Feb 2023
7 : * @brief This file contains dir data producers, reading from the files in
8 : * directory
9 : * @see https://github.com/nnstreamer/nntrainer
10 : * @author Jijoong Moon <jijoong.moon@samsung.com>
11 : * @bug No known bugs except for NYI items
12 : *
13 : */
14 :
15 : #include <dir_data_producers.h>
16 : #include <filesystem>
17 :
18 : #include <memory>
19 : #include <numeric>
20 : #include <random>
21 : #include <sys/stat.h>
22 : #include <vector>
23 :
24 : #include <common_properties.h>
25 : #include <nntrainer_error.h>
26 : #include <node_exporter.h>
27 : #include <util_func.h>
28 :
29 : /**
30 : * @brief this function helps to read the image
31 : * currently only bmp image is supported and extend the image type will be
32 : * remain as TODO. ( BGR --> RGB )
33 : * @param path file path
34 : * @param inputs float data for image pixel
35 : * @param width width
36 : * @param height height
37 : */
38 0 : static void readImage(const std::string path, float *input, uint width,
39 : uint height) {
40 0 : FILE *f = fopen(path.c_str(), "rb");
41 :
42 0 : if (f == nullptr)
43 0 : throw std::invalid_argument("Cannot open file: " + path);
44 :
45 0 : unsigned char info[54];
46 0 : size_t result = fread(info, sizeof(unsigned char), 54, f);
47 0 : NNTR_THROW_IF(result != 54, std::invalid_argument)
48 0 : << "Cannot read bmp header";
49 :
50 0 : uint w = *(int *)&info[18];
51 0 : uint h = *(int *)&info[22];
52 :
53 0 : size_t row_padded = (width * 3 + 3) & (~3);
54 0 : unsigned char *data = new unsigned char[row_padded];
55 :
56 0 : for (uint i = 0; i < height; i++) {
57 0 : result = fread(data, sizeof(unsigned char), row_padded, f);
58 0 : NNTR_THROW_IF(result != row_padded, std::invalid_argument)
59 0 : << "Cannot read bmp pixel data";
60 :
61 0 : for (uint j = 0; j < width; j++) {
62 :
63 0 : input[height * i + j] = (float)data[j * 3 + 2];
64 :
65 0 : input[(height * width) + height * i + j] = (float)data[j * 3 + 1];
66 :
67 0 : input[(height * width) * 2 + height * i + j] = (float)data[j * 3];
68 : }
69 : }
70 :
71 0 : delete[] data;
72 0 : fclose(f);
73 0 : }
74 :
75 : namespace nntrainer {
76 :
77 0 : DirDataProducer::DirDataProducer() :
78 0 : dir_data_props(new Props()),
79 : num_class(0),
80 0 : num_data_total(0) {}
81 :
82 0 : DirDataProducer::DirDataProducer(const std::string &dir_path) :
83 0 : dir_data_props(new Props(props::DirPath(dir_path))),
84 : num_class(0),
85 0 : num_data_total(0) {}
86 :
87 0 : DirDataProducer::~DirDataProducer() {}
88 :
89 0 : const std::string DirDataProducer::getType() const {
90 0 : return DirDataProducer::type;
91 : }
92 :
93 0 : bool DirDataProducer::isMultiThreadSafe() const {
94 : /// @todo make this true, it is needed to test multiple worker scenario
95 0 : return false;
96 : }
97 :
98 0 : void DirDataProducer::setProperty(const std::vector<std::string> &properties) {
99 0 : auto left = loadProperties(properties, *dir_data_props);
100 0 : NNTR_THROW_IF(!left.empty(), std::invalid_argument)
101 0 : << "There are unparsed properties, size: " << left.size();
102 0 : }
103 :
104 : DataProducer::Generator
105 0 : DirDataProducer::finalize(const std::vector<TensorDim> &input_dims,
106 : const std::vector<TensorDim> &label_dims,
107 : void *user_data) {
108 :
109 0 : auto dir_path = std::get<props::DirPath>(*dir_data_props).get();
110 :
111 0 : for (const auto &entry : std::filesystem::directory_iterator(dir_path))
112 0 : class_names.push_back(entry.path());
113 :
114 0 : num_class = class_names.size();
115 :
116 0 : size_t id = 0;
117 0 : size_t num_data = 0;
118 0 : for (auto c_name : class_names) {
119 0 : num_data = 0;
120 0 : std::filesystem::directory_iterator itr(c_name);
121 0 : while (itr != std::filesystem::end(itr)) {
122 0 : const std::filesystem::directory_entry &entry = *itr;
123 0 : std::string p = std::filesystem::absolute(entry.path()).string();
124 0 : if (p.compare(".") && p.compare("..")) {
125 0 : num_data++;
126 0 : data_list.push_back(std::make_pair(id, p));
127 : }
128 0 : itr++;
129 : }
130 :
131 0 : id++;
132 0 : num_data_total += num_data;
133 : }
134 :
135 : /// @todo expand this to non onehot case
136 0 : NNTR_THROW_IF(std::any_of(label_dims.begin(), label_dims.end(),
137 : [](const TensorDim &dim) {
138 : return dim.channel() != 1 || dim.height() != 1;
139 : }),
140 : std::invalid_argument)
141 0 : << "Label dimension containing channel or height not allowed";
142 :
143 0 : auto sz = size(input_dims, label_dims);
144 :
145 0 : NNTR_THROW_IF(sz == 0, std::invalid_argument)
146 0 : << "size is zero, dataproducer does not provide anything";
147 :
148 0 : return [sz, input_dims, this](unsigned int idx, std::vector<Tensor> &inputs,
149 0 : std::vector<Tensor> &labels) {
150 0 : NNTR_THROW_IF(idx >= sz, std::range_error)
151 0 : << "given index is out of bound, index: " << idx << " size: " << sz;
152 :
153 0 : std::string file_name = data_list[idx].second;
154 :
155 0 : readImage(file_name, inputs[0].getData(), input_dims[0].width(),
156 0 : input_dims[0].height());
157 :
158 0 : unsigned int c_id = data_list[idx].first;
159 :
160 0 : std::memset(labels[0].getData(), 0.0, num_class * sizeof(float));
161 :
162 0 : labels[0].getData()[c_id] = 1.0;
163 :
164 0 : return idx == sz - 1;
165 0 : };
166 : }
167 :
168 : unsigned int
169 0 : DirDataProducer::size(const std::vector<TensorDim> &input_dims,
170 : const std::vector<TensorDim> &label_dims) const {
171 :
172 0 : return num_data_total;
173 : }
174 :
175 : } // namespace nntrainer
|
