xulei2020
4 years ago
parent
40222f59a7
commit
406c586374
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,173 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_ALBUM_ANDROID_OP_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_ALBUM_ANDROID_OP_H_
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <queue>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <unordered_map>
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/engine/data_buffer.h"
|
||||
#include "minddata/dataset/engine/data_schema.h"
|
||||
#include "minddata/dataset/util/path.h"
|
||||
#include "minddata/dataset/util/queue.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
// Forward declares
|
||||
template <typename T>
|
||||
class Queue;
|
||||
|
||||
// Define row information as a list of file objects to read
|
||||
using FolderImages = std::shared_ptr<std::pair<std::string, std::queue<std::string>>>;
|
||||
|
||||
/// \class AlbumOp
|
||||
class AlbumOp {
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] file_dir - directory of Album
|
||||
/// \param[in] do_decode - decode image files
|
||||
/// \param[in] schema_file - schema file
|
||||
/// \param[in] exts - set of file extensions to read, if empty, read everything under the dir
|
||||
AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
|
||||
const std::set<std::string> &exts);
|
||||
|
||||
/// \brief Constructor
|
||||
/// \param[in] file_dir - directory of Album
|
||||
/// \param[in] do_decode - decode image files
|
||||
/// \param[in] schema_file - schema file
|
||||
/// \param[in] exts - set of file extensions to read, if empty, read everything under the dir
|
||||
/// \param[in] index - the specific file index
|
||||
AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
|
||||
const std::set<std::string> &exts, uint32_t index);
|
||||
|
||||
/// \brief Destructor.
|
||||
~AlbumOp() = default;
|
||||
|
||||
/// \brief Initialize AlbumOp related var, calls the function to walk all files
|
||||
/// \return - The error code returned
|
||||
Status PrescanEntry();
|
||||
|
||||
/// \brief Initialize AlbumOp related var, calls the function to walk all files
|
||||
/// \return - The error code returned
|
||||
bool GetNextRow(std::unordered_map<std::string, std::shared_ptr<Tensor>> *map_row);
|
||||
|
||||
/// \brief Check if image ia valid.Only support JPEG/PNG/GIF/BMP
|
||||
/// This function could be optimized to return the tensor to reduce open/closing files
|
||||
/// \return bool - if file is bad then return false
|
||||
bool CheckImageType(const std::string &file_name, bool *valid);
|
||||
|
||||
// Op name getter
|
||||
// @return Name of the current Op
|
||||
std::string Name() const { return "AlbumOp"; }
|
||||
|
||||
private:
|
||||
/// \brief Load image to tensor
|
||||
/// \param[in] image_file Image name of file
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadImageTensor(const std::string &image_file, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load vector of ints to tensor, append tensor to tensor
|
||||
/// \param[in] json_obj Json object containing multi-dimensional label
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load vector of floatss to tensor, append tensor to tensor
|
||||
/// \param[in] json_obj Json object containing array data
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load string array into a tensor, append tensor to tensor
|
||||
/// \param[in] json_obj Json object containing string tensor
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load string into a tensor, append tensor to tensor
|
||||
/// \param[in] json_obj Json object containing string tensor
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadStringTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load float value to tensor
|
||||
/// \param[in] json_obj Json object containing float
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadFloatTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load int value to tensor
|
||||
/// \param[in] json_obj Json object containing int
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadIntTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load emtpy tensor to tensor
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadEmptyTensor(uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load id from file name to tensor
|
||||
/// \param[in] file The file name to get ID from
|
||||
/// \param[in] col_num Column num in schema
|
||||
/// \param[inout] Tensor to push to
|
||||
/// \return Status The error code returned
|
||||
Status LoadIDTensor(const std::string &file, uint32_t col_num, TensorPtr *tensor);
|
||||
|
||||
/// \brief Load a tensor according to a json file
|
||||
/// \param[in] row_id_type row_id - id for this tensor row
|
||||
/// \param[in] ImageColumns file Json file location
|
||||
/// \param[inout] TensorRow Json content stored into a tensor row
|
||||
/// \return Status The error code returned
|
||||
Status LoadTensorRow(row_id_type row_id, const std::string &file,
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> *map_row);
|
||||
|
||||
std::string folder_path_; // directory of image folder
|
||||
bool decode_;
|
||||
std::vector<std::string> columns_to_load_;
|
||||
std::set<std::string> extensions_; // extensions allowed
|
||||
std::unique_ptr<DataSchema> data_schema_;
|
||||
std::string schema_file_;
|
||||
int64_t row_cnt_;
|
||||
int64_t current_cnt_;
|
||||
int64_t buf_cnt_;
|
||||
int64_t dirname_offset_;
|
||||
bool sampler_;
|
||||
int64_t sampler_index_;
|
||||
std::vector<std::string> image_rows_;
|
||||
std::unordered_map<std::string, int32_t> column_name_id_map_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_ALBUM_ANDROID_OP_H_
|
||||
Loading…
Reference in new issue