Eric
4 years ago
committed by
xulei2020
parent
3f81802498
commit
9953757ff4
@ -1,190 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_ALLOCATOR_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_ALLOCATOR_H_
|
||||
|
||||
#include <cstdlib>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include "include/memory_pool.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
// The following conforms to the requirements of
|
||||
// std::allocator. Do not rename/change any needed
|
||||
// requirements, e.g. function names, typedef etc.
|
||||
template <typename T>
|
||||
class Allocator {
|
||||
public:
|
||||
template <typename U>
|
||||
friend class Allocator;
|
||||
|
||||
using value_type = T;
|
||||
using pointer = T *;
|
||||
using const_pointer = const T *;
|
||||
using reference = T &;
|
||||
using const_reference = const T &;
|
||||
using size_type = uint64_t;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
|
||||
template <typename U>
|
||||
struct rebind {
|
||||
using other = Allocator<U>;
|
||||
};
|
||||
|
||||
using propagate_on_container_copy_assignment = std::true_type;
|
||||
using propagate_on_container_move_assignment = std::true_type;
|
||||
using propagate_on_container_swap = std::true_type;
|
||||
|
||||
explicit Allocator(const std::shared_ptr<MemoryPool> &b) : pool_(b) {}
|
||||
|
||||
~Allocator() = default;
|
||||
|
||||
template <typename U>
|
||||
explicit Allocator(Allocator<U> const &rhs) : pool_(rhs.pool_) {}
|
||||
|
||||
template <typename U>
|
||||
bool operator==(Allocator<U> const &rhs) const {
|
||||
return pool_ == rhs.pool_;
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
bool operator!=(Allocator<U> const &rhs) const {
|
||||
return pool_ != rhs.pool_;
|
||||
}
|
||||
|
||||
pointer allocate(std::size_t n) {
|
||||
void *p = nullptr;
|
||||
Status rc = pool_->Allocate(n * sizeof(T), &p);
|
||||
if (rc.IsOk()) {
|
||||
return reinterpret_cast<pointer>(p);
|
||||
} else if (rc == StatusCode::kMDOutOfMemory) {
|
||||
throw std::bad_alloc();
|
||||
} else {
|
||||
throw std::exception();
|
||||
}
|
||||
}
|
||||
|
||||
void deallocate(pointer p, std::size_t n = 0) noexcept { pool_->Deallocate(p); }
|
||||
|
||||
size_type max_size() { return pool_->get_max_size(); }
|
||||
|
||||
private:
|
||||
std::shared_ptr<MemoryPool> pool_;
|
||||
};
|
||||
/// \brief It is a wrapper of unique_ptr with a custom Allocator class defined above
|
||||
template <typename T, typename C = std::allocator<T>, typename... Args>
|
||||
Status MakeUnique(std::unique_ptr<T[], std::function<void(T *)>> *out, C alloc, size_t n, Args &&... args) {
|
||||
RETURN_UNEXPECTED_IF_NULL(out);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(n > 0, "size must be positive");
|
||||
try {
|
||||
T *data = alloc.allocate(n);
|
||||
// Some of our implementation of allocator (e.g. NumaAllocator) don't throw std::bad_alloc.
|
||||
// So we have to catch for null ptr
|
||||
if (data == nullptr) {
|
||||
return Status(StatusCode::kMDOutOfMemory);
|
||||
}
|
||||
if (!std::is_arithmetic<T>::value) {
|
||||
for (auto i = 0; i < n; i++) {
|
||||
std::allocator_traits<C>::construct(alloc, &(data[i]), std::forward<Args>(args)...);
|
||||
}
|
||||
}
|
||||
auto deleter = [](T *p, C f_alloc, size_t f_n) {
|
||||
if (!std::is_arithmetic<T>::value && std::is_destructible<T>::value) {
|
||||
for (auto i = 0; i < f_n; ++i) {
|
||||
std::allocator_traits<C>::destroy(f_alloc, &p[i]);
|
||||
}
|
||||
}
|
||||
f_alloc.deallocate(p, f_n);
|
||||
};
|
||||
*out = std::unique_ptr<T[], std::function<void(T *)>>(data, std::bind(deleter, std::placeholders::_1, alloc, n));
|
||||
} catch (const std::bad_alloc &e) {
|
||||
return Status(StatusCode::kMDOutOfMemory);
|
||||
} catch (const std::exception &e) {
|
||||
RETURN_STATUS_UNEXPECTED(e.what());
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
/// \brief It is a wrapper of the above custom unique_ptr with some additional methods
|
||||
/// \tparam T The type of object to be allocated
|
||||
/// \tparam C Allocator. Default to std::allocator
|
||||
template <typename T, typename C = std::allocator<T>>
|
||||
class MemGuard {
|
||||
public:
|
||||
using allocator = C;
|
||||
MemGuard() : n_(0) {}
|
||||
explicit MemGuard(allocator a) : n_(0), alloc_(a) {}
|
||||
// There is no copy constructor nor assignment operator because the memory is solely owned by this object.
|
||||
MemGuard(const MemGuard &) = delete;
|
||||
MemGuard &operator=(const MemGuard &) = delete;
|
||||
// On the other hand, We can support move constructor
|
||||
MemGuard(MemGuard &&lhs) noexcept : n_(lhs.n_), alloc_(std::move(lhs.alloc_)), ptr_(std::move(lhs.ptr_)) {}
|
||||
MemGuard &operator=(MemGuard &&lhs) noexcept {
|
||||
if (this != &lhs) {
|
||||
this->deallocate();
|
||||
n_ = lhs.n_;
|
||||
alloc_ = std::move(lhs.alloc_);
|
||||
ptr_ = std::move(lhs.ptr_);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
/// \brief Explicitly deallocate the memory if allocated
|
||||
void deallocate() {
|
||||
if (ptr_) {
|
||||
ptr_.reset();
|
||||
}
|
||||
}
|
||||
/// \brief Allocate memory (with emplace feature). Previous one will be released. If size is 0, no new memory is
|
||||
/// allocated.
|
||||
/// \param n Number of objects of type T to be allocated
|
||||
/// \tparam Args Extra arguments pass to the constructor of T
|
||||
template <typename... Args>
|
||||
Status allocate(size_t n, Args &&... args) noexcept {
|
||||
deallocate();
|
||||
n_ = n;
|
||||
return MakeUnique(&ptr_, alloc_, n, std::forward<Args>(args)...);
|
||||
}
|
||||
~MemGuard() noexcept { deallocate(); }
|
||||
/// \brief Getter function
|
||||
/// \return The pointer to the memory allocated
|
||||
T *GetPointer() const { return ptr_.get(); }
|
||||
/// \brief Getter function
|
||||
/// \return The pointer to the memory allocated
|
||||
T *GetMutablePointer() { return ptr_.get(); }
|
||||
/// \brief Overload [] operator to access a particular element
|
||||
/// \param x index to the element. Must be less than number of element allocated.
|
||||
/// \return pointer to the x-th element
|
||||
T *operator[](size_t x) { return GetMutablePointer() + x; }
|
||||
/// \brief Overload [] operator to access a particular element
|
||||
/// \param x index to the element. Must be less than number of element allocated.
|
||||
/// \return pointer to the x-th element
|
||||
T *operator[](size_t x) const { return GetPointer() + x; }
|
||||
/// \brief Return how many bytes are allocated in total
|
||||
/// \return Number of bytes allocated in total
|
||||
size_t GetSizeInBytes() const { return n_ * sizeof(T); }
|
||||
|
||||
private:
|
||||
size_t n_;
|
||||
allocator alloc_;
|
||||
std::unique_ptr<T[], std::function<void(T *)>> ptr_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_ALLOCATOR_H_
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,59 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_MEMORY_POOL_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_MEMORY_POOL_H_
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include "include/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
// Abstract class of a memory pool
|
||||
class MemoryPool {
|
||||
public:
|
||||
// Allocate a block of size n
|
||||
virtual Status Allocate(size_t, void **) = 0;
|
||||
|
||||
// Enlarge or shrink a block from oldSz to newSz
|
||||
virtual Status Reallocate(void **, size_t old_sz, size_t new_sz) = 0;
|
||||
|
||||
// Free a pointer
|
||||
virtual void Deallocate(void *) = 0;
|
||||
|
||||
// What is the maximum size I can allocate ?
|
||||
virtual uint64_t get_max_size() const = 0;
|
||||
|
||||
virtual int PercentFree() const = 0;
|
||||
|
||||
// Destructor
|
||||
virtual ~MemoryPool() {}
|
||||
};
|
||||
|
||||
Status DeMalloc(std::size_t s, void **p, bool);
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
void *operator new(std::size_t, mindspore::Status *, std::shared_ptr<mindspore::dataset::MemoryPool>);
|
||||
|
||||
void *operator new[](std::size_t, mindspore::Status *, std::shared_ptr<mindspore::dataset::MemoryPool>);
|
||||
|
||||
void operator delete(void *, std::shared_ptr<mindspore::dataset::MemoryPool>);
|
||||
|
||||
void operator delete[](void *, std::shared_ptr<mindspore::dataset::MemoryPool>);
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_MEMORY_POOL_H_
|
||||
@ -1,126 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_PATH_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_PATH_H_
|
||||
|
||||
#include <dirent.h>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "include/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class Path {
|
||||
public:
|
||||
class DirIterator {
|
||||
public:
|
||||
static std::shared_ptr<DirIterator> OpenDirectory(Path *f);
|
||||
|
||||
~DirIterator();
|
||||
|
||||
bool hasNext();
|
||||
|
||||
Path next();
|
||||
|
||||
private:
|
||||
explicit DirIterator(Path *f);
|
||||
|
||||
Path *dir_;
|
||||
DIR *dp_;
|
||||
struct dirent *entry_;
|
||||
};
|
||||
|
||||
explicit Path(const std::string &);
|
||||
|
||||
explicit Path(const char *);
|
||||
|
||||
~Path() = default;
|
||||
|
||||
Path(const Path &);
|
||||
|
||||
Path &operator=(const Path &);
|
||||
|
||||
Path(Path &&) noexcept;
|
||||
|
||||
Path &operator=(Path &&) noexcept;
|
||||
|
||||
std::string toString() const { return path_; }
|
||||
|
||||
Path operator+(const Path &);
|
||||
|
||||
Path operator+(const std::string &);
|
||||
|
||||
Path operator+(const char *);
|
||||
|
||||
Path &operator+=(const Path &rhs);
|
||||
|
||||
Path &operator+=(const std::string &);
|
||||
|
||||
Path &operator+=(const char *);
|
||||
|
||||
Path operator/(const Path &);
|
||||
|
||||
Path operator/(const std::string &);
|
||||
|
||||
Path operator/(const char *);
|
||||
|
||||
bool operator==(const Path &rhs) const { return (path_ == rhs.path_); }
|
||||
|
||||
bool operator!=(const Path &rhs) const { return (path_ != rhs.path_); }
|
||||
|
||||
bool operator<(const Path &rhs) const { return (path_ < rhs.path_); }
|
||||
|
||||
bool operator>(const Path &rhs) const { return (path_ > rhs.path_); }
|
||||
|
||||
bool operator<=(const Path &rhs) const { return (path_ <= rhs.path_); }
|
||||
|
||||
bool operator>=(const Path &rhs) const { return (path_ >= rhs.path_); }
|
||||
|
||||
bool Exists();
|
||||
|
||||
bool IsDirectory();
|
||||
|
||||
Status CreateDirectory();
|
||||
|
||||
Status CreateDirectories();
|
||||
|
||||
std::string Extension() const;
|
||||
|
||||
std::string ParentPath();
|
||||
|
||||
Status Remove();
|
||||
|
||||
Status CreateFile(int *fd);
|
||||
|
||||
Status OpenFile(int *fd, bool create = false);
|
||||
|
||||
Status CloseFile(int fd) const;
|
||||
|
||||
Status TruncateFile(int fd) const;
|
||||
|
||||
std::string Basename();
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &os, const Path &s);
|
||||
|
||||
private:
|
||||
static char separator_;
|
||||
std::string path_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_PATH_H_
|
||||
@ -1,105 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STATUS_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STATUS_H_
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
#define DEPRECATED __attribute__((deprecated))
|
||||
#elif defined(_MSC_VER)
|
||||
#define DEPRECATED __declspec(deprecated)
|
||||
#else
|
||||
#pragma message("WARNING: You need to implement DEPRECATED for this compiler")
|
||||
#define DEPRECATED
|
||||
#endif
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "include/ms_status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
#define RETURN_IF_NOT_OK(_s) \
|
||||
do { \
|
||||
Status __rc = (_s); \
|
||||
if (__rc.IsError()) { \
|
||||
return __rc; \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
#define RETURN_STATUS_UNEXPECTED(_e) \
|
||||
do { \
|
||||
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, _e); \
|
||||
} while (false)
|
||||
|
||||
#define CHECK_FAIL_RETURN_UNEXPECTED(_condition, _e) \
|
||||
do { \
|
||||
if (!(_condition)) { \
|
||||
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, _e); \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
#define CHECK_FAIL_RETURN_SYNTAX_ERROR(_condition, _e) \
|
||||
do { \
|
||||
if (!(_condition)) { \
|
||||
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, _e); \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
#define CHECK_FAIL_RETURN_SYNTAX_ERROR(_condition, _e) \
|
||||
do { \
|
||||
if (!(_condition)) { \
|
||||
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, _e); \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
#define RETURN_UNEXPECTED_IF_NULL(_ptr) \
|
||||
do { \
|
||||
if ((_ptr) == nullptr) { \
|
||||
std::string err_msg = "The pointer[" + std::string(#_ptr) + "] is null."; \
|
||||
RETURN_STATUS_UNEXPECTED(err_msg); \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
#define RETURN_OK_IF_TRUE(_condition) \
|
||||
do { \
|
||||
if (_condition) { \
|
||||
return Status::OK(); \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
#define RETURN_STATUS_SYNTAX_ERROR(_e) \
|
||||
do { \
|
||||
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, _e); \
|
||||
} while (false)
|
||||
|
||||
#define RETURN_SECOND_IF_ERROR(_s, _r) \
|
||||
do { \
|
||||
Status __rc = (_s); \
|
||||
if (__rc.IsError()) { \
|
||||
MS_LOG(ERROR) << __rc; \
|
||||
return _r; \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
#if !defined(_WIN32) && !defined(_WIN64)
|
||||
const float MAX_MEMORY_USAGE_THRESHOLD = 0.95;
|
||||
float GetMemoryUsage();
|
||||
#endif
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STATUS_H_
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,83 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_TENSOR_HELPERS_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_TENSOR_HELPERS_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "include/constants.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class Slice {
|
||||
public:
|
||||
Slice() : start_(0), stop_(0), step_(0) {}
|
||||
Slice(dsize_t start, dsize_t stop, dsize_t step) : start_(start), stop_(stop), step_(step) {}
|
||||
Slice(dsize_t start, dsize_t stop) : start_(start), stop_(stop), step_(1) {}
|
||||
explicit Slice(dsize_t stop) : start_(0), stop_(stop), step_(1) {}
|
||||
Slice(Slice const &slice) = default;
|
||||
|
||||
~Slice() = default;
|
||||
|
||||
bool valid() const { return step_ != 0; }
|
||||
dsize_t start_;
|
||||
dsize_t stop_;
|
||||
dsize_t step_;
|
||||
};
|
||||
|
||||
class SliceOption {
|
||||
public:
|
||||
explicit SliceOption(bool all) : all_(all) {}
|
||||
explicit SliceOption(std::vector<dsize_t> indices) : indices_(indices) {}
|
||||
explicit SliceOption(Slice slice) : slice_(slice) {}
|
||||
SliceOption(SliceOption const &slice) = default;
|
||||
|
||||
~SliceOption() = default;
|
||||
|
||||
// only one of the following will be valid
|
||||
// given indices to slice the Tensor.
|
||||
std::vector<dsize_t> indices_ = {};
|
||||
// Slice object. All start, stop and step are 0 if invalid.
|
||||
Slice slice_;
|
||||
bool all_ = false;
|
||||
};
|
||||
|
||||
/// Recursive helper function to generate indices based on vector of SliceOptions. It recursively iterates through each
|
||||
/// range represented by slice_options to generate a list of indices to be sliced.
|
||||
/// \param[out] matrix Generated nested vector of indices
|
||||
/// Example: For a 4 x 2 tensor, and with slice_list = {SliceOption({0})} (the first row), matrix will become
|
||||
/// {{0}}. For slice_list = {SliceOption(all), SliceOption({0})} (the first column), matrix will become
|
||||
/// {{0, 0}, {1, 0}, {2, 0}, {3, 0}}.
|
||||
/// For slice_list = {SliceOption({0, 2})}, matrix will become {{0}, {2}}. The size of each nested array is always
|
||||
/// equal to (slice_list).size().
|
||||
/// \param[in] depth used to keep track of recursion level
|
||||
/// \param[in] numbers vector used to represent current index
|
||||
/// \param[in] matrix 2D vector to be populated with desired indices
|
||||
/// \param[in] slice_options vector of SliceOption objects
|
||||
void IndexGeneratorHelper(int8_t depth, std::vector<dsize_t> *numbers, const std::vector<SliceOption> &slice_list,
|
||||
std::vector<std::vector<dsize_t>> *matrix);
|
||||
|
||||
/// Generate indices based on vector of SliceOptions
|
||||
/// Calls the recursive helper function IndexGeneratorHelper
|
||||
/// \param[in] slice_list vector of SliceOption objects. Note: If the user passes
|
||||
/// {SliceOption(true), SliceOption(true)}, it will return a M x 2 vector, instead of reducing it to
|
||||
/// {SliceOption(true)} first to only generate a M x 1 vector.
|
||||
/// \return std::vector<std::vector<dsize_t>> 2D vector of generated indices, M x (slice_list).size()
|
||||
std::vector<std::vector<dsize_t>> IndexGenerator(const std::vector<SliceOption> &slice_list);
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_TENSOR_HELPERS_H_
|
||||
@ -1,176 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_TENSOR_SHAPE_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_TENSOR_SHAPE_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <ostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "include/constants.h"
|
||||
#include "include/status.h"
|
||||
#include "include/allocator.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
using IntAlloc = Allocator<dsize_t>;
|
||||
// Class that represents a shape of a Tensor. A shape can be:
|
||||
// -# Known shape (mKnown = true)
|
||||
// -# Scalar --> empty vector --> <>
|
||||
// -# n-Dim --> not empty vector --> <d1, d2, d2, d3, ...> where di is >= 0\n
|
||||
// Example: <1,2>, <1>, <1,13,10,11,1>
|
||||
// -# Unknown shape (mKnown = false)
|
||||
// -# Rank is unknown --> empty vector --> <>
|
||||
// -# one or more dim is unknown --> not empty vector --> <d1, d2, d2, d3, ...> where di is unknown\n
|
||||
// Example: <3,?> (the 1st dim is unknown)\n
|
||||
// <2,?,?,?> (all dims but the 0th dim are unknown)
|
||||
|
||||
/// \brief TensorShape supports any dim > 0 and < 2^31-1
|
||||
|
||||
class TensorShape {
|
||||
public:
|
||||
static constexpr dsize_t kDimUnknown = -1; // constant for an unknown dimension
|
||||
|
||||
// Force the compiler to not create a no-arg constructor
|
||||
TensorShape() = delete;
|
||||
|
||||
/// \brief Create a Shape from an initialization list (e.g., TensorShape s = {2,2}).
|
||||
/// If one of the dims is set to DIM_UNKNOWN, the shape will flagged as unKnown
|
||||
/// \param[in] list
|
||||
explicit TensorShape(const std::initializer_list<dsize_t> &list);
|
||||
|
||||
/// \brief Create a Shape from a vector (e.g., TensorShape s = std::vector<dsize_t>({2,2}) ).
|
||||
/// If one of the dims is set to DIM_UNKNOWN, the shape will flagged as unKnown
|
||||
/// \param[in] list
|
||||
explicit TensorShape(const std::vector<dsize_t> &list);
|
||||
|
||||
/// \brief Copy constructor
|
||||
/// \param[in] shape
|
||||
TensorShape(const TensorShape &shape);
|
||||
|
||||
~TensorShape() = default;
|
||||
|
||||
/// \brief Create a scalar Shape (i.e., empty shape with mKnown = true)
|
||||
/// \return TensorShape
|
||||
static TensorShape CreateScalar() { return TensorShape({}); }
|
||||
|
||||
/// \brief Create a shape with an unknown rank.
|
||||
/// \return TensorShape
|
||||
static TensorShape CreateUnknownRankShape();
|
||||
|
||||
/// \brief Create a shape with a known rank .
|
||||
/// \return TensorShape
|
||||
static TensorShape CreateUnknownShapeWithRank(dsize_t rank);
|
||||
|
||||
/// \brief Insert a new dim into a copy of the current shape.
|
||||
/// \param[in] dim to be added
|
||||
/// \param[in] axis the index where dim should be added
|
||||
/// \return New modified shape
|
||||
TensorShape InsertDim(dsize_t axis, dsize_t dim) const;
|
||||
|
||||
/// \brief Insert new dim at index 0. For example, <2,4> --> PrependDim(4) --> <4,2,4>
|
||||
/// \param[in] dim
|
||||
/// \return
|
||||
TensorShape PrependDim(dsize_t dim) const;
|
||||
|
||||
/// \brief Insert a new dim at the end of the shape. For example, <2,4> --> AppendDim(4) --> <2,4,4>
|
||||
/// \param[in] dim
|
||||
/// \return
|
||||
TensorShape AppendDim(dsize_t dim) const;
|
||||
|
||||
dsize_t Size() const { return raw_shape_.size(); }
|
||||
|
||||
dsize_t Rank() const { return raw_shape_.size(); }
|
||||
|
||||
bool known() const { return known_; }
|
||||
|
||||
bool empty() const { return raw_shape_.empty(); }
|
||||
|
||||
dsize_t NumOfElements() const;
|
||||
|
||||
bool operator==(const TensorShape &rhs) const { return known_ == rhs.known_ && raw_shape_ == rhs.raw_shape_; }
|
||||
|
||||
bool operator!=(const TensorShape &rhs) const { return !(rhs == *this); }
|
||||
|
||||
dsize_t operator[](const dsize_t index) const {
|
||||
if (index < 0) return raw_shape_[raw_shape_.size() + index];
|
||||
return raw_shape_[index];
|
||||
}
|
||||
|
||||
/// \brief Return the Shape as a vector
|
||||
/// \return
|
||||
std::vector<dsize_t> AsVector() const;
|
||||
|
||||
/// \brief Returns the class info as a string
|
||||
/// \return
|
||||
std::string ToString() const {
|
||||
std::stringstream ss;
|
||||
ss << *this;
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
/// \brief Actual print function used by operator<<
|
||||
/// \param out output string stream
|
||||
void Print(std::ostream &out) const;
|
||||
|
||||
/// \brief << Stream output operator overload
|
||||
/// This allows you to print the info using stream operators
|
||||
/// \param[in] out - reference to the output stream being overloaded
|
||||
/// \param[in] rO - reference to the TensorShape to display
|
||||
/// \return - the output stream must be returned
|
||||
friend std::ostream &operator<<(std::ostream &out, const TensorShape &so) {
|
||||
so.Print(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
/// \brief Checks if the given index is a valid index for this tensor.
|
||||
/// For example: Tensor<3,4> Index<1,1> is valid. But Index<4,1> or <1> are not.
|
||||
/// \param[in] index
|
||||
/// \return bool
|
||||
bool IsValidIndex(const std::vector<dsize_t> &index) const;
|
||||
|
||||
TensorShape Squeeze() const;
|
||||
|
||||
std::vector<dsize_t> Strides() const;
|
||||
|
||||
/// \brief Returns the location of the item assuming row major memory layout.
|
||||
/// \param[in] index
|
||||
/// \param[out] flat_index
|
||||
/// \return
|
||||
Status ToFlatIndex(const std::vector<dsize_t> &index, dsize_t *flat_index) const;
|
||||
|
||||
private:
|
||||
// True if known and valid shape, false otherwise
|
||||
bool known_;
|
||||
// Vector to keep the dims of the shape.
|
||||
std::vector<dsize_t, IntAlloc> raw_shape_;
|
||||
// Vector to keep the strides of the shape. The size is rank+1
|
||||
std::vector<dsize_t, IntAlloc> strides_;
|
||||
|
||||
/// \brief Internal utility function to iterate over a list,
|
||||
/// check if the dim is valid and then insert it into the shape.
|
||||
/// \param[in] list Iterable list
|
||||
/// \return true if the shape is valid and no overflow would be generated when counting the number of elements.
|
||||
/// False otherwise.
|
||||
template <typename T>
|
||||
void AddListToShape(const T &list);
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_TENSOR_SHAPE_H_
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue