m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Removed log_adapter dependency for minddata lite

Browse Source 
Brute force fix?

Added missing import

Added removal of new cache functionality

Fix batch

Remove crc and cache optimizer

Remove affine

Remove cut mix batch

Added securec.h

Removed unused crc import

Fixed minor arm64 compile issues for tensor ops

Removed excess code from de_tensor

Added vision import

remove random device ops

Removed commented line

Added extra layer to switch adapter logic

Added securec

Added float fix
pull/7418/head
Eric 4 years ago
parent 25dd36059b
commit 1f11ccce06
52 changed files with 182 additions and 52 deletions
Show Stats Download Patch File Download Diff File

20
mindspore/ccsrc/minddata/dataset/api/de_tensor.cc
Unescape View File

@ -14,14 +14,13 @@
* limitations under the License.
*/
#include "minddata/dataset/include/de_tensor.h"
#include "minddata/dataset/include/type_id.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/include/de_tensor.h"
#include "minddata/dataset/include/type_id.h"
#include "mindspore/core/ir/dtype/type_id.h"
#include "utils/hashing.h"
#include "mindspore/lite/internal/include/ms_tensor.h"
#include "mindspore/core/utils/convert_utils_base.h"
#include "utils/hashing.h"
namespace mindspore {
namespace tensor {
@ -124,19 +123,6 @@ int DETensor::ElementsNum() const {
return this->tensor_impl_->Size();
}
std::size_t DETensor::hash() const {
MS_ASSERT(this->tensor_impl_ != nullptr);
auto shape = this->shape();
std::size_t hash_value = std::hash<int>{}(SizeToInt(this->data_type()));
hash_value = hash_combine(hash_value, std::hash<size_t>{}(shape.size()));
// hash all elements may costly, so only take at most 4 elements into account based on
// some experiments.
for (size_t i = 0; (i < shape.size()) && (i < 4); ++i) {
hash_value = hash_combine(hash_value, (std::hash<int>{}(shape[i])));
}
return hash_value;
}
size_t DETensor::Size() const {
MS_ASSERT(this->tensor_impl_ != nullptr);
return this->tensor_impl_->SizeInBytes();

2
mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc
Unescape View File

@ -41,6 +41,7 @@
#include "minddata/dataset/engine/datasetops/source/text_file_op.h"
#include "minddata/dataset/engine/datasetops/source/voc_op.h"
#include "minddata/dataset/kernels/py_func_op.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/status.h"
#include "minddata/mindrecord/include/shard_category.h"
@ -51,7 +52,6 @@
#include "minddata/mindrecord/include/shard_shuffle.h"
#include "minddata/mindrecord/include/shard_writer.h"
#include "pybind11/stl.h"
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
namespace mindspore {

6
mindspore/ccsrc/minddata/dataset/core/config_manager.cc
Unescape View File

@ -20,7 +20,11 @@
#include <string>
#include <utility>
#include "mindspore/core/utils/log_adapter.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
#include "minddata/dataset/util/system_pool.h"
namespace mindspore {

4
mindspore/ccsrc/minddata/dataset/core/data_type.cc
Unescape View File

@ -18,7 +18,11 @@
#include "minddata/dataset/core/pybind_support.h"
#endif
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

6
mindspore/ccsrc/minddata/dataset/core/tensor.h
Unescape View File

@ -21,7 +21,11 @@
#include <string>
#include <vector>
#include "./securec.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
@ -33,12 +37,12 @@
#include "pybind11/stl.h"
#endif
#include "utils/ms_utils.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor_helpers.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/util/status.h"
#include "utils/ms_utils.h"
#ifndef ENABLE_ANDROID
#include "proto/example.pb.h"
#else

4
mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc
Unescape View File

@ -20,7 +20,11 @@
#include <limits>
#include "utils/ms_utils.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
#include "minddata/dataset/core/constants.h"
namespace mindspore {

6
mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc vendored
Unescape View File

@ -13,11 +13,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/engine/cache/cache_grpc_server.h"
#include <limits>
#include "minddata/dataset/engine/cache/cache_grpc_server.h"
#include "minddata/dataset/engine/cache/cache_server.h"
#include "minddata/dataset/util/path.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {
CacheServerGreeterImpl::CacheServerGreeterImpl(int32_t port, int32_t shared_memory_sz_in_gb)

4
mindspore/ccsrc/minddata/dataset/engine/data_schema.cc
Unescape View File

@ -26,7 +26,11 @@
#include "utils/ms_utils.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/core/tensor_shape.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

1
mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc
Unescape View File

@ -21,7 +21,6 @@
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc
Unescape View File

@ -19,8 +19,12 @@
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/engine/execution_tree.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#include "utils/system/crc32c.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc
Unescape View File

@ -26,7 +26,11 @@
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/util/task_manager.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

6
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc
Unescape View File

@ -30,8 +30,12 @@
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/system/crc32c.h"
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {
@ -385,6 +389,7 @@ Status DatasetOp::FetchRemoveSampler(std::shared_ptr<Sampler> *sampler) {
return Status::OK();
}
#ifndef ENABLE_ANDROID
uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) {
std::stringstream ss;
op->tree_->Print(ss, op);
@ -431,6 +436,7 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) {
uint32_t cache_crc = system::Crc32c::GetMaskCrc32cValue(ss_str.c_str(), ss_str.length());
return cache_crc;
}
#endif
void DatasetOp::UpdateRepeatAndEpochCounter() {
op_current_repeats_++;

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h
Unescape View File

@ -338,8 +338,10 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
/// \return Status error code
Status FetchRemoveSampler(std::shared_ptr<Sampler> *sampler);
#ifndef ENABLE_ANDROID
// Computes a CRC value for the operator
static uint32_t GenerateCRC(const std::shared_ptr<DatasetOp> &op);
#endif
/// \brief A helper templated function for casting "this" pointer to shared_ptr<derived>
/// Similar to shared_from_this, except this one will give you the derived class as shared_ptr

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc
Unescape View File

@ -22,7 +22,11 @@
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc
Unescape View File

@ -29,7 +29,11 @@
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/kernels/tensor_op.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
#include "minddata/dataset/util/task_manager.h"
namespace mindspore {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc
Unescape View File

@ -31,7 +31,11 @@
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/util/task_manager.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc
Unescape View File

@ -26,7 +26,11 @@
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc
Unescape View File

@ -25,7 +25,11 @@
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc
Unescape View File

@ -23,7 +23,11 @@
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc
Unescape View File

@ -34,7 +34,11 @@
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/status.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc
Unescape View File

@ -24,7 +24,11 @@
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc
Unescape View File

@ -30,7 +30,11 @@
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "minddata/dataset/engine/opt/pass.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc
Unescape View File

@ -22,7 +22,11 @@
#include "minddata/dataset/engine/opt/pass.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

4
mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc
Unescape View File

@ -25,8 +25,8 @@
#ifndef ENABLE_ANDROID
#include "minddata/dataset/engine/opt/pre/cache_transform_pass.h"
#include "minddata/dataset/engine/opt/post/repeat_pass.h"
#endif
#include "minddata/dataset/engine/opt/pre/cache_error_pass.h"
#endif
#include "minddata/dataset/engine/opt/pre/epoch_injection_pass.h"
#include "mindspore/ccsrc/minddata/dataset/engine/opt/optional/tensor_op_fusion_pass.h"
#include "minddata/dataset/engine/perf/profiling.h"
@ -236,7 +236,9 @@ Status ExecutionTree::PrepareTreePreAction() {
std::vector<std::unique_ptr<Pass>> pre_actions;
// Construct pre actions
MS_LOG(INFO) << "Running pre pass loops.";
#ifndef ENABLE_ANDROID
pre_actions.push_back(std::make_unique<CacheErrorPass>());
#endif
pre_actions.push_back(std::make_unique<EpochInjectionPass>());
pre_actions.push_back(std::make_unique<RemovalPass>());
#ifndef ENABLE_ANDROID

4
mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc
Unescape View File

@ -18,7 +18,11 @@
#include <string>
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
#include "mindspore/lite/src/common/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {

Some files were not shown because too many files have changed in this diff Show More

Write Preview
Loading…
Cancel
Save