From 115224eeba214ed1ddbf0e1bdb99adebeb3531af Mon Sep 17 00:00:00 2001 From: yefeng Date: Fri, 25 Dec 2020 15:37:51 +0800 Subject: [PATCH] 021-benchmark-to-master-1 --- mindspore/lite/tools/benchmark/benchmark.cc | 94 ++++++--------------- mindspore/lite/tools/benchmark/benchmark.h | 16 ++-- 2 files changed, 34 insertions(+), 76 deletions(-) diff --git a/mindspore/lite/tools/benchmark/benchmark.cc b/mindspore/lite/tools/benchmark/benchmark.cc index fd184ef761..018760ea18 100644 --- a/mindspore/lite/tools/benchmark/benchmark.cc +++ b/mindspore/lite/tools/benchmark/benchmark.cc @@ -159,7 +159,8 @@ int Benchmark::ReadInputFile() { } // calibData is FP32 -int Benchmark::ReadCalibData(bool new_data, const char *calib_data_path) { +int Benchmark::ReadCalibData() { + const char *calib_data_path = flags_->benchmark_data_file_.c_str(); // read calib data std::ifstream in_file(calib_data_path); if (!in_file.good()) { @@ -231,11 +232,7 @@ int Benchmark::ReadTensorData(std::ifstream &in_file_stream, const std::string & MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name; return RET_ERROR; } - if (has_new_data_) { - this->new_benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); - } else { - this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); - } + this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); return RET_OK; } @@ -243,51 +240,26 @@ int Benchmark::CompareOutput() { std::cout << "================ Comparing Output data ================" << std::endl; float total_bias = 0; int total_size = 0; - if (new_benchmark_data_.size() > benchmark_data_.size()) { - for (const auto &calib_tensor : new_benchmark_data_) { - std::string node_or_tensor_name = calib_tensor.first; - tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); - if (tensor == nullptr) { - MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; - return RET_ERROR; - } - int ret; - if (tensor->data_type() == kObjectTypeString) { - ret = CompareStringData(node_or_tensor_name, tensor, new_benchmark_data_); - } else { - ret = - CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, new_benchmark_data_); - } - if (ret != RET_OK) { - MS_LOG(ERROR) << "Error in CompareData"; - std::cerr << "Error in CompareData" << std::endl; - std::cout << "=======================================================" << std::endl << std::endl; - return ret; - } + for (const auto &calib_tensor : benchmark_data_) { + std::string node_or_tensor_name = calib_tensor.first; + tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; + return RET_ERROR; } - } else { - for (const auto &calib_tensor : benchmark_data_) { - std::string node_or_tensor_name = calib_tensor.first; - tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); - if (tensor == nullptr) { - MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; - return RET_ERROR; - } - int ret; - if (tensor->data_type() == kObjectTypeString) { - ret = CompareStringData(node_or_tensor_name, tensor, benchmark_data_); - } else { - ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, benchmark_data_); - } - if (ret != RET_OK) { - MS_LOG(ERROR) << "Error in CompareData"; - std::cerr << "Error in CompareData" << std::endl; - std::cout << "=======================================================" << std::endl << std::endl; - return ret; - } + int ret; + if (tensor->data_type() == kObjectTypeString) { + ret = CompareStringData(node_or_tensor_name, tensor); + } else { + ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size); + } + if (ret != RET_OK) { + MS_LOG(ERROR) << "Error in CompareData"; + std::cerr << "Error in CompareData" << std::endl; + std::cout << "=======================================================" << std::endl << std::endl; + return ret; } } - float mean_bias; if (total_size != 0) { mean_bias = total_bias / float_t(total_size) * 100; @@ -319,8 +291,7 @@ tensor::MSTensor *Benchmark::GetTensorByNodeOrTensorName(const std::string &node return tensor; } -int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor, - std::unordered_map benchmark_data) { +int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor) { auto iter = this->benchmark_data_.find(name); if (iter != this->benchmark_data_.end()) { std::vector calib_strings = iter->second->strings_data; @@ -343,8 +314,7 @@ int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tens } int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias, - int *total_size, - std::unordered_map benchmark_data) { + int *total_size) { float bias = 0; auto mutableData = tensor->MutableData(); if (mutableData == nullptr) { @@ -353,19 +323,19 @@ int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::M } switch (msCalibDataType) { case TypeId::kNumberTypeFloat: { - bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); + bias = CompareData(name, tensor->shape(), mutableData); break; } case TypeId::kNumberTypeInt8: { - bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); + bias = CompareData(name, tensor->shape(), mutableData); break; } case TypeId::kNumberTypeUInt8: { - bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); + bias = CompareData(name, tensor->shape(), mutableData); break; } case TypeId::kNumberTypeInt32: { - bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); + bias = CompareData(name, tensor->shape(), mutableData); break; } default: @@ -453,20 +423,12 @@ int Benchmark::MarkAccuracy() { std::cerr << "Inference error " << status << std::endl; return status; } - const char *calib_data_path = flags_->benchmark_data_file_.c_str(); - status = ReadCalibData(false, calib_data_path); + status = ReadCalibData(); if (status != RET_OK) { MS_LOG(ERROR) << "Read calib data error " << status; std::cerr << "Read calib data error " << status << std::endl; - has_new_data_ = true; - status = ReadCalibData(true, flags_->benchmark_data_file_.append("_1").c_str()); - if (status != RET_OK) { - MS_LOG(ERROR) << "no new data."; - has_new_data_ = false; - return status; - } + return status; } - status = CompareOutput(); if (status != RET_OK) { MS_LOG(ERROR) << "Compare output error " << status; diff --git a/mindspore/lite/tools/benchmark/benchmark.h b/mindspore/lite/tools/benchmark/benchmark.h index df38d690a2..cf8d97a954 100644 --- a/mindspore/lite/tools/benchmark/benchmark.h +++ b/mindspore/lite/tools/benchmark/benchmark.h @@ -129,7 +129,7 @@ class MS_API Benchmark { int ReadInputFile(); - int ReadCalibData(bool new_data, const char *calib_data_path); + int ReadCalibData(); int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector &dims); @@ -137,11 +137,10 @@ class MS_API Benchmark { tensor::MSTensor *GetTensorByNodeOrTensorName(const std::string &node_or_tensor_name); - int CompareStringData(const std::string &name, tensor::MSTensor *tensor, - std::unordered_map benchmark_data); + int CompareStringData(const std::string &name, tensor::MSTensor *tensor); int CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias, - int *total_size, std::unordered_map benchmark_data); + int *total_size); int InitCallbackParameter(); @@ -151,11 +150,10 @@ class MS_API Benchmark { // tensorData need to be converter first template - float CompareData(const std::string &nodeName, const std::vector &msShape, const void *tensor_data, - std::unordered_map benchmark_data) { + float CompareData(const std::string &nodeName, const std::vector &msShape, const void *tensor_data) { const T *msTensorData = static_cast(tensor_data); - auto iter = benchmark_data.find(nodeName); - if (iter != benchmark_data.end()) { + auto iter = this->benchmark_data_.find(nodeName); + if (iter != this->benchmark_data_.end()) { std::vector castedMSShape; size_t shapeSize = 1; for (int64_t dim : msShape) { @@ -240,13 +238,11 @@ class MS_API Benchmark { int MarkAccuracy(); private: - bool has_new_data_ = false; BenchmarkFlags *flags_; session::LiteSession *session_{nullptr}; std::vector ms_inputs_; std::unordered_map> ms_outputs_; std::unordered_map benchmark_data_; - std::unordered_map new_benchmark_data_; std::unordered_map data_type_map_{{"FLOAT", TypeId::kNumberTypeFloat}, {"INT8", TypeId::kNumberTypeInt8}, {"INT32", TypeId::kNumberTypeInt32},