m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!10513 [MS_LITE] benchamrk to master

Browse Source 
From: @YeFeng_24
Reviewed-by: @hangangqiang,@zhang_xue_tong
Signed-off-by: @hangangqiang,@zhang_xue_tong
pull/10513/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent 2ae5591bb2 115224eeba
commit 6c698421b5
2 changed files with 34 additions and 76 deletions
Show Stats Download Patch File Download Diff File

94
mindspore/lite/tools/benchmark/benchmark.cc
Unescape View File

@ -159,7 +159,8 @@ int Benchmark::ReadInputFile() {
} }
// calibData is FP32 // calibData is FP32
int Benchmark::ReadCalibData(bool new_data, const char *calib_data_path) { int Benchmark::ReadCalibData() {
const char *calib_data_path = flags_->benchmark_data_file_.c_str();
// read calib data // read calib data
std::ifstream in_file(calib_data_path); std::ifstream in_file(calib_data_path);
if (!in_file.good()) { if (!in_file.good()) {
@ -231,11 +232,7 @@ int Benchmark::ReadTensorData(std::ifstream &in_file_stream, const std::string &
MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name; MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name;
return RET_ERROR; return RET_ERROR;
} }
if (has_new_data_) { this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
this->new_benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
} else {
this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
}
return RET_OK; return RET_OK;
} }
@ -243,51 +240,26 @@ int Benchmark::CompareOutput() {
std::cout << "================ Comparing Output data ================" << std::endl; std::cout << "================ Comparing Output data ================" << std::endl;
float total_bias = 0; float total_bias = 0;
int total_size = 0; int total_size = 0;
if (new_benchmark_data_.size() > benchmark_data_.size()) { for (const auto &calib_tensor : benchmark_data_) {
for (const auto &calib_tensor : new_benchmark_data_) { std::string node_or_tensor_name = calib_tensor.first;
std::string node_or_tensor_name = calib_tensor.first; tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); if (tensor == nullptr) {
if (tensor == nullptr) { MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; return RET_ERROR;
return RET_ERROR;
}
int ret;
if (tensor->data_type() == kObjectTypeString) {
ret = CompareStringData(node_or_tensor_name, tensor, new_benchmark_data_);
} else {
ret =
CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, new_benchmark_data_);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
}
} }
} else { int ret;
for (const auto &calib_tensor : benchmark_data_) { if (tensor->data_type() == kObjectTypeString) {
std::string node_or_tensor_name = calib_tensor.first; ret = CompareStringData(node_or_tensor_name, tensor);
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); } else {
if (tensor == nullptr) { ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size);
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; }
return RET_ERROR; if (ret != RET_OK) {
} MS_LOG(ERROR) << "Error in CompareData";
int ret; std::cerr << "Error in CompareData" << std::endl;
if (tensor->data_type() == kObjectTypeString) { std::cout << "=======================================================" << std::endl << std::endl;
ret = CompareStringData(node_or_tensor_name, tensor, benchmark_data_); return ret;
} else {
ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, benchmark_data_);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
}
} }
} }
float mean_bias; float mean_bias;
if (total_size != 0) { if (total_size != 0) {
mean_bias = total_bias / float_t(total_size) * 100; mean_bias = total_bias / float_t(total_size) * 100;
@ -319,8 +291,7 @@ tensor::MSTensor *Benchmark::GetTensorByNodeOrTensorName(const std::string &node
return tensor; return tensor;
} }
int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor, int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor) {
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
auto iter = this->benchmark_data_.find(name); auto iter = this->benchmark_data_.find(name);
if (iter != this->benchmark_data_.end()) { if (iter != this->benchmark_data_.end()) {
std::vector<std::string> calib_strings = iter->second->strings_data; std::vector<std::string> calib_strings = iter->second->strings_data;
@ -343,8 +314,7 @@ int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tens
} }
int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias, int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
int *total_size, int *total_size) {
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
float bias = 0; float bias = 0;
auto mutableData = tensor->MutableData(); auto mutableData = tensor->MutableData();
if (mutableData == nullptr) { if (mutableData == nullptr) {
@ -353,19 +323,19 @@ int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::M
} }
switch (msCalibDataType) { switch (msCalibDataType) {
case TypeId::kNumberTypeFloat: { case TypeId::kNumberTypeFloat: {
bias = CompareData<float>(name, tensor->shape(), mutableData, benchmark_data); bias = CompareData<float>(name, tensor->shape(), mutableData);
break; break;
} }
case TypeId::kNumberTypeInt8: { case TypeId::kNumberTypeInt8: {
bias = CompareData<int8_t>(name, tensor->shape(), mutableData, benchmark_data); bias = CompareData<int8_t>(name, tensor->shape(), mutableData);
break; break;
} }
case TypeId::kNumberTypeUInt8: { case TypeId::kNumberTypeUInt8: {
bias = CompareData<uint8_t>(name, tensor->shape(), mutableData, benchmark_data); bias = CompareData<uint8_t>(name, tensor->shape(), mutableData);
break; break;
} }
case TypeId::kNumberTypeInt32: { case TypeId::kNumberTypeInt32: {
bias = CompareData<int32_t>(name, tensor->shape(), mutableData, benchmark_data); bias = CompareData<int32_t>(name, tensor->shape(), mutableData);
break; break;
} }
default: default:
@ -453,20 +423,12 @@ int Benchmark::MarkAccuracy() {
std::cerr << "Inference error " << status << std::endl; std::cerr << "Inference error " << status << std::endl;
return status; return status;
} }
const char *calib_data_path = flags_->benchmark_data_file_.c_str(); status = ReadCalibData();
status = ReadCalibData(false, calib_data_path);
if (status != RET_OK) { if (status != RET_OK) {
MS_LOG(ERROR) << "Read calib data error " << status; MS_LOG(ERROR) << "Read calib data error " << status;
std::cerr << "Read calib data error " << status << std::endl; std::cerr << "Read calib data error " << status << std::endl;
has_new_data_ = true; return status;
status = ReadCalibData(true, flags_->benchmark_data_file_.append("_1").c_str());
if (status != RET_OK) {
MS_LOG(ERROR) << "no new data.";
has_new_data_ = false;
return status;
}
} }
status = CompareOutput(); status = CompareOutput();
if (status != RET_OK) { if (status != RET_OK) {
MS_LOG(ERROR) << "Compare output error " << status; MS_LOG(ERROR) << "Compare output error " << status;

16
mindspore/lite/tools/benchmark/benchmark.h
Unescape View File

@ -129,7 +129,7 @@ class MS_API Benchmark {
int ReadInputFile(); int ReadInputFile();
int ReadCalibData(bool new_data, const char *calib_data_path); int ReadCalibData();
int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector<size_t> &dims); int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector<size_t> &dims);
@ -137,11 +137,10 @@ class MS_API Benchmark {
tensor::MSTensor *GetTensorByNodeOrTensorName(const std::string &node_or_tensor_name); tensor::MSTensor *GetTensorByNodeOrTensorName(const std::string &node_or_tensor_name);
int CompareStringData(const std::string &name, tensor::MSTensor *tensor, int CompareStringData(const std::string &name, tensor::MSTensor *tensor);
std::unordered_map<std::string, CheckTensor *> benchmark_data);
int CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias, int CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
int *total_size, std::unordered_map<std::string, CheckTensor *> benchmark_data); int *total_size);
int InitCallbackParameter(); int InitCallbackParameter();
@ -151,11 +150,10 @@ class MS_API Benchmark {
// tensorData need to be converter first // tensorData need to be converter first
template <typename T> template <typename T>
float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data, float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data) {
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
const T *msTensorData = static_cast<const T *>(tensor_data); const T *msTensorData = static_cast<const T *>(tensor_data);
auto iter = benchmark_data.find(nodeName); auto iter = this->benchmark_data_.find(nodeName);
if (iter != benchmark_data.end()) { if (iter != this->benchmark_data_.end()) {
std::vector<size_t> castedMSShape; std::vector<size_t> castedMSShape;
size_t shapeSize = 1; size_t shapeSize = 1;
for (int64_t dim : msShape) { for (int64_t dim : msShape) {
@ -240,13 +238,11 @@ class MS_API Benchmark {
int MarkAccuracy(); int MarkAccuracy();
private: private:
bool has_new_data_ = false;
BenchmarkFlags *flags_; BenchmarkFlags *flags_;
session::LiteSession *session_{nullptr}; session::LiteSession *session_{nullptr};
std::vector<mindspore::tensor::MSTensor *> ms_inputs_; std::vector<mindspore::tensor::MSTensor *> ms_inputs_;
std::unordered_map<std::string, std::vector<mindspore::tensor::MSTensor *>> ms_outputs_; std::unordered_map<std::string, std::vector<mindspore::tensor::MSTensor *>> ms_outputs_;
std::unordered_map<std::string, CheckTensor *> benchmark_data_; std::unordered_map<std::string, CheckTensor *> benchmark_data_;
std::unordered_map<std::string, CheckTensor *> new_benchmark_data_;
std::unordered_map<std::string, TypeId> data_type_map_{{"FLOAT", TypeId::kNumberTypeFloat}, std::unordered_map<std::string, TypeId> data_type_map_{{"FLOAT", TypeId::kNumberTypeFloat},
{"INT8", TypeId::kNumberTypeInt8}, {"INT8", TypeId::kNumberTypeInt8},
{"INT32", TypeId::kNumberTypeInt32}, {"INT32", TypeId::kNumberTypeInt32},

Write Preview
Loading…
Cancel
Save