m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!10513 [MS_LITE] benchamrk to master

Browse Source 
From: @YeFeng_24
Reviewed-by: @hangangqiang,@zhang_xue_tong
Signed-off-by: @hangangqiang,@zhang_xue_tong
pull/10513/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent 2ae5591bb2 115224eeba
commit 6c698421b5
2 changed files with 34 additions and 76 deletions
Show Stats Download Patch File Download Diff File

94
mindspore/lite/tools/benchmark/benchmark.cc
Unescape View File

@ -159,7 +159,8 @@ int Benchmark::ReadInputFile() {
}
// calibData is FP32
int Benchmark::ReadCalibData(bool new_data, const char *calib_data_path) {
int Benchmark::ReadCalibData() {
const char *calib_data_path = flags_->benchmark_data_file_.c_str();
// read calib data
std::ifstream in_file(calib_data_path);
if (!in_file.good()) {
@ -231,11 +232,7 @@ int Benchmark::ReadTensorData(std::ifstream &in_file_stream, const std::string &
MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name;
return RET_ERROR;
}
if (has_new_data_) {
this->new_benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
} else {
this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
}
this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
return RET_OK;
}
@ -243,51 +240,26 @@ int Benchmark::CompareOutput() {
std::cout << "================ Comparing Output data ================" << std::endl;
float total_bias = 0;
int total_size = 0;
if (new_benchmark_data_.size() > benchmark_data_.size()) {
for (const auto &calib_tensor : new_benchmark_data_) {
std::string node_or_tensor_name = calib_tensor.first;
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
return RET_ERROR;
}
int ret;
if (tensor->data_type() == kObjectTypeString) {
ret = CompareStringData(node_or_tensor_name, tensor, new_benchmark_data_);
} else {
ret =
CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, new_benchmark_data_);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
}
for (const auto &calib_tensor : benchmark_data_) {
std::string node_or_tensor_name = calib_tensor.first;
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
return RET_ERROR;
}
} else {
for (const auto &calib_tensor : benchmark_data_) {
std::string node_or_tensor_name = calib_tensor.first;
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
return RET_ERROR;
}
int ret;
if (tensor->data_type() == kObjectTypeString) {
ret = CompareStringData(node_or_tensor_name, tensor, benchmark_data_);
} else {
ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, benchmark_data_);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
}
int ret;
if (tensor->data_type() == kObjectTypeString) {
ret = CompareStringData(node_or_tensor_name, tensor);
} else {
ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
}
}
float mean_bias;
if (total_size != 0) {
mean_bias = total_bias / float_t(total_size) * 100;
@ -319,8 +291,7 @@ tensor::MSTensor *Benchmark::GetTensorByNodeOrTensorName(const std::string &node
return tensor;
}
int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor,
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor) {
auto iter = this->benchmark_data_.find(name);
if (iter != this->benchmark_data_.end()) {
std::vector<std::string> calib_strings = iter->second->strings_data;
@ -343,8 +314,7 @@ int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tens
}
int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
int *total_size,
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
int *total_size) {
float bias = 0;
auto mutableData = tensor->MutableData();
if (mutableData == nullptr) {
@ -353,19 +323,19 @@ int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::M
}
switch (msCalibDataType) {
case TypeId::kNumberTypeFloat: {
bias = CompareData<float>(name, tensor->shape(), mutableData, benchmark_data);
bias = CompareData<float>(name, tensor->shape(), mutableData);
break;
}
case TypeId::kNumberTypeInt8: {
bias = CompareData<int8_t>(name, tensor->shape(), mutableData, benchmark_data);
bias = CompareData<int8_t>(name, tensor->shape(), mutableData);
break;
}
case TypeId::kNumberTypeUInt8: {
bias = CompareData<uint8_t>(name, tensor->shape(), mutableData, benchmark_data);
bias = CompareData<uint8_t>(name, tensor->shape(), mutableData);
break;
}
case TypeId::kNumberTypeInt32: {
bias = CompareData<int32_t>(name, tensor->shape(), mutableData, benchmark_data);
bias = CompareData<int32_t>(name, tensor->shape(), mutableData);
break;
}
default:
@ -453,20 +423,12 @@ int Benchmark::MarkAccuracy() {
std::cerr << "Inference error " << status << std::endl;
return status;
}
const char *calib_data_path = flags_->benchmark_data_file_.c_str();
status = ReadCalibData(false, calib_data_path);
status = ReadCalibData();
if (status != RET_OK) {
MS_LOG(ERROR) << "Read calib data error " << status;
std::cerr << "Read calib data error " << status << std::endl;
has_new_data_ = true;
status = ReadCalibData(true, flags_->benchmark_data_file_.append("_1").c_str());
if (status != RET_OK) {
MS_LOG(ERROR) << "no new data.";
has_new_data_ = false;
return status;
}
return status;
}
status = CompareOutput();
if (status != RET_OK) {
MS_LOG(ERROR) << "Compare output error " << status;

16
mindspore/lite/tools/benchmark/benchmark.h
Unescape View File

@ -129,7 +129,7 @@ class MS_API Benchmark {
int ReadInputFile();
int ReadCalibData(bool new_data, const char *calib_data_path);
int ReadCalibData();
int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector<size_t> &dims);
@ -137,11 +137,10 @@ class MS_API Benchmark {
tensor::MSTensor *GetTensorByNodeOrTensorName(const std::string &node_or_tensor_name);
int CompareStringData(const std::string &name, tensor::MSTensor *tensor,
std::unordered_map<std::string, CheckTensor *> benchmark_data);
int CompareStringData(const std::string &name, tensor::MSTensor *tensor);
int CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
int *total_size, std::unordered_map<std::string, CheckTensor *> benchmark_data);
int *total_size);
int InitCallbackParameter();
@ -151,11 +150,10 @@ class MS_API Benchmark {
// tensorData need to be converter first
template <typename T>
float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data,
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data) {
const T *msTensorData = static_cast<const T *>(tensor_data);
auto iter = benchmark_data.find(nodeName);
if (iter != benchmark_data.end()) {
auto iter = this->benchmark_data_.find(nodeName);
if (iter != this->benchmark_data_.end()) {
std::vector<size_t> castedMSShape;
size_t shapeSize = 1;
for (int64_t dim : msShape) {
@ -240,13 +238,11 @@ class MS_API Benchmark {
int MarkAccuracy();
private:
bool has_new_data_ = false;
BenchmarkFlags *flags_;
session::LiteSession *session_{nullptr};
std::vector<mindspore::tensor::MSTensor *> ms_inputs_;
std::unordered_map<std::string, std::vector<mindspore::tensor::MSTensor *>> ms_outputs_;
std::unordered_map<std::string, CheckTensor *> benchmark_data_;
std::unordered_map<std::string, CheckTensor *> new_benchmark_data_;
std::unordered_map<std::string, TypeId> data_type_map_{{"FLOAT", TypeId::kNumberTypeFloat},
{"INT8", TypeId::kNumberTypeInt8},
{"INT32", TypeId::kNumberTypeInt32},

Write Preview
Loading…
Cancel
Save