m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!2971 Reg op info from local config file

Browse Source 
Merge pull request !2971 from jjfeing/master
pull/2971/MERGE
mindspore-ci-bot 5 years ago committed by Gitee
parent 761cdbfade 53277f8c02
commit 7fd22f8ede
5 changed files with 454 additions and 21 deletions
Show Stats Download Patch File Download Diff File

383
config/op_info.config
View File

File diff suppressed because one or more lines are too long

6
mindspore/ccsrc/kernel/oplib/opinfo.h
Unescape View File

@ -103,13 +103,13 @@ class OpInfo {
partial_flag_ = opinfo.partial_flag_;
dynamic_format_ = opinfo.dynamic_format_;
op_pattern_ = opinfo.op_pattern();
for (auto attr : opinfo.attrs_ptr()) {
for (const auto &attr : opinfo.attrs_ptr()) {
attrs_ptr_.push_back(std::make_shared<OpAttr>(*attr));
}
for (auto input : opinfo.inputs_ptr()) {
for (const auto &input : opinfo.inputs_ptr()) {
inputs_ptr_.push_back(std::make_shared<OpIOInfo>(*input));
}
for (auto output : opinfo.outputs_ptr()) {
for (const auto &output : opinfo.outputs_ptr()) {
outputs_ptr_.push_back(std::make_shared<OpIOInfo>(*output));
}
ref_infos_ = opinfo.ref_infos();

76
mindspore/ccsrc/kernel/oplib/oplib.cc
Unescape View File

@ -19,6 +19,7 @@
#include <unordered_map>
#include <memory>
#include <map>
#include <fstream>
#include "utils/log_adapter.h"
#include "utils/overload.h"
#include "utils/context/ms_context.h"
@ -59,7 +60,7 @@ constexpr auto kNeedCompile = "need_compile";
constexpr auto kShape = "shape";
std::vector<std::shared_ptr<OpInfo>> OpLib::op_info_;
std::string ImplTypeToStr(OpImplyType impl_type) {
static std::string ImplTypeToStr(OpImplyType impl_type) {
switch (impl_type) {
case kTBE:
return kTbe;
@ -124,6 +125,50 @@ void OpLib::DecodeTBESpecificInfo(const nlohmann::json &obj, const std::shared_p
}
}
bool OpLib::RegOpFromLocalInfo() {
MS_LOG(INFO) << "Start";
static bool has_load = false;
if (has_load) {
return true;
}
has_load = true;
std::string dir = common::GetEnv("MINDSPORE_OP_INFO_PATH");
if (dir.empty()) {
MS_LOG(INFO) << "MindSpore op info path does not been setted. use op info from python pass.";
return true;
}
char real_path[PATH_MAX] = {0};
if (dir.size() >= PATH_MAX) {
MS_LOG(ERROR) << "Op info path is invalid: " << dir;
return false;
}
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(real_path, common::SafeCStr(dir), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Op info path is invalid: " << dir;
return false;
}
#else
if (realpath(common::SafeCStr(dir), real_path) == nullptr) {
MS_LOG(ERROR) << "Op info path is invalid: " << dir;
return false;
}
#endif
MS_LOG(INFO) << "Start to read op info from local file.";
std::ifstream file(real_path);
if (!file.is_open()) {
MS_LOG(ERROR) << "Find op info file failed.";
return false;
}
std::string line;
while (getline(file, line)) {
if (!line.empty()) {
(void)OpLib::RegOp(line, "");
}
}
MS_LOG(INFO) << "End";
return true;
}
bool OpLib::DecodeOpInfo(const nlohmann::json &obj, const mindspore::kernel::OpImplyType imply_type,
const std::string &impl_path) {
std::shared_ptr<OpInfo> op_info = std::make_shared<OpInfo>();
@ -160,14 +205,16 @@ bool OpLib::DecodeOpInfo(const nlohmann::json &obj, const mindspore::kernel::OpI
return false;
}
}
if (CheckRepetition(op_info)) {
MS_LOG(WARNING) << "This op info has been already registed. op name: " << op_info->op_name()
<< ", impl type: " << ImplTypeToStr(op_info->imply_type())
<< ", impl path: " << op_info->impl_path();
return true;
}
if (!GetRefInfo(op_info)) {
MS_LOG(ERROR) << "GetRefInfo Failed";
return false;
}
if (!CheckRepetition(op_info)) {
MS_LOG(ERROR) << "CheckRepetition Failed";
return false;
}
op_info_.push_back(op_info);
return true;
}
@ -269,6 +316,9 @@ bool OpLib::DecodeInputOutput(const nlohmann::json &obj, const OpImplyType imply
}
std::shared_ptr<OpInfo> OpLib::FindOp(const std::string &op_name, OpImplyType imply_type) {
if (!OpLib::RegOpFromLocalInfo()) {
MS_LOG(INFO) << "Warning reg local op info failed.";
}
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
bool is_gpu = (context->device_target() == kGPUDevice);
@ -283,8 +333,8 @@ std::shared_ptr<OpInfo> OpLib::FindOp(const std::string &op_name, OpImplyType im
return op_info;
}
}
MS_LOG(DEBUG) << "FindOp failed: opname: " << op_name << ", imply_type: " << ImplTypeToStr(imply_type)
<< ", current op num: " << op_info_.size();
MS_LOG(INFO) << "FindOp failed: opname: " << op_name << ", imply_type: " << ImplTypeToStr(imply_type)
<< ", current op num: " << op_info_.size();
return nullptr;
}
@ -313,17 +363,19 @@ bool OpLib::GetRefInfo(const std::shared_ptr<OpInfo> &op_info) {
}
bool OpLib::CheckRepetition(const std::shared_ptr<OpInfo> &op_info) {
bool has_register = false;
MS_EXCEPTION_IF_NULL(op_info);
for (const auto &exist_op_info : op_info_) {
MS_EXCEPTION_IF_NULL(exist_op_info);
if (exist_op_info->op_name() == op_info->op_name() && exist_op_info->imply_type() == op_info->imply_type() &&
exist_op_info->impl_path() != op_info->impl_path()) {
MS_LOG(ERROR) << "Op has already exist, please use other name, op name: " << op_info->op_name()
<< " op type: " << ImplTypeToStr(op_info->imply_type());
return false;
exist_op_info->impl_path() == op_info->impl_path()) {
MS_LOG(INFO) << "Op has already exist, please use other name, op name: " << op_info->op_name()
<< " op type: " << ImplTypeToStr(op_info->imply_type());
has_register = true;
break;
}
}
return true;
return has_register;
}
} // namespace kernel
} // namespace mindspore

8
mindspore/ccsrc/kernel/oplib/oplib.h
Unescape View File

@ -28,11 +28,8 @@ class OpLib {
public:
OpLib() = default;
virtual ~OpLib() = default;
bool RegOp(const std::string &json_string, const std::string &impl_path);
static void RegOpInfo(std::shared_ptr<OpInfo> opinfo) {
op_info_.emplace_back(opinfo);
return;
}
static bool RegOp(const std::string &json_string, const std::string &impl_path);
static void RegOpInfo(const std::shared_ptr<OpInfo> &opinfo) { op_info_.emplace_back(opinfo); }
static std::shared_ptr<OpInfo> FindOp(const std::string &op_name, OpImplyType imply_type);
static const std::vector<std::shared_ptr<OpInfo>> &GetAllOpsInfo() { return op_info_; }
@ -40,6 +37,7 @@ class OpLib {
static std::vector<std::shared_ptr<OpInfo>> op_info_;
private:
static bool RegOpFromLocalInfo();
static bool DecodeOpInfo(const nlohmann::json &obj, const OpImplyType imply_type, const std::string &impl_path);
static bool DecodeAttr(const nlohmann::json &obj, const OpImplyType imply_type,
const std::shared_ptr<OpInfo> &op_info);

2
mindspore/ccsrc/pipeline/init.cc
Unescape View File

@ -323,7 +323,7 @@ PYBIND11_MODULE(_c_expression, m) {
(void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib")
.def(py::init())
.def("reg_op", &OpLib::RegOp, "Register op info.");
.def_static("reg_op", &OpLib::RegOp, "Register op info.");
#ifdef ENABLE_GPU_COLLECTIVE
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective,
"Init gpu collective communication mode.");

Write Preview
Loading…
Cancel
Save