make backend/optimizer pybind free

pull/5312/head
zhousiyi 5 years ago
parent 44784461e1
commit c25e37e7bf

@ -108,7 +108,7 @@
#include "utils/ms_context.h"
#include "utils/config_manager.h"
#include "debug/anf_ir_dump.h"
#include "debug/anf_ir_utils.h"
#include "debug/dump_proto.h"
namespace mindspore {
namespace opt {

@ -34,7 +34,7 @@
#include "runtime/device/ascend/ascend_stream_assign.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "debug/anf_ir_dump.h"
#include "debug/anf_ir_utils.h"
#include "debug/dump_proto.h"
#include "utils/ms_utils.h"
#include "backend/optimizer/common/helper.h"
#include "runtime/device/kernel_runtime_manager.h"

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debug/anf_ir_utils.h"
#include "debug/dump_proto.h"
#include "backend/session/gpu_session.h"
#include "runtime/device/gpu/kernel_info_setter.h"
#include "runtime/device/gpu/gpu_kernel_build.h"

@ -2258,64 +2258,4 @@ std::vector<FuncGraphPtr> ImportIR(const std::string &filename) {
parser.ParseFile();
return parser.GetFuncGraphs();
}
#ifdef ENABLE_DUMP_IR
void DumpIRProto(const FuncGraphPtr &func_graph, const std::string &suffix) {
if (func_graph == nullptr) {
MS_LOG(ERROR) << "Func graph is nullptr";
return;
}
auto ms_context = MsContext::GetInstance();
if (ms_context == nullptr) {
MS_LOG(ERROR) << "ms_context is nullptr";
return;
}
auto save_graphs_path = ms_context->save_graphs_path();
if (save_graphs_path.empty()) {
save_graphs_path = ".";
}
std::string file_path = save_graphs_path + "/" + "ms_output_" + suffix + ".pb";
if (file_path.size() > PATH_MAX) {
MS_LOG(ERROR) << "File path " << file_path << " is too long.";
return;
}
char real_path[PATH_MAX] = {0};
char *real_path_ret = nullptr;
#if defined(_WIN32) || defined(_WIN64)
real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX);
#else
real_path_ret = realpath(file_path.c_str(), real_path);
#endif
if (nullptr == real_path_ret) {
MS_LOG(DEBUG) << "dir " << file_path << " does not exit.";
} else {
std::string path_string = real_path;
if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) {
MS_LOG(ERROR) << "Modify file:" << real_path << " to rw fail.";
return;
}
}
// write to pb file
std::ofstream ofs(real_path);
if (!ofs.is_open()) {
MS_LOG(ERROR) << "Open file '" << real_path << "' failed!";
return;
}
ofs << GetFuncGraphProtoString(func_graph);
ofs.close();
// set file mode to read only by user
ChangeFileMode(file_path, S_IRUSR);
}
#else
void DumpIRProto(const FuncGraphPtr &, const std::string &) {
static bool already_printed = false;
if (already_printed) {
return;
}
already_printed = true;
MS_LOG(WARNING) << "The functionality of dumping function graph IR in protobuf format is disabled, "
<< "please recompile source to enable it. See help of building script.";
}
#endif
} // namespace mindspore

@ -112,14 +112,6 @@ void ExportIR(const std::string &filename, const std::string &id, const FuncGrap
void ExportIR(const std::string &filename, const std::vector<TaggedGraph> &graphs);
std::vector<FuncGraphPtr> ImportIR(const std::string &filename);
std::string GetFuncGraphProtoString(const FuncGraphPtr &func_graph);
void DumpIRProto(const FuncGraphPtr &func_graph, const std::string &suffix);
std::string GetOnnxProtoString(const FuncGraphPtr &func_graph);
std::string GetBinaryProtoString(const FuncGraphPtr &func_graph);
} // namespace mindspore
#endif // MINDSPORE_CCSRC_DEBUG_ANF_IR_UTILS_H_

@ -13,16 +13,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debug/dump_proto.h"
#include <algorithm>
#include <fstream>
#include <map>
#include <memory>
#include <utility>
#include <algorithm>
#include <vector>
#include "debug/anf_ir_utils.h"
#include "proto/anf_ir.pb.h"
#include "ir/graph_utils.h"
#include "utils/ms_context.h"
#include "utils/symbolic.h"
#include "utils/utils.h"
namespace mindspore {
class ProtoExporter {
@ -514,4 +518,64 @@ std::string GetFuncGraphProtoString(const FuncGraphPtr &func_graph) {
ProtoExporter exporter;
return exporter.GetFuncGraphProtoString(func_graph);
}
#ifdef ENABLE_DUMP_IR
void DumpIRProto(const FuncGraphPtr &func_graph, const std::string &suffix) {
if (func_graph == nullptr) {
MS_LOG(ERROR) << "Func graph is nullptr";
return;
}
auto ms_context = MsContext::GetInstance();
if (ms_context == nullptr) {
MS_LOG(ERROR) << "ms_context is nullptr";
return;
}
auto save_graphs_path = ms_context->save_graphs_path();
if (save_graphs_path.empty()) {
save_graphs_path = ".";
}
std::string file_path = save_graphs_path + "/" + "ms_output_" + suffix + ".pb";
if (file_path.size() > PATH_MAX) {
MS_LOG(ERROR) << "File path " << file_path << " is too long.";
return;
}
char real_path[PATH_MAX] = {0};
char *real_path_ret = nullptr;
#if defined(_WIN32) || defined(_WIN64)
real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX);
#else
real_path_ret = realpath(file_path.c_str(), real_path);
#endif
if (nullptr == real_path_ret) {
MS_LOG(DEBUG) << "dir " << file_path << " does not exit.";
} else {
std::string path_string = real_path;
if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) {
MS_LOG(ERROR) << "Modify file:" << real_path << " to rw fail.";
return;
}
}
// write to pb file
std::ofstream ofs(real_path);
if (!ofs.is_open()) {
MS_LOG(ERROR) << "Open file '" << real_path << "' failed!";
return;
}
ofs << GetFuncGraphProtoString(func_graph);
ofs.close();
// set file mode to read only by user
ChangeFileMode(file_path, S_IRUSR);
}
#else
void DumpIRProto(const FuncGraphPtr &, const std::string &) {
static bool already_printed = false;
if (already_printed) {
return;
}
already_printed = true;
MS_LOG(WARNING) << "The functionality of dumping function graph IR in protobuf format is disabled, "
<< "please recompile source to enable it. See help of building script.";
}
#endif
} // namespace mindspore

@ -0,0 +1,33 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_DEBUG_DUMP_PROTO_H_
#define MINDSPORE_CCSRC_DEBUG_DUMP_PROTO_H_
#include <string>
#include "ir/func_graph.h"
namespace mindspore {
std::string GetFuncGraphProtoString(const FuncGraphPtr &func_graph);
std::string GetOnnxProtoString(const FuncGraphPtr &func_graph);
std::string GetBinaryProtoString(const FuncGraphPtr &func_graph);
void DumpIRProto(const FuncGraphPtr &func_graph, const std::string &suffix);
} // namespace mindspore
#endif // MINDSPORE_CCSRC_DEBUG_DUMP_PROTO_H_

@ -27,6 +27,8 @@
#include "pybind_api/ir/tensor_py.h"
#include "frontend/operator/ops.h"
#include "abstract/infer_functions.h"
#include "utils/convert_utils_py.h"
namespace mindspore {
namespace abstract {
enum State {

@ -17,10 +17,14 @@
*/
#include "frontend/optimizer/cse.h"
#include <vector>
#include <set>
#include <unordered_map>
#include "abstract/abstract_function.h"
#include "utils/flags.h"
namespace mindspore {
/* namespace to support opt */
namespace opt {

@ -24,23 +24,16 @@
#include <memory>
#include "ir/anf.h"
#include "ir/manager.h"
#include "frontend/optimizer/optimizer.h"
namespace mindspore {
/* namespace to support opt */
namespace opt {
// Common subexpression elimination.
class CSE {
public:
explicit CSE(bool report_changes = true) : report_changes_(report_changes) {}
CSE() = default;
virtual ~CSE() = default;
bool operator()(const FuncGraphPtr &root, const OptimizerPtr &optimizer) {
bool chg = Cse(root, optimizer->resource()->manager());
return chg && report_changes_;
}
virtual bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool check_side_effect = true) const;
virtual bool CheckRandomEffect(const AnfNodePtr &main, const AnfNodePtr &node) const;
@ -51,7 +44,6 @@ class CSE {
bool BuildOrderGroupAndDoReplace(const FuncGraphManagerPtr manager) const;
bool DoReplace(const FuncGraphManagerPtr manager, const std::vector<std::size_t> &order_group,
std::unordered_map<std::size_t, std::vector<AnfNodePtr>> *groups) const;
bool report_changes_;
};
BasePtr AbsOf(const AnfNodePtr &node);

@ -0,0 +1,53 @@
/**
* This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/).
*
* Copyright 2019-2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_FRONTEND_OPTIMIZER_CSE_PASS_H_
#define MINDSPORE_CCSRC_FRONTEND_OPTIMIZER_CSE_PASS_H_
#include <vector>
#include <unordered_map>
#include <memory>
#include "frontend/optimizer/cse.h"
#include "frontend/optimizer/optimizer.h"
namespace mindspore {
/* namespace to support opt */
namespace opt {
// Common subexpression elimination.
class CSEPass : public CSE {
public:
explicit CSEPass(bool report_changes = true) : CSE(), report_changes_(report_changes) {}
virtual ~CSEPass() = default;
bool operator()(const FuncGraphPtr &root, const OptimizerPtr &optimizer) {
bool chg = Cse(root, optimizer->resource()->manager());
return chg && report_changes_;
}
private:
bool report_changes_;
};
BasePtr AbsOf(const AnfNodePtr &node);
} // namespace opt
} // namespace mindspore
#endif // MINDSPORE_CCSRC_FRONTEND_OPTIMIZER_CSE_PASS_H_

@ -21,6 +21,7 @@
#include <string>
#include "pipeline/jit/parse/python_adapter.h"
#include "utils/convert_utils_py.h"
using mindspore::tensor::Tensor;

@ -28,7 +28,7 @@
#include "pipeline/jit/validator.h"
#include "pipeline/jit/remove_value_node_dup.h"
#include "frontend/optimizer/optimizer.h"
#include "frontend/optimizer/cse.h"
#include "frontend/optimizer/cse_pass.h"
#include "frontend/optimizer/graph_kernel_reuse.h"
#include "frontend/optimizer/clean.h"
#include "frontend/optimizer/irpass.h"
@ -158,7 +158,7 @@ OptPassGroupMap GetOptPassesA(const opt::irpass::OptimizeIRPassLib &irpass) {
{"resolve", resolve_pass},
{"a_after_grad", a_after_grad},
{"renormalize", opt::OptPassConfig::Renormalize()},
{"cse", opt::OptPassConfig(opt::CSE(false))},
{"cse", opt::OptPassConfig(opt::CSEPass(false))},
{"a_3", a_3}});
return map_a;
@ -192,7 +192,7 @@ OptPassGroupMap GetOptPassesB(const opt::irpass::OptimizeIRPassLib &irpass) {
{"b_1", b_1},
{"b_2", b_2},
{"renormalize", opt::OptPassConfig::Renormalize()},
{"cse", opt::OptPassConfig(opt::CSE(false))},
{"cse", opt::OptPassConfig(opt::CSEPass(false))},
});
return map;
}
@ -205,7 +205,7 @@ OptPassGroupMap GetOptPassesGraphKernelA(const opt::irpass::OptimizeIRPassLib &i
{"graph_kernel_reuse", opt::OptPassConfig(opt::GraphKernelReuse())},
{"interface_fusion", interface_fusion},
{"renormalize", opt::OptPassConfig::Renormalize()},
{"cse", opt::OptPassConfig(opt::CSE(false))},
{"cse", opt::OptPassConfig(opt::CSEPass(false))},
});
return map;
}

@ -29,9 +29,11 @@
#include "pipeline/jit/parse/data_converter.h"
#include "frontend/optimizer/ad/dfunctor.h"
#include "debug/anf_ir_dump.h"
#include "debug/dump_proto.h"
#include "debug/anf_ir_utils.h"
#include "utils/config_manager.h"
#include "utils/convert_utils.h"
#include "utils/convert_utils_py.h"
#include "utils/context/context_extends.h"
#include "vm/segment_runner.h"
#include "frontend/parallel/context.h"

@ -30,6 +30,7 @@
#include "transform/graph_ir/graph_runner.h"
#include "debug/draw.h"
#include "abstract/abstract_value.h"
#include "utils/convert_utils_py.h"
namespace mindspore {
namespace pipeline {

@ -17,11 +17,13 @@
#include "pipeline/jit/remove_value_node_dup.h"
#include "ir/anf.h"
#include "ir/func_graph.h"
#include "ir/tensor.h"
#include "ir/manager.h"
#include "frontend/optimizer/cse.h"
#include "utils/log_adapter.h"
#include "utils/hashing.h"
#include "utils/convert_utils.h"
namespace mindspore {
namespace pipeline {

@ -34,6 +34,7 @@
#include "pipeline/jit/resource.h"
#include "pipeline/jit/parse/resolve.h"
#include "utils/convert_utils.h"
#include "utils/convert_utils_py.h"
#include "utils/ms_context.h"
#include "pipeline/jit/parse/data_converter.h"
#include "abstract/primitive_infer_map.h"

@ -30,6 +30,7 @@
#include "utils/ms_context.h"
#include "utils/context/context_extends.h"
#include "utils/config_manager.h"
#include "utils/convert_utils_py.h"
#include "frontend/operator/ops.h"
#include "frontend/operator/composite/composite.h"
#include "frontend/operator/composite/do_signature.h"

@ -21,6 +21,7 @@
#include "pipeline/jit/parse/data_converter.h"
#include "pybind11/pytypes.h"
#include "utils/convert_utils_base.h"
#include "utils/convert_utils_py.h"
#include "utils/primitive_utils.h"
#include "utils/base_ref_extends.h"
#include "utils/ms_context.h"

@ -21,6 +21,13 @@
#include <thread>
#include <atomic>
#include "pybind11/pybind11.h"
#include "utils/ms_utils.h"
#include "utils/convert_utils_base.h"
namespace py = pybind11;
namespace mindspore {
namespace context {
#ifdef ENABLE_GE

@ -22,7 +22,6 @@
#include <memory>
#include "utils/ms_context.h"
#include "utils/tensorprint_utils.h"
#include "utils/convert_utils.h"
#ifndef NO_DLIB
#include "tdt/tsd_client.h"

File diff suppressed because it is too large Load Diff

@ -25,14 +25,12 @@
#include <unordered_map>
#include <unordered_set>
#include "pybind11/pybind11.h"
#include "utils/convert_utils_base.h"
#include "utils/any.h"
#include "base/base_ref.h"
#include "base/base.h"
#include "ir/anf.h"
namespace py = pybind11;
#include "ir/func_graph.h"
namespace mindspore {
namespace tensor {
@ -40,19 +38,9 @@ class Tensor;
using TensorPtr = std::shared_ptr<Tensor>;
} // namespace tensor
py::object AnyToPyData(const Any &value);
py::object BaseRefToPyData(const BaseRef &value);
bool BaseRefToBool(const BaseRef &in, bool *out);
bool BaseRefToInt(const ValuePtr &v, int *value);
bool ValueToBool(const ValuePtr &in, bool *out);
py::object ValuePtrToPyData(const ValuePtr &value);
AbstractBasePtr PyListDtype2AbstractTensor(const py::object &shape_obj, const py::object &type_obj,
const py::object &min_shape = py::none(),
const py::object &max_shape = py::none());
bool IsGraphOutputValueNodeOrParameter(const AnfNodePtr &output, const py::tuple &args,
const std::shared_ptr<py::object> &ret_val);
// Isomorphism
struct PairHasher {

File diff suppressed because it is too large Load Diff

@ -0,0 +1,43 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_UTILS_CONVERT_UTILS_PY_H_
#define MINDSPORE_CCSRC_UTILS_CONVERT_UTILS_PY_H_
#include <memory>
#include "pybind11/pybind11.h"
#include "utils/convert_utils_base.h"
#include "utils/any.h"
#include "utils/base_ref_extends.h"
#include "ir/anf.h"
namespace py = pybind11;
namespace mindspore {
py::object AnyToPyData(const Any &value);
py::object BaseRefToPyData(const BaseRef &value);
py::object ValuePtrToPyData(const ValuePtr &value);
bool IsGraphOutputValueNodeOrParameter(const AnfNodePtr &output, const py::tuple &args,
const std::shared_ptr<py::object> &ret_val);
AbstractBasePtr PyListDtype2AbstractTensor(const py::object &shape_obj, const py::object &type_obj,
const py::object &min_shape = py::none(),
const py::object &max_shape = py::none());
} // namespace mindspore
#endif // MINDSPORE_CCSRC_UTILS_CONVERT_UTILS_PY_H_

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save