!9199 add paralle split

From: @kisnwang
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/backend/session/session_basic.cc

@@ -1421,6 +1421,13 @@ std::vector<AnfNodePtr> ExtendNodeUsers(const FuncGraphManagerPtr &front_func_gr
       continue;
     }
     if (IsPrimitiveCNode(user.first, prim::kPrimDepend)) {
+      auto depend_cnode = user.first->cast<CNodePtr>();
+      if (depend_cnode == nullptr) {
+        continue;
+      }
+      if (front_node != depend_cnode->input(1)) {
+        continue;
+      }
       auto res = ExtendNodeUsers(front_func_graph_manager, user.first);
       result.insert(result.end(), res.begin(), res.end());
       continue;

mindspore/ccsrc/pybind_api/utils/ms_context_py.cc

@@ -83,6 +83,7 @@ REGISTER_PYBIND_DEFINE(MsContextPy, ([](const py::module *m) {
                            .value("precompile_only", MsCtxParam::MS_CTX_PRECOMPILE_ONLY)
                            .value("enable_profiling", MsCtxParam::MS_CTX_ENABLE_PROFILING)
                            .value("save_graphs", MsCtxParam::MS_CTX_SAVE_GRAPHS_FLAG)
+                           .value("enable_parallel_split", MsCtxParam::MS_CTX_ENABLE_PARALLEL_SPLIT)
                            .value("max_device_memory", MsCtxParam::MS_CTX_MAX_DEVICE_MEMORY)
                            .value("mode", MsCtxParam::MS_CTX_EXECUTION_MODE)
                            .value("device_target", MsCtxParam::MS_CTX_DEVICE_TARGET)

mindspore/core/utils/ms_context.cc

@@ -72,6 +72,7 @@ MsContext::MsContext(const std::string &policy, const std::string &target) {
   set_param<std::string>(MS_CTX_PRINT_FILE_PATH, "");
   set_param<bool>(MS_CTX_ENABLE_GRAPH_KERNEL, false);
   set_param<bool>(MS_CTX_ENABLE_SPARSE, false);
+  set_param<bool>(MS_CTX_ENABLE_PARALLEL_SPLIT, false);
 
   backend_policy_ = policy_map_[policy];
 }

mindspore/core/utils/ms_context.h

@@ -73,6 +73,7 @@ enum MsCtxParam : unsigned {
   MS_CTX_PRECOMPILE_ONLY,
   MS_CTX_ENABLE_PROFILING,
   MS_CTX_SAVE_GRAPHS_FLAG,
+  MS_CTX_ENABLE_PARALLEL_SPLIT,
   MS_CTX_TYPE_BOOL_END,
 
   // paramater of type int