Online infer support dynamic dims

ge/common/types.cc

@@ -476,6 +476,9 @@ REGISTER_OPTYPE_DEFINE(HVDCALLBACKALLGATHER, "HorovodAllgather");
 REGISTER_OPTYPE_DEFINE(HVDCALLBACKBROADCAST, "HorovodBroadcast");
 REGISTER_OPTYPE_DEFINE(HVDWAIT, "HorovodWait");
 
+// aicpu op for online_infer dynamic_dims
+REGISTER_OPTYPE_DEFINE(GETDYNAMICDIMS, "GetDynamicDims");
+
 const std::string MODEL_ATTR_TASKS = "tasks";
 const std::string MODEL_ATTR_TASK_GEN_BASE_ADDR = "task_gen_base_addr";
 const std::string MODEL_ATTR_TASK_GEN_WEIGHT_ADDR = "task_gen_weight_addr";

ge/executor/CMakeLists.txt

@@ -61,6 +61,7 @@ set(SRC_LIST
     "../graph/load/new_model_manager/task_info/model_exit_task_info.cc"
     "../graph/load/new_model_manager/task_info/super_kernel/super_kernel_factory.cc"
     "../graph/load/new_model_manager/task_info/super_kernel/super_kernel.cc"
+    "../graph/common/local_context.cc"
     "../opskernel_manager/ops_kernel_builder_manager.cc"
     "../single_op/single_op_manager.cc"
     "../single_op/single_op_model.cc"

ge/executor/module.mk

@@ -63,6 +63,7 @@ local_ge_executor_src_files :=  \
     ../single_op/task/aicpu_kernel_task_builder.cc \
     ../hybrid/hybrid_davinci_model_stub.cc\
     ../hybrid/node_executor/aicpu/aicpu_ext_info.cc \
+    ../graph/common/local_context.cc \
 
 local_ge_executor_c_include :=             \
     proto/insert_op.proto                  \

ge/graph/load/new_model_manager/davinci_model.h

@@ -47,6 +47,7 @@
 #include "mmpa/mmpa_api.h"
 #include "proto/task.pb.h"
 #include "task_info/task_info.h"
+#include "graph/common/local_context.h"
 
 namespace ge {
 // op debug need 2048 bits buffer
@@ -523,7 +524,6 @@ class DavinciModel {
   bool is_inner_p2p_mem_base_;
   // input data manager
   DataInputer *data_inputer_;
-
   int64_t load_begin_time_;
   int64_t load_end_time_;
   struct timeInfo time_info_;
@@ -842,6 +842,14 @@ class DavinciModel {
   void ParseAIPPInfo(std::string in_out_info, InputOutputDims &dims_info);
   void SetLabelForDynamic(const NodePtr &node);
 
+  void ParseDynamicOutShape(const std::vector<std::string> &str_info, std::vector<vector<int64_t>> &vec_info);
+  bool IsGetNextSinkDynamic(const OpDescPtr &op_desc);
+  void GetAllGearsInfo(const NodePtr &node);
+  Status GetGetDynamicDimsNodeInfo(const NodePtr &node);
+  Status GetGearAndRealOutSizeInfo(size_t input_count, const NodePtr &node);
+  Status GetRealOutputSizeOfMerge(size_t input_index, const NodePtr &merge_node);
+  Status GetGearAndRealOutShapeInfo(size_t input_count, const OpDescPtr &op_desc);
+
   bool is_model_has_inited_;
   uint32_t model_id_;
   uint32_t runtime_model_id_;
@@ -994,6 +1002,17 @@ class DavinciModel {
   void *op_debug_addr_ = nullptr;
   void *p2p_debug_addr_ = nullptr;
   bool is_new_model_desc_{false};
+  bool is_online_infer_dynamic_ = false;
+  bool is_getnext_sink_dynamic_ = false;
+  std::vector<int64_t> cur_dynamic_dims_;
+  void *netoutput_last_input_addr_ = nullptr;
+  int64_t netoutput_last_input_size_ = 0;
+  size_t shape_of_cur_dynamic_dims_ = 0;
+  // key: input_index: input is merge node; value: each gear info and each output size
+  std::map<size_t, std::map<vector<int64_t>, int64_t>> merge_nodes_gear_and_real_out_size_info_;
+  // key: input_index: input is merge node; value: each gear info and each output shape
+  std::map<size_t, std::map<vector<int64_t>, vector<int64_t>>> merge_nodes_gear_and_real_out_shape_info_;
+  std::vector<std::vector<int64_t>> all_gears_info_;
 };
 }  // namespace ge
 #endif  // GE_GRAPH_LOAD_NEW_MODEL_MANAGER_DAVINCI_MODEL_H_

ge/graph/load/new_model_manager/model_manager.cc

@@ -29,6 +29,8 @@
 #include "graph/load/new_model_manager/davinci_model.h"
 #include "graph/load/new_model_manager/davinci_model_parser.h"
 #include "model/ge_root_model.h"
+#include "graph/common/local_context.h"
+#include "common/formats/utils/formats_trans_utils.h"
 
 namespace ge {
 thread_local uint32_t device_count = 0;
@@ -405,6 +407,10 @@ Status ModelManager::Unload(uint32_t model_id) {
   }
   std::lock_guard<std::mutex> lock(exeception_infos_mutex_);
   exception_infos_.clear();
+  for (auto addr : shape_data_addrs_[model_id]) {
+    delete[] addr;
+  }
+  shape_data_addrs_.erase(model_id);
   return SUCCESS;
 }
 
@@ -444,6 +450,34 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d
   return SUCCESS;
 }
 
+Status ModelManager::GetCurDynamicDims(const vector<vector<int64_t>> &user_real_input_dims,
+                                       const vector<pair<string, vector<int64_t>>> &user_input_dims,
+                                       vector<int64_t> &cur_dynamic_dims) {
+  GELOGD(" Start get cur dynamic dims.");
+  if (user_real_input_dims.size() != user_input_dims.size()) {
+    GELOGE(INTERNAL_ERROR,
+           "The input count of user: %zu should be equal to the data count of graph: %zu",
+           user_real_input_dims.size(), user_input_dims.size());
+    return INTERNAL_ERROR;
+  }
+
+  for (size_t i = 0; i < user_input_dims.size(); ++i) {
+    if (user_real_input_dims[i].size() != user_input_dims[i].second.size()) {
+      GELOGE(INTERNAL_ERROR,
+             "The shape size: %zu of dynamic input: %s should be equal to the shape size of input shape: %zu.",
+             user_real_input_dims[i].size(), user_input_dims[i].first.c_str(), user_input_dims[i].second.size());
+      return INTERNAL_ERROR;
+    }
+    for (size_t j = 0; j < user_input_dims.at(i).second.size(); ++j) {
+      if (user_input_dims.at(i).second.at(j) < 0) {
+        cur_dynamic_dims.emplace_back(user_real_input_dims[i][j]);
+      }
+    }
+  }
+  GELOGD("Cur dynamic dims is %s.", formats::JoinToString(cur_dynamic_dims).c_str());
+  return SUCCESS;
+}
+
 ///
 /// @ingroup domi_ome
 /// @brief load Input and output TensorInfo for Model
@@ -461,13 +495,31 @@ Status ModelManager::DataInputTensor(uint32_t model_id, const std::vector<InputT
   input_data.timeout = 0;
   input_data.timestamp = 0;
   input_data.index = 0;
-
   for (size_t i = 0; i < inputs.size(); ++i) {
     DataBuffer data;
     data.data = inputs[i].data;
     data.length = inputs[i].length;
     input_data.blobs.push_back(data);
   }
+  if (!GetLocalOmgContext().user_input_dims.empty() && GetLocalOmgContext().need_multi_batch) {
+    std::vector<int64_t> cur_dynamic_dims;
+    if (!GetLocalOmgContext().user_real_input_dims.empty()) {
+      if (GetCurDynamicDims(GetLocalOmgContext().user_real_input_dims, GetLocalOmgContext().user_input_dims,
+                            cur_dynamic_dims) != SUCCESS) {
+        GELOGE(INTERNAL_ERROR, "[Train_Dynamic] Failed to Parse real_dynamic_dims.");
+        return INTERNAL_ERROR;
+      }
+      DataBuffer data;
+      data.data = new(std::nothrow) int64_t[cur_dynamic_dims.size()];
+      GE_CHECK_NOTNULL(data.data);
+      uint64_t length = static_cast<uint64_t>(cur_dynamic_dims.size() * sizeof(int64_t));
+      GE_CHK_BOOL_EXEC(memcpy_s(data.data, length, cur_dynamic_dims.data(), length) == EOK, return INTERNAL_ERROR,
+                       "Failed to memcpy data.");
+      data.length = length;
+      input_data.blobs.push_back(data);
+      shape_data_addrs_[model_id].emplace_back(reinterpret_cast<int64_t *>(data.data));
+    }
+  }
 
   OutputData output_data;
   output_data.model_id = model_id;

ge/graph/load/new_model_manager/model_manager.h

@@ -124,6 +124,18 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
 
   ge::Status DataInputTensor(uint32_t model_id, const std::vector<InputTensorInfo> &inputs);
 
+  ///
+  /// @ingroup domi_ome
+  /// @brief Get cur_dynamic_dims for all input.
+  /// @param [in] vector<vector<uint64_t>> &user_real_input_dims: dims info of all user_inputs.
+  /// @param [in] vector<pair<string, vector<int64_t>>> &user_input_dims: key:name. value:dynamic dims from option.
+  /// @param [out] vector<uint64_t> &cur_dynamic_dims: real dims gather, where the index of -1.
+  /// @return 0: SUCCESS / others: INTERNAL_ERROR
+  ///
+  Status GetCurDynamicDims(const vector<vector<int64_t>> &user_real_input_dims,
+                           const vector<pair<string, vector<int64_t>>> &user_input_dims,
+                           vector<int64_t> &cur_dynamic_dims);
+
   ///
   /// @ingroup domi_ome
   /// @brief model start to run
@@ -352,6 +364,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
   std::map<uintptr_t, std::map<std::string, CustAICPUKernelPtr>> cust_aicpu_so_;
 
   static DumpProperties dump_properties_;
+  std::map<uint32_t, std::vector<int64_t *>> shape_data_addrs_;
 };
 }  // namespace ge
 

ge/graph/manager/graph_manager.h

@@ -72,6 +72,7 @@ class GraphManager {
   ///
   Status AddGraph(const GraphId &graph_id, const Graph &graph, const std::map<std::string, std::string> &options,
                   const OmgContext &omg_context);
+  Status InitDynamicParams(ComputeGraphPtr &compute_graph);
 
   ///
   /// @ingroup ge_graph
@@ -215,6 +216,12 @@ class GraphManager {
   static Status ProcessSubGraphWithMultiThreads(GraphManager *graph_manager, GraphId root_graph_id,
                                                 const SubGraphInfoPtr &sub_graph_info_ptr, uint64_t session_id,
                                                 const GEThreadLocalContext &ge_context);
+  Status ParseInputsDims(const std::vector<InputTensorInfo> &input_tensor);
+  Status DistinguishGetNextAndData(ComputeGraphPtr &graph, vector<NodePtr> &data_nodes,
+                                   vector<NodePtr> &getnext_nosink_nodes, vector<NodePtr> &getnext_sink_nodes);
+  void ParseInputsDimsForData(const std::vector<InputTensorInfo> &input_tensor);
+  Status ParseInputsDimsForGetNexNosinkAndData(const vector<NodePtr> &dynamic_nodes,
+                                               const std::vector<InputTensorInfo> &input_tensor);
   Status PreRun(const GraphNodePtr &graph_node, const std::vector<GeTensor> &inputs, GeRootModelPtr &ge_root_model,
                 uint64_t session_id = INVALID_SESSION_ID);
 
@@ -370,7 +377,7 @@ class GraphManager {
   BlockingQueue<RunArgs> run_args_q_{};
   std::thread prerun_thread_;
   std::thread run_thread_;
-
+  ComputeGraphPtr compute_graph_;
   std::map<GraphId, GraphNodePtr> graph_map_;
   std::map<GraphId, ModelCacheHelperPtr> cache_helper_map_;
 

ge/graph/manager/graph_manager_utils.h

@@ -249,6 +249,9 @@ struct GraphManagerOptions {
   std::string save_original_model;
   std::string build_mode;
   std::string build_step;
+  std::string input_shape;
+  std::string dynamic_dims;
+  int32_t dynamic_node_type = -1;
   GraphManagerOptions()
       : stream_num(1),
         perf_level(domi::GEN_TASK_WITHOUT_FUSION),

ge/graph/preprocess/graph_preprocess.cc

@@ -130,6 +130,8 @@ const char *const kMbatchSwitchnName = "mbatch-switch-name";
 
 // the size of user defined output datatype or format string after split by ":".
 const size_t kUserDefinedElementCount = 2;
+const int kDataOutIndex = 0;
+const int64_t kInvalidDynaimcDimsType = -1;
 
 OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) {
   GeTensorPtr tensor = MakeShared<GeTensor>();
@@ -1160,6 +1162,9 @@ Status GraphPrepare::UpdateInput(const std::vector<GeTensor> &user_input) {
         return FAILED;
       }
 
+      if (IsDynamicDims(input_node)) {
+        continue;
+      }
       GeTensorDesc desc(user_input[index].GetTensorDesc());
       auto format = desc.GetFormat();
       auto origin_format = desc.GetOriginFormat();
@@ -1221,10 +1226,7 @@ Status GraphPrepare::UpdateInput(const std::vector<GeTensor> &user_input) {
 
       if (!options_.train_graph_flag) {
         Status ret = AdjustDataOpOutput(input_node);
-        if (ret != SUCCESS) {
-          GELOGE(ret, "AdjustDataOpOutput fail, ret:%u", ret);
-          return ret;
-        }
+        GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "AdjustDataOpOutput fail, ret:%u", ret); return ret);
       }
     }
   }
@@ -1570,6 +1572,22 @@ Status GraphPrepare::VerifyConstOp(const NodePtr &node) {
   return SUCCESS;
 }
 
+bool GraphPrepare::IsDynamicDims(const NodePtr &input_node) {
+  auto data_shape = NodeUtils::GetOutputDesc(*input_node, kDataOutIndex).GetShape();
+  const auto &dims = data_shape.GetDims();
+  bool all_is_positive = false;
+  if (std::all_of(dims.begin(), dims.end(), [](int64_t val) { return val >= 0; })) {
+    all_is_positive = true;
+  }
+  if (!all_is_positive && !options_.input_shape.empty() && !options_.dynamic_dims.empty() &&
+      options_.dynamic_node_type != kInvalidDynaimcDimsType) {
+    GELOGI("No need to check and update desc info, the dims of %s is %s.", input_node->GetName().c_str(),
+           formats::JoinToString(dims).c_str());
+    return true;
+  }
+  return false;
+}
+
 Status GraphPrepare::CheckUserInput(const std::vector<GeTensor> &user_input) {
   if (GetLocalOmgContext().is_dynamic_input) {
     return SUCCESS;
@@ -1595,6 +1613,9 @@ Status GraphPrepare::CheckUserInput(const std::vector<GeTensor> &user_input) {
         GELOGE(GE_GRAPH_INIT_FAILED, "user_input size:%zu, data op index:%ld.", user_input.size(), index);
         return GE_GRAPH_INIT_FAILED;
       }
+      if (IsDynamicDims(input_node)) {
+        continue;
+      }
       GeTensorDesc desc(user_input[index].GetTensorDesc());
 
       for (size_t i = 0; i < desc.GetShape().GetDimNum(); ++i) {

ge/graph/preprocess/graph_preprocess.h

@@ -84,6 +84,7 @@ class GraphPrepare {
 
   Status GraphEquivalentTransformation();
   void TypeConversionOfConstant();
+  bool IsDynamicDims(const NodePtr &input_node);
 
   ge::ComputeGraphPtr compute_graph_;
   GraphManagerOptions options_;

ge/graph/preprocess/multi_batch_copy_graph.h

@@ -55,9 +55,7 @@ class MultiBatchGraphCopyer {
       data_name_order_.push_back(item.first);
     }
   }
-  void SetDataToDynamicInfo(const map<string, vector<vector<int64_t>>> &designate_shape) {
-    data_to_dynamic_info_ = designate_shape;
-  }
+
   void SetDynamicType(const DynamicType dynamic_type) {
     dynamic_type_ = dynamic_type;
   }
@@ -69,15 +67,26 @@ class MultiBatchGraphCopyer {
 
   // label status for origin_all_nodes_
   Status LabelStatus();
+  Status LabelInBatchBranchStatus();
+  void LabelStatusForData(const NodePtr &data);
+  void LabelStatusForGetNextSink(const NodePtr &data);
   // add nodes functions
   Status CreateNewNodes();
 
   NodePtr InsertShapeDataNode();
-  Status InsertSwitchNForData(const NodePtr &data);
+  NodePtr InsertGetDynamicDimsNode();
+
+  Status InsertSwitchNAndUpdateMaxShape(const NodePtr &node);
+
+  Status InsertSwitchNForData(const NodePtr &node, const size_t &out_anchor_index, const size_t &peer_in_anchor_index,
+                              std::vector<std::pair<Node *, NodePtr>> &dynamic_out_to_switchn);
+
   Status InsertIdentityAfterSwitchN();
-  Status UpdateMaxShapeToData(const NodePtr &data);
+  Status UpdateMaxShapeToData(const NodePtr &node, size_t out_anchor_index);
+  Status UpdateShapeOfShapeNode(const NodePtr &node, size_t out_anchor_index);
 
   Status InsertMergeForEdgeNode(const NodePtr &node);
+  Status LinkGetDynamicDimsToNetOutput(const NodePtr &node);
 
   /// Insert a merge node for src node `node` on output index `index`. The merge node will be used to merge all nodes
   /// in batch-branch to one output to the node out of the batch-branch.
@@ -95,12 +104,16 @@ class MultiBatchGraphCopyer {
 
   // link edges functions
   Status LinkEdges();
-  Status LinkDataToSwitchN(const NodePtr &data);
+  Status AddAttrForGetDynamicDims(const NodePtr &node);
+  Status AddLinkForGetDynamicDims(const NodePtr &node);
+
+  Status LinkDataToSwitchN(const NodePtr &data, const NodePtr &switchn, const int &out_index);
   Status LinkToMerge(const NodePtr &node);
   Status LinkToNodeInBranch(const NodePtr &node);
   Status LinkToNodeOutBranch(const NodePtr &node);
-  Status LinkDataToMerge(const NodePtr &data, const NodePtr &merge);
+  Status LinkDataToMerge(const NodePtr &data, const NodePtr &merge, const NodePtr &switchn);
   Status LinkNodeToMerge(const NodePtr &node, int out_index, const NodePtr &merge);
+  NodePtr FindSwitchnNodeForDataEdge(const OutDataAnchorPtr &data_out_anchor, const NodePtr &origin_node);
   Status CopyInDataEdges(const NodePtr &origin_node, int batch_num, const NodePtr &copyed_node);
   Status CopyInControlEdges(const NodePtr &node, int batch_num, const NodePtr &copyed_node);
   Status CheckAndParseDynamicData();
@@ -127,6 +140,11 @@ class MultiBatchGraphCopyer {
   // the data nodes, and the SwitchN nodes inserted after it
   std::map<Node *, NodePtr> data_nodes_to_switchn_;
 
+  // the getnext_sink nodes, and the SwitchN nodes inserted after it
+  std::vector<std::vector<std::pair<Node *, NodePtr>>> getnext_nodes_to_switchn_;
+  std::vector<std::vector<std::pair<int, int>>> outidx_inidx_mappings_;
+  std::vector<std::pair<int, int>> outidx_inidx_mapping_;
+
   // the nodes on the in/out-batch-branch edge, and the merge nodes inserted after it
   std::map<Node *, std::vector<NodePtr>> nodes_to_merge_nodes_;
 
@@ -142,6 +160,9 @@ class MultiBatchGraphCopyer {
 
   // dynamic type : dynamic batch,, dynamic image size, dynamic dims.
   DynamicType dynamic_type_ = DynamicType::kDynamicUnknown;
+
+  std::vector<std::pair<size_t, size_t>> getnext_sink_dynamic_out_mapping_;
+  bool getnext_sink_dynamic_dims_ = false;
 };
 }  // namespace multibatch
 }  // namespace ge

ge/graph/preprocess/multi_batch_options.h

@@ -28,6 +28,21 @@ namespace ge {
 namespace multibatch {
 ///
 /// @ingroup ge
+/// @brief Update Dynamic Param from Options.
+/// @param [in] ComputeGraphPtr &graph: the train graph
+/// @return SUCCESS: valid / PARAM_INVALID: invalid.
+///
+Status CheckSequenceOfOptions(ComputeGraphPtr &graph, std::vector<NodePtr> &data_nodes,
+                              std::vector<NodePtr> &getnext_nosink_nodes, std::vector<NodePtr> &getnext_sink_nodes);
+
+Status UpdateNameOfInputShape(ComputeGraphPtr &graph, const vector<NodePtr> &data_nodes,
+                              const vector<NodePtr> &getnext_nosink_nodes, const vector<NodePtr> &getnext_sink_nodes);
+
+Status DeleteIdentityInsertByAdapter(ComputeGraphPtr &graph);
+
+Status CheckNegativeCountOfOptions(const std::vector<std::vector<int64_t>> &shapes);
+///
+/// @ingroup ge
 /// @brief Init Dynamic Param from Options.
 /// @param [out] std::vector<std::vector<int64_t>> &shapes: Result for Params.
 /// @return true: Configed for Multi batch / false: Not configed for Multi batch.

inc/external/ge/ge_api_types.h

@@ -174,6 +174,9 @@ const std::string HCOM_PARALLEL = "ge.hcomParallel";
 // configure whether to use dynamic batch size
 const char *const kDynamicBatchSize = "ge.dynamicBatchSize";
 
+const std::string INPUT_SHAPE = "ge.inputShape";
+
+const std::string DYNAMIC_NODE_TYPE = "ge.dynamicNodeType";
 // configure whether to use dynamic image size
 const char *const kDynamicImageSize = "ge.dynamicImageSize";
 

inc/framework/common/types.h

@@ -526,6 +526,9 @@ REGISTER_OPTYPE_DECLARE(HVDCALLBACKALLGATHER, "HorovodAllgather");
 REGISTER_OPTYPE_DECLARE(HVDCALLBACKBROADCAST, "HorovodBroadcast");
 REGISTER_OPTYPE_DECLARE(HVDWAIT, "HorovodWait");
 
+// aicpu op for online_infer dynamic_dims
+REGISTER_OPTYPE_DECLARE(GETDYNAMICDIMS, "GetDynamicDims");
+
 enum InputMode { INPUT = 0, CONST };
 
 // Definition of the processing status enum of the process module

inc/framework/omg/omg_inner_types.h

@@ -115,6 +115,10 @@ struct OmgContext {
   std::string dynamic_batch_size;
   std::string dynamic_image_size;
   std::string dynamic_dims;
+  std::string dynamic_node_type;
+  std::vector<std::vector<int64_t>> user_real_input_dims;
+  std::vector<int64_t> cur_dynamic_dims;
+  bool need_multi_batch = false;
 };
 }  // namespace ge
 

metadef

@@ -1 +1 @@
-Subproject commit 37465b85d30b67a0edcc6ea4acd2f11a9697c7af
+Subproject commit 2f299e7b532f0cc7031c35e5f0fe7a52ee4f4697

parser

@@ -1 +1 @@
-Subproject commit 5fa1f3ed9b1785b9fd1623d624de91838dff615e
+Subproject commit 40c6b462724ad0bc1b76df1fb4fbd0df0e6c0536