trailing time optimization for the last allreduce of gradients in distribution mode

4 years ago · 5444703edf
parent 444550246f
commit 5444703edf
2 changed files with 308 additions and 0 deletions
--- a/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc
+++ b/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc
--- a/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.h
+++ b/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.h
@ -161,6 +161,13 @@ class AscendStreamAssign {
  void GetProcessedStream(const NotNull<KernelGraphPtr> &graph_ptr);
  void GetNeedActiveStreams(const NotNull<KernelGraphPtr> &graph_ptr);
  void ReorderIndependentOrders(const NotNull<KernelGraphPtr> &graph_ptr);
+
+  void CheckScenario(const NotNull<KernelGraphPtr> &graph_ptr, vector<CNodePtr> *last_grad_and_status);
+  CNodePtr GetCNodesNeededMoved(vector<CNodePtr> *moved_backward_cnodes, vector<CNodePtr> *moved_forward_cnodes,
+                                const vector<CNodePtr> &last_grad_and_status, const NotNull<KernelGraphPtr> &graph_ptr);
+  void FinetuneSubgraphExecOrder(vector<CNodePtr> *cnodes);
+  void TrailingTimeOptimizationByReorder(const NotNull<KernelGraphPtr> &graph_ptr);
+
  uint32_t GetMaxIndexTarget(const NotNull<KernelGraphPtr> &graph_ptr);
  uint32_t GetIndexByKey(const NotNull<KernelGraphPtr> &graph_ptr, const CNodeKey &key);
  uint32_t GetIndependentStreamSwitchStreamId(const NotNull<KernelGraphPtr> &graph_ptr);