!11534 profiler: optimized the code of processing memory usage and added memory breakdowns for each execution id

From: @zhangyunshu
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/CMakeLists.txt

@@ -126,8 +126,8 @@ list(APPEND MINDSPORE_PROTO_LIST ${COMM_PROTO_SRCS})
 include_directories("${CMAKE_BINARY_DIR}/profiler/device/common")
 file(GLOB_RECURSE PROFILER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
   "profiler/device/common/memory_profiling.proto")
-ms_protobuf_generate(PROFILER_MEM_PROTO_SRCS PROFILER_MEM_PROTO_HDRS ${PROFILER_PROTO_LIST})
-list(APPEND MINDSPORE_PROTO_LIST ${PROFILER_MEM_PROTO_SRCS})
+ms_protobuf_generate_py(PROFILER_MEM_PROTO_PY PROFILER_MEM_PROTO_HDRS_PY PROFILER_MEM_PROTO_PYS ${PROFILER_PROTO_LIST})
+list(APPEND MINDSPORE_PROTO_LIST ${PROFILER_MEM_PROTO_PY})
 
 if(ENABLE_DEBUGGER)
     # debugger: compile proto files

mindspore/profiler/common/proto_files/__init__.py

@@ -1,15 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""The proto files for profiler."""

mindspore/profiler/common/proto_files/memory_usage.proto

@@ -1,50 +0,0 @@
-/**
- * Copyright 2021 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-package profiler;
-
-message MemoryProto {
-    repeated GraphMemProto graph_mem = 1;  // memory usage of multiple graphs
-    int64 total_mem = 2;  // total allocated memory on device
-}
-
-message GraphMemProto {
-    int64 graph_id = 1;  // graph id
-    int64 static_mem = 2;  // size of allocated static memory for current graph
-    repeated NodeMemProto node_mems = 3;  // execution nodes
-    repeated TensorMemProto tensor_mems = 4;  // all tensors
-    string fp_start = 5;  // node name of fp start
-    string bp_end = 6;  // node name of bp end
-}
-
-message NodeMemProto {
-    string node_name = 1;  // node name
-    int64 node_id = 2;  // node id with respect to the execution order
-    repeated int64 input_tensor_id = 3;  // input tensor id
-    repeated int64 output_tensor_id = 4;  // output tensor id
-    repeated int64 workspace_tensor_id = 5;  // workspace tensor id
-}
-
-message TensorMemProto {
-    int64 tensor_id = 1;  // tensor id
-    int64 size = 2;  // aligned tensor size
-    string type = 3;  // tensor type, e.g. Common, OutputOnly
-    int64 life_start = 4;  // node id at which memory allocated
-    int64 life_end = 5;  // node id at which memory deallocated
-    string life_long = 6;  // the type of tensor lifetime, e.g. LifeLongGraphAll
-}

mindspore/profiler/parser/container.py

@@ -13,8 +13,6 @@
 # limitations under the License.
 # ============================================================================
 """The container of metadata used in profiler parser."""
-import heapq
-
 GIGABYTES = 1024 * 1024 * 1024
 
 
@@ -131,6 +129,7 @@ class MemoryGraph:
         self.bp_end = None
         self.lines = []
         self.nodes = {}
+        self.breakdowns = []
 
     def to_dict(self):
         """Convert Graph to dict."""
@@ -140,7 +139,8 @@ class MemoryGraph:
             'nodes': self.nodes,
             'fp_start': self.fp_start,
             'bp_end': self.bp_end,
-            'lines': self.lines
+            'lines': self.lines,
+            'breakdowns': self.breakdowns
         }
 
         return graph
@@ -152,17 +152,15 @@ class MemoryNode:
 
     Args:
         node_proto (proto): Node proto.
-        graph_id (int): Graph id.
     """
-    def __init__(self, node_proto, graph_id):
+    def __init__(self, node_proto):
         self._node_proto = node_proto
-        self.graph_id = graph_id
         self.node_id = node_proto.node_id
         self.name = node_proto.node_name
         self.fullname = ""
-        self.input_ids = [t_id for t_id in node_proto.input_tensor_id]
-        self.output_ids = [t_id for t_id in node_proto.output_tensor_id]
-        self.workspace_ids = [t_id for t_id in node_proto.workspace_tensor_id]
+        self.input_ids = list(node_proto.input_tensor_id)
+        self.output_ids = list(node_proto.output_tensor_id)
+        self.workspace_ids = list(node_proto.workspace_tensor_id)
         self.inputs = []
         self.outputs = []
         self.workspaces = []
@@ -181,8 +179,7 @@ class MemoryNode:
             'size': self.size,
             'allocated': self.mem_change,
             'inputs': self.inputs,
-            'outputs': self.outputs,
-            'workspaces': self.workspaces
+            'outputs': self.outputs
         }
 
         return node
@@ -194,9 +191,8 @@ class MemoryTensor:
 
     Args:
         tensor_proto (proto): Tensor proto.
-        graph_id (int): Graph id.
     """
-    def __init__(self, tensor_proto, graph_id):
+    def __init__(self, tensor_proto):
         self._tensor_proto = tensor_proto
         self.tensor_id = tensor_proto.tensor_id
         self.life_long = tensor_proto.life_long
@@ -204,48 +200,25 @@ class MemoryTensor:
         self.life_end = tensor_proto.life_end
         self.size = tensor_proto.size / GIGABYTES
         self.type = tensor_proto.type
-        self.graph_id = graph_id
+        self.shape = ""
+        self.format = ""
+        self.dtype = ""
+        self.source_node = ""
+        self.name = ""
 
     def to_dict(self):
         """Convert Tensor to a dict."""
         tensor = {
+            'tensor_name': self.name,
             'tensor_id': self.tensor_id,
             'size': self.size,
             'type': self.type,
+            'shape': self.shape,
+            'format': self.format,
+            'data_type': self.dtype,
             'life_long': self.life_long,
             'life_start': self.life_start,
             'life_end': self.life_end
         }
 
         return tensor
-
-
-class MemoryQueue:
-    """
-    A priority queue to keep specified number of active nodes in memory activities.
-
-    Args:
-        size (int): The upper limit of nodes to be saved.
-    """
-    def __init__(self, size):
-        self._queue = []
-        self._index = 0
-        self._size = size
-
-    def push(self, item, priority):
-        """
-        Push a node into MemoryQueue.
-
-        Args:
-            item (tuple): Node item including id, name, etc.
-            priority (int): The priority of the item.
-        """
-        if self._index < self._size:
-            heapq.heappush(self._queue, (-priority, item))
-            self._index += 1
-        else:
-            heapq.heappushpop(self._queue, (-priority, item))
-
-    def get_items(self):
-        """Get the elements in MemoryQueue."""
-        return self._queue