|
|
|
@ -18,7 +18,7 @@ import time
|
|
|
|
|
from enum import Enum
|
|
|
|
|
|
|
|
|
|
from mindspore import log as logger, context
|
|
|
|
|
from mindspore.communication.management import release
|
|
|
|
|
from mindspore.communication.management import release, init, get_rank
|
|
|
|
|
from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
|
|
|
|
|
ProfilerIOException, ProfilerException
|
|
|
|
|
from mindspore.profiler.common.util import get_file_names, fwrite_format
|
|
|
|
@ -93,6 +93,8 @@ class Profiler:
|
|
|
|
|
self._gpu_profiler = GPUProfiler.get_instance()
|
|
|
|
|
self._gpu_profiler.init(self._output_path)
|
|
|
|
|
self._gpu_profiler.step_profiling_enable(True)
|
|
|
|
|
init()
|
|
|
|
|
self._dev_id = get_rank()
|
|
|
|
|
|
|
|
|
|
if kwargs:
|
|
|
|
|
logger.warning("Params not be supported yet on GPU.")
|
|
|
|
@ -291,18 +293,10 @@ class Profiler:
|
|
|
|
|
"""Used for gpu, generate timeline info, write to json format file."""
|
|
|
|
|
try:
|
|
|
|
|
size_limit = 100 * 1024 * 1024 # 100MB
|
|
|
|
|
#stastic the number of dev_id
|
|
|
|
|
file_list = os.listdir(self._output_path)
|
|
|
|
|
dev_id_list = []
|
|
|
|
|
for file_name in file_list:
|
|
|
|
|
if file_name.startswith('gpu_op_detail'):
|
|
|
|
|
_dev_id = file_name.split('.')[0].split('_')[-1]
|
|
|
|
|
dev_id_list.append(_dev_id)
|
|
|
|
|
for dev_id in dev_id_list:
|
|
|
|
|
timeline_generator = GpuTimelineGenerator(self._output_path, dev_id)
|
|
|
|
|
timeline_generator.init_timeline()
|
|
|
|
|
timeline_generator.write_timeline(size_limit)
|
|
|
|
|
timeline_generator.write_timeline_summary()
|
|
|
|
|
timeline_generator = GpuTimelineGenerator(self._output_path, self._dev_id)
|
|
|
|
|
timeline_generator.init_timeline()
|
|
|
|
|
timeline_generator.write_timeline(size_limit)
|
|
|
|
|
timeline_generator.write_timeline_summary()
|
|
|
|
|
except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
|
|
|
|
|
logger.warning('Fail to write timeline data: %s', err)
|
|
|
|
|
raise RuntimeError('Fail to write timeline data.')
|
|
|
|
|