m53297601
/
graphengine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

For step info.

Browse Source
pull/1376/head
unknown 4 years ago
parent f5eded92c3
commit 6ae6d053c0
8 changed files with 97 additions and 0 deletions
Show Stats Download Patch File Download Diff File

2
ge/CMakeLists.txt
Unescape View File

@ -708,6 +708,7 @@ target_compile_definitions(ge_runner PRIVATE
DAVINCI_CLOUD
google=ascend_private
FUNC_VISIBILITY
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
)
target_compile_options(ge_runner PRIVATE
@ -783,6 +784,7 @@ target_compile_definitions(ge_compiler PRIVATE
COMPILE_OMG_PACKAGE
google=ascend_private
FUNC_VISIBILITY
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
)
target_compile_options(ge_compiler PRIVATE

54
ge/common/profiling/profiling_manager.cc
Unescape View File

@ -24,6 +24,7 @@
#include "graph/types.h"
#include "runtime/base.h"
#include "graph/load/model_manager/davinci_model.h"
#include "mmpa/mmpa_api.h"
namespace {
const char *const kTrainingTrace = "training_trace";
@ -46,6 +47,7 @@ const std::string kOptype = "op_type";
const std::string kBlockDim = "block_dims";
const std::string kTaskId = "task_id";
const std::string kStreamId = "stream_id";
const std::string kThreadId = "thread_id";
const std::string kShapeType = "shape_type";
const std::string kCurIterNum = "cur_iter_num";
const std::string kTaskType = "task_type";
@ -286,6 +288,58 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin
#endif
}
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfileStepInfo(
uint64_t index_id, uint64_t model_id, uint16_t tag_id, rtStream_t stream, int32_t device_id) {
#ifdef DAVINCI_SUPPORT_PROFILING
rtError_t rt_ret = RT_ERROR_NONE;
#ifndef ONLY_COMPILE_OPEN_SRC
GELOGD("Profiling Step Info TraceTask execute async start, index_id = %lu, model_id = %lu, tag_id = %u",
index_id, model_id, tag_id);
rt_ret = rtProfilerTraceEx(index_id, model_id, tag_id, stream);
if (rt_ret != RT_ERROR_NONE) {
GELOGE(RT_FAILED, "[Call][rtProfilerTraceEx] failed, ret: 0x%X", rt_ret);
return RT_ERROR_TO_GE_STATUS(rt_ret);
}
GELOGD("Profiling Step Info TraceTask execute async success, index_id = %lu, model_id = %lu, tag_id = %u",
index_id, model_id, tag_id);
#endif
mmTimespec timespec = mmGetTickCount();
// 1000 ^ 3 converts second to nanosecond
int64_t time = timespec.tv_sec * 1000 * 1000 * 1000 + timespec.tv_nsec;
uint32_t task_id = 0;
uint32_t stream_id = 0;
rt_ret = rtGetTaskIdAndStreamID(&task_id, &stream_id);
if (rt_ret != RT_ERROR_NONE) {
GELOGE(RT_FAILED, "[Get][RtsInfo] task_id and stream_id failed, ret: 0x%X.", rt_ret);
return RT_ERROR_TO_GE_STATUS(rt_ret);
}
GELOGD("Get profiling args, task_id[%u], stream_id[%u]", task_id, stream_id);
Json step_info;
step_info[kIndexId] = index_id;
step_info[kModeleId] = model_id;
step_info[kTimeStamp] = time;
step_info[kTagId] = tag_id;
step_info[kTaskId] = task_id;
step_info[kStreamId] = stream_id;
step_info[kThreadId] = mmGetTid();
std::string reported_data;
try {
reported_data = step_info.dump(kInteval, ' ', false, Json::error_handler_t::ignore);
} catch (std::exception &e) {
GELOGE(FAILED, "Failed to convert JSON to string, reason: %s.", e.what());
} catch (...) {
GELOGE(FAILED, "Failed to convert JSON to string.");
}
reported_data.append(",")
.append("\n");
ProfilingManager::Instance().ReportData(device_id, reported_data, "step_info");
#endif
return SUCCESS;
}
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportData(
const int32_t &device_id, const string &data, const string &tag_name) {
#ifdef DAVINCI_SUPPORT_PROFILING

1
ge/common/profiling/profiling_manager.h
Unescape View File

@ -97,6 +97,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager {
void GetFpBpPoint(std::string &fp_point, std::string &bp_point);
void GetOpInputOutputInfo(const OpDescPtr &op, TaskDescInfo &task_desc_info) const;
void ReportData(const int32_t &device_id, const std::string &data, const std::string &tag_name);
Status ProfileStepInfo(uint64_t index_id, uint64_t model_id, uint16_t tag_id, rtStream_t stream, int32_t device_id);
private:
Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf);
Status ParseOptions(const std::string &options);

2
ge/executor/CMakeLists.txt
Unescape View File

@ -179,6 +179,7 @@ target_compile_definitions(ge_executor PRIVATE
google=ascend_private
$<IF:$<STREQUAL:${TARGET_SYSTEM_NAME},Windows>,OS_TYPE=WIN,OS_TYPE=0>
$<$<STREQUAL:${TARGET_SYSTEM_NAME},Windows>:SECUREC_USING_STD_SECURE_LIB=0 NOMINMAX>
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
LOG_CPP
)
@ -225,6 +226,7 @@ target_compile_definitions(ge_executor_shared PRIVATE
DAVINCI_SUPPORT_PROFILING
google=ascend_private
FUNC_VISIBILITY
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
)
target_include_directories(ge_executor_shared PRIVATE

13
ge/graph/load/model_manager/davinci_model.cc
Unescape View File

@ -3693,12 +3693,25 @@ Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputDa
GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_PRE_PROC_END));
if (!task_list_.empty()) {
uint64_t index_id = iterator_count_ + 1;
uint64_t model_id = static_cast<uint64_t>(model_id_);
int32_t device_id = static_cast<int32_t>(device_id_);
// tag_id 0 means step begin, 1 meas step end.
if (profiling_model_execute_on) {
GE_CHK_STATUS_RET_NOLOG(
ProfilingManager::Instance().ProfileStepInfo(index_id, model_id, 0, rt_model_stream_, device_id));
}
GELOGD("rtModelExecute do");
GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_INFER_START));
rtError_t rt_ret = rtModelExecute(rt_model_handle_, rt_model_stream_, 0);
GE_CHK_RT_EXEC(rt_ret, return RT_ERROR_TO_GE_STATUS(rt_ret));
GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_INFER_END));
GELOGD("rtModelExecute end");
if (profiling_model_execute_on) {
GE_CHK_STATUS_RET_NOLOG(
ProfilingManager::Instance().ProfileStepInfo(index_id, model_id, 0, rt_model_stream_, device_id));
}
iterator_count_++;
}
if (!is_async_mode_) {

12
ge/hybrid/executor/hybrid_model_executor.cc
Unescape View File

@ -18,6 +18,7 @@
#include "graph/ge_context.h"
#include "graph/runtime_inference_context.h"
#include "common/dump/dump_manager.h"
#include "common/profiling/profiling_manager.h"
namespace ge {
namespace hybrid {
@ -77,9 +78,20 @@ Status HybridModelExecutor::ExecuteGraphInternal(SubgraphExecutor &executor,
GE_CHK_STATUS_RET_NOLOG(ResetExecutionContext(context_));
RECORD_MODEL_EXECUTION_EVENT(&context_, "[InitContext] End");
// tag_id 0 means step begin, 1 meas step end.
uint64_t index_id = context_.iteration + 1;
uint64_t model_id = static_cast<uint64_t>(model_->GetModelId());
int32_t device_id = static_cast<int32_t>(device_id_);
auto &prof_mgr = ProfilingManager::Instance();
if (prof_mgr.ProfilingModelExecuteOn()) {
GE_CHK_STATUS_RET_NOLOG(prof_mgr.ProfileStepInfo(index_id, model_id, 0, stream_, device_id));
}
HYBRID_CHK_STATUS_RET(executor.ExecuteAsync(args.inputs, args.input_desc, args.outputs),
"Failed to execute partitioned call.");
RECORD_MODEL_EXECUTION_EVENT(&context_, "[ExecuteAsync] End");
if (prof_mgr.ProfilingModelExecuteOn()) {
GE_CHK_STATUS_RET_NOLOG(prof_mgr.ProfileStepInfo(index_id, model_id, 1, stream_, device_id));
}
HYBRID_CHK_STATUS_RET(executor.Synchronize(), "Failed to sync root graph.");
RECORD_MODEL_EXECUTION_EVENT(&context_, "[Synchronize] End");

7
tests/ut/ge/CMakeLists.txt
Unescape View File

@ -840,6 +840,8 @@ add_library(ge_ut_common STATIC ${COMMON_SRC_FILES} ${PROTO_HDRS})
target_compile_definitions(ge_ut_common PRIVATE
google=ascend_private
DAVINCI_SUPPORT_PROFILING
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
)
target_compile_options(ge_ut_common PRIVATE
@ -860,6 +862,8 @@ add_library(ge_ut_common_format STATIC ${COMMON_SRC_FILES} ${COMMON_FORMAT_SRC_F
target_compile_definitions(ge_ut_common_format PRIVATE
google=ascend_private
DAVINCI_SUPPORT_PROFILING
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
)
target_compile_options(ge_ut_common_format PRIVATE
@ -1012,6 +1016,7 @@ add_library(ge_single_op STATIC ${SINGLE_OP_SRC_FILES} ${PROTO_HDRS})
target_compile_definitions(ge_single_op PRIVATE
google=ascend_private
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
)
target_compile_options(ge_single_op PRIVATE
@ -1108,6 +1113,8 @@ target_compile_options(ut_libge_distinct_load_utest PRIVATE
target_compile_definitions(ut_libge_distinct_load_utest PRIVATE
google=ascend_private
DAVINCI_SUPPORT_PROFILING
$<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
)
target_link_libraries(ut_libge_distinct_load_utest

6
tests/ut/ge/profiling/ge_profiling_manager_unittest.cc
Unescape View File

@ -78,3 +78,9 @@ TEST_F(UtestGeProfilinganager, plungin_init_) {
EXPECT_EQ(ret, INTERNAL_ERROR);
ProfilingManager::Instance().prof_cb_.msprofReporterCallback = nullptr;
}
TEST_F(UtestGeProfilinganager, test_step_info) {
ProfilingManager::Instance().prof_cb_.msprofReporterCallback = ReporterCallback;
EXPECT_EQ(ProfilingManager::Instance().ProfileStepInfo(0, 0, 0, nullptr, 0), SUCCESS);
ProfilingManager::Instance().prof_cb_.msprofReporterCallback = nullptr;
}

Write Preview
Loading…
Cancel
Save