From 877569858f88218bc8ed61d1084145c387c2b5aa Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 18 Dec 2020 15:38:33 +0800 Subject: [PATCH 1/5] parse bp fp point --- ge/common/profiling/profiling_manager.cc | 37 ++++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index 214f58f4..f6b1c0ce 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -809,28 +809,41 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpP return; } // ProfApi mode and training trace is set - try { - char env_profiling_options[MSPROF_OPTIONS_DEF_LEN_MAX] = { 0x00 }; + // Parse options first + char env_profiling_options[MSPROF_OPTIONS_DEF_LEN_MAX] = { 0x00 }; + bool is_profiling_valid = false; + std::string profiling_options; + if (ge::GetContext().GetOption(OPTION_EXEC_PROFILING_OPTIONS, profiling_options) == SUCCESS && + !profiling_options.empty()) { + is_profiling_valid = true; + } else { INT32 ret = mmGetEnv("PROFILING_OPTIONS", env_profiling_options, MSPROF_OPTIONS_DEF_LEN_MAX); if (ret != EN_OK) { GELOGI("PROFILING_OPTIONS env is not exist."); return; } GELOGI("Parse env PROFILING_OPTIONS:%s.", env_profiling_options); - Json prof_options = Json::parse(env_profiling_options); + profiling_options = env_profiling_options; + is_profiling_valid = true; + } + if (is_profiling_valid) { + try { + Json prof_options = Json::parse(profiling_options); - fp_point_ = prof_options[kFpPoint]; - bp_point_ = prof_options[kBpPoint]; + fp_point_ = prof_options[kFpPoint]; + bp_point_ = prof_options[kBpPoint]; - fp_point = fp_point_; - bp_point = bp_point_; - if (!fp_point_.empty() && !bp_point_.empty()) { - GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); + fp_point = fp_point_; + bp_point = bp_point_; + if (!fp_point_.empty() && !bp_point_.empty()) { + GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); + } + } catch (...) { + GELOGW("Json prof options is invalid."); + return; } - } catch (...) { - GELOGE(FAILED, "Json prof options is invalid."); - return; } + return; } From b3005174f071a46d52a4091335ce4fd0947351d9 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 18 Dec 2020 19:59:42 +0800 Subject: [PATCH 2/5] update profiling format --- ge/common/profiling/ge_profiling.cc | 4 +-- ge/common/profiling/profiling_manager.cc | 3 +- ge/common/profiling/profiling_manager.h | 33 ++++++++++--------- inc/framework/common/profiling/ge_profiling.h | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index 640f77a1..2717f215 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -112,7 +112,6 @@ ge::Status RegProfCtrlCallback(MsprofCtrlCallback func) { if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { GELOGW("Msprof ctrl callback is exist, just ignore it."); } else { - GELOGI("GE register Msprof ctrl callback."); ge::ProfilingManager::Instance().SetMsprofCtrlCallback(func); } return ge::SUCCESS; @@ -124,7 +123,6 @@ ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { return ge::PARAM_INVALID; } // Pass MsprofSetDeviceCallback to runtime - GELOGI("GE pass setdevice callback to runtime."); ge::Status rt_ret = rtRegDeviceStateCallback(kRtSetDeviceRegName.c_str(), static_cast(func)); if (rt_ret != ge::SUCCESS) { GELOGE(rt_ret, "Pass MsprofSetDeviceCallback to runtime failed!"); @@ -158,7 +156,7 @@ ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t le if (type != kProfCommandhandleFinalize) { GE_CHECK_NOTNULL(data); } - ProfCommandHandleData *prof_config_param = (ProfCommandHandleData *)data; + ProfCommandHandleData *prof_config_param = reinterpret_cast(data); auto iter = kProfCommandTypeMap.find(type); if (iter == kProfCommandTypeMap.end()) { GELOGW("The prof comand type is invalid."); diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index f6b1c0ce..01369a2c 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -805,7 +805,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpP if (!fp_point_.empty() && !bp_point_.empty()) { fp_point = fp_point_; bp_point = bp_point_; - GELOGI("Bp Fp have been initialized in env or options. bp_point: %s, fp_point: %s", bp_point.c_str(), fp_point.c_str()); + GELOGI("Bp Fp have been initialized in env or options. bp_point: %s, fp_point: %s", bp_point.c_str(), + fp_point.c_str()); return; } // ProfApi mode and training trace is set diff --git a/ge/common/profiling/profiling_manager.h b/ge/common/profiling/profiling_manager.h index 5fa4fac4..718c8cc9 100755 --- a/ge/common/profiling/profiling_manager.h +++ b/ge/common/profiling/profiling_manager.h @@ -36,21 +36,21 @@ using Json = nlohmann::json; namespace { const std::string GE_PROFILING_MODULE = "Framework"; // DataTypeConfig MASK - #define PROF_ACL_API_MASK 0x0001 - #define PROF_TASK_TIME_MASK 0x0002 - #define PROF_AICORE_METRICS_MASK 0x0004 - #define PROF_AICPU_TRACE_MASK 0x0008 - #define PROF_MODEL_EXECUTE_MASK 0x0010 - #define PROF_RUNTIME_API_MASK 0x0020 - #define PROF_RUNTIME_TRACE_MASK 0x0040 - #define PROF_SCHEDULE_TIMELINE_MASK 0x0080 - #define PROF_SCHEDULE_TRACE_MASK 0x0100 - #define PROF_AIVECTORCORE_METRICS_MASK 0x0200 - #define PROF_SUBTASK_TIME_MASK 0x0400 - #define PROF_TRAINING_TRACE_MASK 0x0800 - #define PROF_HCCL_TRACE_MASK 0x1000 - #define PROF_DATA_PROCESS_MASK 0x2000 - #define PROF_MODEL_LOAD_MASK 0x8000000000000000 + const int PROF_ACL_API_MASK = 0x0001; + const int PROF_TASK_TIME_MASK = 0x0002; + const int PROF_AICORE_METRICS_MASK = 0x0004; + const int PROF_AICPU_TRACE_MASK = 0x0008; + const int PROF_MODEL_EXECUTE_MASK = 0x0010; + const int PROF_RUNTIME_API_MASK = 0x0020; + const int PROF_RUNTIME_TRACE_MASK = 0x0040; + const int PROF_SCHEDULE_TIMELINE_MASK = 0x0080; + const int PROF_SCHEDULE_TRACE_MASK = 0x0100; + const int PROF_AIVECTORCORE_METRICS_MASK = 0x0200; + const int PROF_SUBTASK_TIME_MASK = 0x0400; + const int PROF_TRAINING_TRACE_MASK = 0x0800; + const int PROF_HCCL_TRACE_MASK = 0x1000; + const int PROF_DATA_PROCESS_MASK = 0x2000; + const int PROF_MODEL_LOAD_MASK = 0x8000000000000000; } // namespace namespace ge { @@ -80,7 +80,8 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { bool ProfilingTrainingTraceOn() const { return is_training_trace_; } bool ProfilingModelLoadOn() const { return is_load_profiling_; } bool ProfilingModelExecuteOn() const; - bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // is_execute_profiling_ only used by ge option and env + // is_execute_profiling_ only used by ge option and env + bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } void ReportProfilingData(uint32_t model_id, const std::vector &task_desc_info, const std::vector &compute_graph_desc_info); void ProfilingTaskDescInfo(uint32_t model_id, const std::vector &task_desc_info, diff --git a/inc/framework/common/profiling/ge_profiling.h b/inc/framework/common/profiling/ge_profiling.h index e56411c9..1e8b29d7 100644 --- a/inc/framework/common/profiling/ge_profiling.h +++ b/inc/framework/common/profiling/ge_profiling.h @@ -20,7 +20,7 @@ #include "ge/ge_api_error_codes.h" #include "toolchain/prof_callback.h" -#define MAX_DEV_NUM (64) +const int MAX_DEV_NUM = 64; enum ProfCommandHandleType { kProfCommandhandleInit = 0, kProfCommandhandleStart, From e177e5c001925b420d294f37559bf0a64f2ae56b Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 18 Dec 2020 20:18:30 +0800 Subject: [PATCH 3/5] optimize log print --- ge/common/profiling/ge_profiling.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index 2717f215..43ed6434 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -181,7 +181,8 @@ ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t le if (type != kProfCommandhandleFinalize) { command.module_index = prof_config_param->profSwitch; } - GELOGI("GE commandhandle execute, Command Type: %d, data type config: 0x%llx", type, command.module_index); + GELOGI("GE commandhandle execute, Command Type: %s, data type config: 0x%llx", iter->second.c_str(), + command.module_index); if (type == kProfCommandhandleStart || type == kProfCommandhandleStop) { GELOGI("Profiling device nums:%s , deviceID:[%s]", prof_params[0].c_str(), prof_params[kDeviceListIndex].c_str()); } From 535077a0671999ba7bef8e067f1aa2f2785cb581 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 18 Dec 2020 20:22:22 +0800 Subject: [PATCH 4/5] formate ge_profiling.h --- inc/framework/common/profiling/ge_profiling.h | 1 + 1 file changed, 1 insertion(+) diff --git a/inc/framework/common/profiling/ge_profiling.h b/inc/framework/common/profiling/ge_profiling.h index 1e8b29d7..f60f60f2 100644 --- a/inc/framework/common/profiling/ge_profiling.h +++ b/inc/framework/common/profiling/ge_profiling.h @@ -21,6 +21,7 @@ #include "toolchain/prof_callback.h" const int MAX_DEV_NUM = 64; + enum ProfCommandHandleType { kProfCommandhandleInit = 0, kProfCommandhandleStart, From 1488a559be71a1b3f1af3f87cd0c07edd84072f4 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 18 Dec 2020 20:25:43 +0800 Subject: [PATCH 5/5] update format --- ge/common/profiling/profiling_manager.h | 30 +++++++++---------- inc/framework/common/profiling/ge_profiling.h | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ge/common/profiling/profiling_manager.h b/ge/common/profiling/profiling_manager.h index 718c8cc9..22fa8f8c 100755 --- a/ge/common/profiling/profiling_manager.h +++ b/ge/common/profiling/profiling_manager.h @@ -36,21 +36,21 @@ using Json = nlohmann::json; namespace { const std::string GE_PROFILING_MODULE = "Framework"; // DataTypeConfig MASK - const int PROF_ACL_API_MASK = 0x0001; - const int PROF_TASK_TIME_MASK = 0x0002; - const int PROF_AICORE_METRICS_MASK = 0x0004; - const int PROF_AICPU_TRACE_MASK = 0x0008; - const int PROF_MODEL_EXECUTE_MASK = 0x0010; - const int PROF_RUNTIME_API_MASK = 0x0020; - const int PROF_RUNTIME_TRACE_MASK = 0x0040; - const int PROF_SCHEDULE_TIMELINE_MASK = 0x0080; - const int PROF_SCHEDULE_TRACE_MASK = 0x0100; - const int PROF_AIVECTORCORE_METRICS_MASK = 0x0200; - const int PROF_SUBTASK_TIME_MASK = 0x0400; - const int PROF_TRAINING_TRACE_MASK = 0x0800; - const int PROF_HCCL_TRACE_MASK = 0x1000; - const int PROF_DATA_PROCESS_MASK = 0x2000; - const int PROF_MODEL_LOAD_MASK = 0x8000000000000000; + const uint64_t PROF_ACL_API_MASK = 0x0001; + const uint64_t PROF_TASK_TIME_MASK = 0x0002; + const uint64_t PROF_AICORE_METRICS_MASK = 0x0004; + const uint64_t PROF_AICPU_TRACE_MASK = 0x0008; + const uint64_t PROF_MODEL_EXECUTE_MASK = 0x0010; + const uint64_t PROF_RUNTIME_API_MASK = 0x0020; + const uint64_t PROF_RUNTIME_TRACE_MASK = 0x0040; + const uint64_t PROF_SCHEDULE_TIMELINE_MASK = 0x0080; + const uint64_t PROF_SCHEDULE_TRACE_MASK = 0x0100; + const uint64_t PROF_AIVECTORCORE_METRICS_MASK = 0x0200; + const uint64_t PROF_SUBTASK_TIME_MASK = 0x0400; + const uint64_t PROF_TRAINING_TRACE_MASK = 0x0800; + const uint64_t PROF_HCCL_TRACE_MASK = 0x1000; + const uint64_t PROF_DATA_PROCESS_MASK = 0x2000; + const uint64_t PROF_MODEL_LOAD_MASK = 0x8000000000000000; } // namespace namespace ge { diff --git a/inc/framework/common/profiling/ge_profiling.h b/inc/framework/common/profiling/ge_profiling.h index f60f60f2..83699754 100644 --- a/inc/framework/common/profiling/ge_profiling.h +++ b/inc/framework/common/profiling/ge_profiling.h @@ -33,7 +33,7 @@ enum ProfCommandHandleType { struct ProfCommandHandleData { uint64_t profSwitch; - uint32_t devNums; // length of device id list + uint32_t devNums; // length of device id list uint32_t devIdList[MAX_DEV_NUM]; uint32_t modelId; };