From 07f5327b18593f9308ddf315278b40069fdd3bd9 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 15:52:41 +0800 Subject: [PATCH] profiling AR version 1 --- ge/CMakeLists.txt | 13 +- ge/client/module.mk | 1 - ge/common/profiling/ge_profiling.cc | 176 ++++++ ge/common/profiling/ge_profiling_cb.cc | 26 + ge/common/profiling/profiling_manager.cc | 513 +++++------------- ge/common/profiling/profiling_manager.h | 97 +--- ge/executor/CMakeLists.txt | 1 + ge/executor/ge_executor.cc | 2 +- ge/executor/module.mk | 1 + ge/ge_runner.mk | 3 +- .../load/new_model_manager/davinci_model.cc | 180 +++--- .../load/new_model_manager/davinci_model.h | 2 +- .../load/new_model_manager/model_manager.cc | 43 +- .../load/new_model_manager/model_manager.h | 2 - ge/hybrid/executor/worker/execution_engine.cc | 3 +- ge/init/gelib.cc | 6 +- ge/single_op/single_op.cc | 3 +- ge/stub/gen_stubapi.py | 2 +- inc/framework/common/profiling/ge_profiling.h | 45 ++ .../common/profiling/ge_profiling_cb.h | 24 + 20 files changed, 510 insertions(+), 633 deletions(-) create mode 100644 ge/common/profiling/ge_profiling.cc create mode 100644 ge/common/profiling/ge_profiling_cb.cc create mode 100644 inc/framework/common/profiling/ge_profiling.h create mode 100644 inc/framework/common/profiling/ge_profiling_cb.h diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index bd0b0c39..7079e432 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -60,6 +60,8 @@ set(TRAIN_SRC_LIST "common/dump/dump_manager.cc" "common/dump/dump_properties.cc" "common/dump/dump_op.cc" + "common/profiling/ge_profiling.cc" + "common/profiling/ge_profiling_cb.cc" "engine_manager/dnnengine_manager.cc" "ge_local_engine/engine/host_cpu_engine.cc" "generator/ge_generator.cc" @@ -332,7 +334,6 @@ set(TRAIN_SRC_LIST "hybrid/hybrid_davinci_model.cc" "executor/ge_executor.cc" "client/ge_api.cc" - "client/ge_prof.cc" "analyzer/analyzer.cc" "ir_build/ge_ir_build.cc" "ir_build/atc_ir_common.cc" @@ -649,7 +650,7 @@ target_link_libraries(ge_runner $ ge_memory adump_server - msprofiler + msprofiler_fwk static_mmpa -Wl,--no-as-needed graph @@ -658,7 +659,6 @@ target_link_libraries(ge_runner register c_sec slog - msprof runtime resource error_manager @@ -777,13 +777,12 @@ target_link_libraries(opensrc_ascendcl PRIVATE register_static error_manager_static adump_server - msprofiler + msprofiler_fwk -Wl,--no-whole-archive -Wl,--no-as-needed c_sec runtime slog - msprof ascend_hal_stub -Wl,--as-needed -lrt @@ -799,12 +798,10 @@ set_target_properties(opensrc_ascendcl PROPERTIES add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc - ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc COMMAND echo "Generating stub files." && ${HI_PYTHON} ${CMAKE_CURRENT_LIST_DIR}/stub/gen_stubapi.py ${GE_CODE_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} && mv ge_ir_build.cc stub_ge_ir_build.cc && mv ge_api.cc stub_ge_api.cc - && mv ge_prof.cc stub_ge_prof.cc && echo "Generating stub files end." #WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} #DEPENDS stub/gen_stubapi.py ${TOP_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} @@ -813,7 +810,6 @@ add_custom_command( add_custom_target(ge_stub DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc - ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc ) ################################################################## @@ -855,7 +851,6 @@ target_include_directories(atc_stub_ge_compiler PRIVATE ############ stub/libge_runner.so ############ add_library(fwk_stub_ge_runner SHARED stub_ge_api.cc - stub_ge_prof.cc stub_ge_ir_build.cc ) diff --git a/ge/client/module.mk b/ge/client/module.mk index 6ac69d31..c942d550 100644 --- a/ge/client/module.mk +++ b/ge/client/module.mk @@ -4,7 +4,6 @@ LOCAL_PATH := $(call my-dir) COMMON_LOCAL_SRC_FILES := \ proto/ge_api.proto \ ge_api.cc \ - ge_prof.cc \ COMMON_LOCAL_C_INCLUDES := \ diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc new file mode 100644 index 00000000..8da6b12a --- /dev/null +++ b/ge/common/profiling/ge_profiling.cc @@ -0,0 +1,176 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/profiling/ge_profiling.h" +#include "runtime/base.h" +#include "common/profiling/profiling_manager.h" +#include "framework/common/debug/ge_log.h" +#include "framework/common/debug/log.h" +#include "graph/load/graph_loader.h" +#include "init/gelib.h" +#include "framework/common/ge_inner_error_codes.h" + +namespace { +const uint32_t kDeviceListIndex = 3; +const std::string kDeviceNums = "devNums"; +const std::string kDeviceIdList = "devIdList"; +const std::string kProfilingInit = "prof_init"; +const std::string kProfilingFinalize = "prof_finalize"; +const std::string kProfilingStart = "prof_start"; +const std::string kProfilingStop = "prof_stop"; +const std::string kProfModelSubscribe = "prof_model_subscribe"; +const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; +const std::string kRtSetDeviceRegName = "profiling"; + +const std::map kProfCommandTypeMap = { + {kProfCommandhandleInit, kProfilingInit}, + {kProfCommandhandleStart, kProfilingStart}, + {kProfCommandhandleStop, kProfilingStop}, + {kProfCommandhandleFinalize, kProfilingFinalize}, + {kProfCommandhandleModelSubscribe, kProfModelSubscribe}, + {kProfCommandhandleModelUnsubscribe, kProfModelUnsubscribe}}; +} // namespace + +bool TransProfConfigToParam(const MsprofCommandHandle &profCommand, vector &prof_config_params) { + prof_config_params.clear(); + prof_config_params.emplace_back(kDeviceNums); + prof_config_params.emplace_back(std::to_string(profCommand.devNums)); + prof_config_params.emplace_back(kDeviceIdList); + std::string devID = ""; + if (profCommand.devNums == 0) { + GELOGW("The device num is invalid."); + return false; + } + for (uint32_t i = 0; i < profCommand.devNums; i++) { + devID.append(std::to_string(profCommand.devIdList[i])); + if (i != profCommand.devNums - 1) { + devID.append(","); + } + } + + prof_config_params.push_back(devID); + return true; +} + +bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { + if (deviceid_list == nullptr) { + GELOGE(ge::PARAM_INVALID, "deviceIdList is nullptr"); + return false; + } + if (device_nums == 0 || device_nums > MAX_DEV_NUM) { + GELOGE(ge::PARAM_INVALID, "The device nums is invalid."); + return false; + } + + // real device num + int32_t dev_count = 0; + rtError_t rt_err = rtGetDeviceCount(&dev_count); + if (rt_err != RT_ERROR_NONE) { + GELOGE(ge::INTERNAL_ERROR, "Get the Device count fail."); + return false; + } + + if (device_nums > static_cast(dev_count)) { + GELOGE(ge::PARAM_INVALID, "Device num(%u) is not in range 1 ~ %d.", device_nums, dev_count); + return false; + } + + std::unordered_set record; + for (size_t i = 0; i < device_nums; ++i) { + uint32_t dev_id = deviceid_list[i]; + if (dev_id >= static_cast(dev_count)) { + GELOGE(ge::PARAM_INVALID, "Device id %u is not in range 0 ~ %d(exclude %d)", dev_id, dev_count, dev_count); + return false; + } + if (record.count(dev_id) > 0) { + GELOGE(ge::PARAM_INVALID, "Device id %u is duplicatedly set", dev_id); + return false; + } + record.insert(dev_id); + } + return true; +} + +ge::Status RegProfCtrlCallback(MsprofCtrlCallback func) { + if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { + GELOGW("Msprof ctrl callback is exist, just ignore it."); + } else { + ge::ProfilingManager::Instance().SetMsprofCtrlCallback(func); + } + return ge::SUCCESS; +} + +ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { + // Pass MsprofSetDeviceCallback to runtime + ge::Status rt_ret = rtRegDeviceStateCallback(kRtSetDeviceRegName.c_str(), static_cast(func)); + if (rt_ret != ge::SUCCESS) { + GELOGE(rt_ret, "Pass MsprofSetDeviceCallback to runtime failed!"); + return rt_ret; + } + return ge::SUCCESS; +} + +ge::Status RegProfReporterCallback(MsprofReporterCallback func) { + if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { + GELOGW("Msprof ctrl callback is exist, just ignore it."); + } else { + ge::ProfilingManager::Instance().SetMsprofReporterCallback(func); + } + // Pass MsprofReporterCallback to runtime + ge::Status rt_ret = rtSetMsprofReporterCallback(func); + if (rt_ret != ge::SUCCESS) { + GELOGE(rt_ret, "Pass MsprofReporterCallback to runtime failed!!"); + return rt_ret; + } + // Pass MsprofReporterCallback to hccl in opskernel so initialize + + return ge::SUCCESS; +} + +ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len) { + GE_CHECK_NOTNULL(data); + MsprofCommandHandle *prof_config_param = (MsprofCommandHandle *)data; + if (!isProfConfigValid(prof_config_param->devIdList, prof_config_param->devNums)) { + return ge::FAILED; + } + std::vector prof_params; + if (!TransProfConfigToParam(*prof_config_param, prof_params)) { + GELOGE(ge::PARAM_INVALID, "Transfer profilerConfig to string vector failed"); + return ge::PARAM_INVALID; + } + auto iter = kProfCommandTypeMap.find(type); + if (iter == kProfCommandTypeMap.end()) { + GELOGW("The prof comand type is invalid."); + return ge::PARAM_INVALID; + } + ge::GraphLoader graph_loader; + ge::Command command; + command.cmd_params.clear(); + command.cmd_type = iter->second; + command.cmd_params = prof_params; + command.module_index = prof_config_param->profSwitch; + GELOGI("GE commandhandle execute, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), + prof_params[kDeviceListIndex].c_str(), command.module_index); + ge::Status ret = graph_loader.CommandHandle(command); + if (ret != ge::SUCCESS) { + GELOGE(ret, "Handle profiling command failed"); + return ge::FAILED; + } + + GELOGI("Successfully execute profiling command 0x%llx.", command.module_index); + return ge::SUCCESS; +} + diff --git a/ge/common/profiling/ge_profiling_cb.cc b/ge/common/profiling/ge_profiling_cb.cc new file mode 100644 index 00000000..230e129d --- /dev/null +++ b/ge/common/profiling/ge_profiling_cb.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/profiling/ge_profiling_cb.h" +#include "init/gelib.h" + +bool IsInitialize() { + std::shared_ptr instance_ptr = ge::GELib::GetInstance(); + if (instance_ptr == nullptr || instance_ptr->InitFlag() == false) { + return false; + } + return true; +} diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index ba03d671..bcf6d366 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -45,7 +45,10 @@ const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; } // namespace namespace ge { -ProfilingManager::ProfilingManager() : subscribe_count_(0) {} +ProfilingManager::ProfilingManager() : is_load_profiling_(false), + is_execute_profiling_(false), + is_training_trace_(false), + subscribe_count_(0) {} ProfilingManager::~ProfilingManager() {} @@ -58,44 +61,25 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In #ifdef DAVINCI_SUPPORT_PROFILING vector().swap(device_id_); subscribe_count_ = 0; - job_id_ = options.job_id; + GELOGI("ProfilingManager::Init job_id:%s", options.job_id.c_str()); - GELOGI("ProfilingManager::Init job_id:%s", job_id_.c_str()); - - - - Status ret; - if (!recv_profiling_config_.empty()) { - GELOGI("Profiling json config from acl:%s", recv_profiling_config_.c_str()); - ret = InitFromAclCfg(recv_profiling_config_); - } else { - ret = InitFromOptions(options); - if (ret == SUCCESS && is_load_profiling_) { - device_id_.push_back(options.device_id); - } - } + struct MsprofGeOptions prof_conf = { 0 }; + Status ret = InitFromOptions(options, prof_conf); if (ret != SUCCESS) { GELOGE(ret, "Failed to init profiling."); return ret; } if (is_load_profiling_) { - // register Framework to profiling - int result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); - if (result != 0) { - GELOGE(FAILED, "Register profiling engine failed."); + int32_t cb_ret = prof_cb_.msprofCtrlCallback( + static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), + static_cast(&prof_conf), sizeof(MsprofGeOptions)); + if (cb_ret != 0) { + GELOGE(FAILED, "call msprofCtrlCallback failed, type:%u, return:%d", + static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), cb_ret); return FAILED; } - // profiling startup first time - GELOGI("Begin to init profiling, device num %zu", device_id_.size()); - for (size_t i = 0; i < device_id_.size(); ++i) { - ret = StartProfiling(0, device_id_[i]); - if (ret != SUCCESS) { - GELOGW("Profiling start failed on device %d.", device_id_[i]); - continue; - } - GELOGI("Profiling init succ on device %d.", device_id_[i]); - } + GELOGI("Profiling init success"); } else { GELOGI("The profiling is off, skip the initialization"); } @@ -103,264 +87,55 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In return SUCCESS; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromAclCfg( - const std::string &config) { +ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOptions &prof_conf) { #ifdef DAVINCI_SUPPORT_PROFILING - try { - is_load_profiling_ = false; - is_execute_profiling_ = false; - profiling_opts_.clear(); - op_trace_conf_.clear(); - Json start_prof_conf = Json::parse(config); - Json &prof_conf = start_prof_conf[kStartCfg][0]; - job_id_ = prof_conf[kJobID]; - auto iter = prof_conf.find(kProfDir); - if (iter != prof_conf.end()) { - prof_dir_ = prof_conf[kProfDir]; - } - Json &device_id = prof_conf[kDeviceID]; - if (device_id.size() != 0) { - vector().swap(device_id_); - bool is_all = false; - for (size_t i = 0; i < device_id.size(); i++) { - std::string device_id_str = device_id[i].get(); - if (device_id_str == "all") { - is_all = true; - break; - } - device_id_.push_back(std::stoi(device_id_str)); - } - if (is_all) { - int32_t count = 0; - rtError_t rt_err = rtGetDeviceCount(&count); - if (rt_err != RT_ERROR_NONE) { - GELOGE(FAILED, "Call rtGetDeviceCount to get device failed."); - } + // enable profiling by env + char env_profiling_mode[MMPA_MAX_PATH] = { 0x00 }; + is_load_profiling_ = false; + is_execute_profiling_ = false; - vector().swap(device_id_); - for (int32_t i = 0; i < count; ++i) { - device_id_.push_back(i); - } - } - } + (void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); + (void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, sizeof(MsprofGeOptions)); - Json &features = prof_conf[kFeatures]; - if (ParseFeaturesFromAclCfg(features) != SUCCESS) { - GELOGE(FAILED, "Parse feature from acl cfg failed."); - return FAILED; - } + if ((env_profiling_mode != nullptr) && (strcmp("true", env_profiling_mode) == 0) + && (strcmp(prof_conf.options, "\0") != 0)) { + // enable profiling by env is_load_profiling_ = true; is_execute_profiling_ = true; - } catch (...) { - GELOGE(FAILED, "Json conf is not invalid !"); - return ge::PARAM_INVALID; - } -#endif - return ge::SUCCESS; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::ParseFeaturesFromAclCfg( - const Json &features) { -#ifdef DAVINCI_SUPPORT_PROFILING - try { - for (size_t i = 0; i < features.size(); ++i) { - const Json &feature = features[i]; - if ((feature.find(kName) == feature.end()) || feature[kName].is_null()) { - continue; - } - const std::string &name = feature[kName]; - if (name == "op_trace") { - const Json &conf = feature[kConf]; - const Json &events = conf[0][kEvents]; - const std::string &ai_core_events = events[0][kAiCoreEvents]; - GELOGI("Op trace config from acl ai_core_events:%s", ai_core_events.c_str()); - is_op_trace_ = true; - ProfMgrConf prof_mgr_conf; - int result = ProfMgrGetConf(ai_core_events, &prof_mgr_conf); - if (result != 0) { - GELOGE(FAILED, "ProfMgrGetConf failed."); - return FAILED; - } - op_trace_conf_ = prof_mgr_conf.conf; - op_trace_iter_num_ = static_cast(op_trace_conf_.size()); - GELOGI("Op trace profiling iter num %d,", op_trace_iter_num_); - } else if (name == "task_trace") { - is_op_trace_ = false; - if (feature.find(kConf) != feature.end()) { - const Json &conf = feature[kConf]; - std::stringstream task_trace_conf; - task_trace_conf << conf; - task_trace_conf_ = task_trace_conf.str(); - } - GELOGI("Task trace config from acl"); - } else if (name == "system_trace") { - is_op_trace_ = false; - const Json &conf = feature[kConf]; - std::stringstream system_trace_conf; - system_trace_conf << conf; - system_trace_conf_ = system_trace_conf.str(); - GELOGI("System trace config from acl"); - } - profiling_opts_.push_back(name); - } - } catch (...) { - GELOGE(ge::PARAM_INVALID, "Json conf feature is not invalid !"); - return ge::PARAM_INVALID; - } -#endif - return ge::SUCCESS; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromOptions(const Options &options) { -#ifdef DAVINCI_SUPPORT_PROFILING - // enable profiling support two ways: env and front end - char profiling_mode_temp[MMPA_MAX_PATH] = { 0x00 }; - char prof_options_temp[MMPA_MAX_PATH] = { 0x00 }; - (void)mmGetEnv("PROFILING_MODE", profiling_mode_temp, MMPA_MAX_PATH); - (void)mmGetEnv("PROFILING_OPTIONS", prof_options_temp, MMPA_MAX_PATH); - const char *profiling_mode = profiling_mode_temp; - const char *prof_options = prof_options_temp; - if ((profiling_mode == nullptr) || (strcmp("true", profiling_mode) != 0) || (prof_options == nullptr)) { - is_load_profiling_ = false; - is_execute_profiling_ = false; + GELOGI("The profiling in env is %s, %s", env_profiling_mode, prof_conf.options); } else { - std::string prof_options_str = std::string(prof_options); - profiling_opts_ = StringUtils::Split(prof_options_str, ':'); - is_load_profiling_ = true; - is_execute_profiling_ = true; - GELOGI("The profiling in env is %s, %s", profiling_mode, prof_options); - } - if (!is_load_profiling_) { - const std::string enable_profiling = "1"; - if (options.profiling_mode != enable_profiling || options.profiling_options.empty()) { - is_load_profiling_ = false; - is_execute_profiling_ = false; + if (options.profiling_mode != "1" || options.profiling_options.empty()) { return SUCCESS; - } else { - profiling_opts_ = StringUtils::Split(options.profiling_options, ':'); - is_load_profiling_ = true; - is_execute_profiling_ = true; - GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), options.profiling_options.c_str()); } - } - // features:'training_trace', 'task_trace' or 'op_trace' etc - if (!profiling_opts_.empty()) { - if (profiling_opts_[0] == "op_trace") { - is_op_trace_ = true; - // op trace get conf - ProfMgrConf prof_mgr_conf; - int result = ProfMgrGetConf("", &prof_mgr_conf); - if (result != 0) { - GELOGE(FAILED, "ProfMgrGetConf failed."); - return FAILED; - } - op_trace_conf_ = prof_mgr_conf.conf; - op_trace_iter_num_ = static_cast(op_trace_conf_.size()); - GELOGI("op trace profiling iter num %d,", op_trace_iter_num_); - } else { - is_op_trace_ = false; - op_trace_iter_num_ = 1; + // enable profiling by ge option + if (memcpy_s(prof_conf.options, sizeof(prof_conf.options), options.profiling_options.c_str(), + sizeof(options.profiling_options.c_str())) != EOK) { + GELOGE(INTERNAL_ERROR, "copy profiling_options failed."); + return INTERNAL_ERROR; } + is_load_profiling_ = true; + is_execute_profiling_ = true; + GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), prof_conf.options); } -#endif - return ge::SUCCESS; -} -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::StartProfiling(int32_t iter_num, - int32_t device_id) { -#ifdef DAVINCI_SUPPORT_PROFILING - if (!profiling_opts_.empty()) { - GELOGI("Start profiling index is %d", iter_num); - // current one docker only use one device - Json p_device; - - try { - // profiling need physical_device_id - p_device[kDeviceID] = std::to_string(device_id); - p_device[kJobID] = job_id_; - p_device[kTraceID] = std::to_string(GetContext().TraceId()); - if (!prof_dir_.empty()) { - p_device[kProfDir] = prof_dir_; - GELOGI("Prof dir: %s.", prof_dir_.c_str()); - } - - Json features; - if (is_op_trace_) { - Json f; - f[kName] = "op_trace"; - Json conf; - if (op_trace_conf_.size() <= static_cast(iter_num)) { - GELOGE(FAILED, "Op trace iter num is invalid!"); - return FAILED; - } - Json events; - events[0] = nlohmann::json::parse(op_trace_conf_[iter_num]); - conf[0][kEvents] = events; - f[kConf] = conf; - features[0] = f; - if (iter_num == 0) { - is_load_ = true; - } - } else { - for (std::vector::size_type i = 0; i < profiling_opts_.size(); i++) { - Json f; - if (profiling_opts_[i] == "system_trace") { - f[kConf] = nlohmann::json::parse(system_trace_conf_); - } else if (profiling_opts_[i] == "task_trace") { - if (!task_trace_conf_.empty()) { - f[kConf] = nlohmann::json::parse(task_trace_conf_); - } - } - f[kName] = profiling_opts_[i]; - features[i] = f; - } - is_load_ = true; - } - p_device[kFeatures] = features; - // only one device, but sProfMgrStartUp API require for device list - Json devices; - devices[0] = p_device; - - Json start_cfg; - start_cfg[kStartCfg] = devices; - - // convert json to string - std::stringstream ss; - ss << start_cfg; - send_profiling_config_ = ss.str(); - GELOGI("Profiling config %s\n", send_profiling_config_.c_str()); - } catch (...) { - GELOGE(FAILED, "Op trace json conf is not invalid !"); - return FAILED; - } + if (!is_load_profiling_) { + return SUCCESS; + } - // runtime startup for profiling - uint64_t module = GetProfilingModule(); - int32_t device_num = 1; - uint32_t device_id_rt = static_cast(device_id); - GE_CHK_RT_RET(rtProfilerStart(module, device_num, &device_id_rt)); - - // call profiling startup API - ProfMgrCfg prof_cfg = {send_profiling_config_}; - void *prof_handle = ProfMgrStartUp(&prof_cfg); - if (prof_handle == nullptr) { - GELOGW("ProfMgrStartUp failed on device %d ", device_id); - return FAILED; - } - GELOGD("StartProfiling, prof_handle: %p", prof_handle); - prof_handle_vec_.push_back(prof_handle); + if (memcpy_s(prof_conf.jobId, sizeof(prof_conf.jobId), options.job_id.c_str(), + sizeof(options.job_id.c_str())) != EOK) { + GELOGE(INTERNAL_ERROR, "copy job_id failed."); + return INTERNAL_ERROR; } + + // get traceId from options + prof_conf.traceId = GetContext().TraceId(); #endif - return SUCCESS; + return ge::SUCCESS; } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProfiling() { #ifdef DAVINCI_SUPPORT_PROFILING - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - if (reporter != nullptr) { - int ret = reporter->Flush(); - GELOGI("Report data end, ret is %d", ret); - } uint64_t module = GetProfilingModule(); int32_t device_num = static_cast(device_id_.size()); auto device_id_ptr = std::unique_ptr(new (std::nothrow) uint32_t[device_num]); @@ -376,15 +151,14 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); } - for (size_t i = 0; i < prof_handle_vec_.size(); ++i) { - int result = ProfMgrStop(prof_handle_vec_[i]); - if (result != 0) { - GELOGW("ProfMgr stop return fail:%d, handle:%p", result, prof_handle_vec_[i]); - } + // stop profiling + int32_t cb_ret = prof_cb_.msprofCtrlCallback(static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), + nullptr, 0); + if (cb_ret != 0) { + GELOGW("call msprofCtrlCallback failed, type:%u, return:%d", + static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), cb_ret); + return; } - vector().swap(prof_handle_vec_); - is_load_ = false; - recv_profiling_config_ = ""; GELOGI("Stop Profiling success."); #endif } @@ -392,12 +166,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingTaskDescInfo( uint32_t model_id, const std::vector &task_desc_info, const int32_t &device_id) { #ifdef DAVINCI_SUPPORT_PROFILING - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - if (reporter == nullptr) { - GELOGI("Profiling report is nullptr!"); - return; - } - std::string data; for (const auto &task : task_desc_info) { std::string model_name = task.model_name; @@ -412,7 +180,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin .append(std::to_string(stream_id)).append(" ") .append(std::to_string(model_id)).append("\n")); - Msprof::Engine::ReporterData reporter_data{}; + ReporterData reporter_data; reporter_data.deviceId = device_id; reporter_data.data = (unsigned char *)data.c_str(); reporter_data.dataLen = data.size(); @@ -422,9 +190,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin return; } - ret = reporter->Report(&reporter_data); - if (ret != SUCCESS) { - GELOGE(ret, "Reporter data of task_desc_info fail!"); + int32_t cb_ret = CallMsprofReport(reporter_data); + if (cb_ret != 0) { + GELOGE(cb_ret, "Reporter data of task_desc_info failed, ret:%d", cb_ret); return; } } @@ -436,9 +204,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingGraphDescInfo( uint32_t model_id, const std::vector &compute_graph_desc_info, const int32_t &device_id) { #ifdef DAVINCI_SUPPORT_PROFILING - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return;); - std::string data; for (const auto &graph : compute_graph_desc_info) { data.append("model_name:") @@ -493,64 +258,52 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin } data.append(" model_id:").append(std::to_string(model_id)); - data.append("\n"); - Msprof::Engine::ReporterData reporter_data{}; - Report(device_id, data, *reporter, reporter_data); - + GraphDescReport(device_id, data); data.clear(); } #endif } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Report( - const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, - Msprof::Engine::ReporterData &reporter_data) { +void ProfilingManager::GraphDescReport(const int32_t &device_id, const string &data) { #ifdef DAVINCI_SUPPORT_PROFILING + ReporterData reporter_data; + int ret = -1; + int32_t cb_ret = -1; size_t index = data.size() / kReportMaxLen; if (index >= 1) { reporter_data.deviceId = device_id; - int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); + ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); for (size_t i = 0; i < index; ++i) { reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * i; reporter_data.dataLen = kReportMaxLen; - ret = reporter.Report(&reporter_data); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); + cb_ret = CallMsprofReport(reporter_data); + GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); } reporter_data.dataLen = data.size() - kReportMaxLen * index; if (reporter_data.dataLen != 0) { reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * index; - ret = reporter.Report(&reporter_data); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); + cb_ret = CallMsprofReport(reporter_data); + GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); } } else { reporter_data.deviceId = device_id; reporter_data.data = (unsigned char *)data.c_str(); reporter_data.dataLen = data.size(); - int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); + ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); - ret = reporter.Report(&reporter_data); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); - } -#endif -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit(const std::string &module) const { -#ifdef DAVINCI_SUPPORT_PROFILING - int ret = Msprof::Engine::UnInit(module); - if (ret != SUCCESS) { - GELOGE(ret, "profiling plugin uninit failed, ret:%d", ret); + cb_ret = CallMsprofReport(reporter_data); + GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); } #endif } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportProfilingData( uint32_t model_id, const std::vector &task_desc_info, - const std::vector &compute_graph_desc_info, - bool check_device) { + const std::vector &compute_graph_desc_info) { #ifdef DAVINCI_SUPPORT_PROFILING int32_t logic_device_id = 0; rtError_t rt_ret = rtGetDevice(&logic_device_id); @@ -559,13 +312,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr return; } GELOGD("current logic_device_id:%d", logic_device_id); - if (check_device) { - auto ret = std::find(device_id_.begin(), device_id_.end(), logic_device_id); - if (ret == device_id_.end()) { - GELOGE(FAILED, "get valid phy_device_id failed, profiling report failed."); - return; - } - } GELOGD("start ProfilingTaskDescInfo."); ProfilingTaskDescInfo(model_id, task_desc_info, logic_device_id); GELOGD("start ProfilingGraphDescInfo."); @@ -574,11 +320,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr #endif } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::SetProfilingConfig( - const std::string &profiling_cfg) { - recv_profiling_config_ = profiling_cfg; -} - FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetProfilingModule() { uint64_t module = PROF_MODEL_EXECUTE_MASK | PROF_RUNTIME_API_MASK | @@ -594,9 +335,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetP return module; } -void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, - uint32_t device_id, - uint64_t module) { +void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module) { #ifdef DAVINCI_SUPPORT_PROFILING if (prof_type == kProfModelSubscribe) { if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { @@ -608,9 +347,13 @@ void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, subs_dev_module_[device_id] = dev_info; } } else if (prof_type == kProfModelUnsubscribe) { - if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { - if (subs_dev_module_[device_id].subscribe_count > 0) { - subs_dev_module_[device_id].subscribe_count--; + auto iter = subs_dev_module_.find(device_id); + if (iter != subs_dev_module_.end()) { + if (iter->second.subscribe_count > 0) { + iter->second.subscribe_count--; + } + if (iter->second.subscribe_count == 0) { + subs_dev_module_.erase(iter); } } } else { @@ -626,10 +369,11 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo uint64_t model_load_mask = module & PROF_MODEL_LOAD_MASK; if ((subscribe_count_ == 0) && (model_load_mask == PROF_MODEL_LOAD_MASK)) { // register framework to profiling - int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); - if (result != SUCCESS) { - GELOGE(FAILED, "Register profiling engine failed."); - return FAILED; + // register Framework to profiling + int32_t cb_ret = PluginInit(); + if (cb_ret != 0) { + GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); + return cb_ret; } GELOGI("Prof subscribe: model load profiling on."); } @@ -647,7 +391,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo UpdateSubscribeDeviceModuleMap(kProfModelSubscribe, device[0], module); // Report profiling data - Status p_ret = davinci_model->ReportProfilingData(false); + Status p_ret = davinci_model->ReportProfilingData(); if (p_ret != SUCCESS) { GELOGE(p_ret, "Report profiling data failed."); return p_ret; @@ -672,6 +416,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo auto iter = subs_dev_module_.find(device[0]); if (iter != subs_dev_module_.end()) { if (subs_dev_module_[device[0]].subscribe_count == 1) { + // The same device_id, only stop at last time rtError_t rt_ret = rtProfilerStop(subs_dev_module_[device[0]].module, dev_num, device); if (rt_ret != RT_ERROR_NONE) { GELOGE(FAILED, "Runtime profiler stop failed."); @@ -679,15 +424,15 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo } } UpdateSubscribeDeviceModuleMap(kProfModelUnsubscribe, device[0], subs_dev_module_[device[0]].module); + } else { + GELOGE(FAILED, "The device_id:%u has not been subscribed, do not need to cancel.", device[0]); + return FAILED; } subscribe_count_--; if (subscribe_count_ == 0) { - int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); - if (ret != SUCCESS) { - GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); - return ret; - } + // profiling plugin uninit at last subscription + PluginUnInit(); } #endif return SUCCESS; @@ -700,11 +445,12 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn if (model_load_mask == PROF_MODEL_LOAD_MASK) { // register Framework to profiling - int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); - if (result != SUCCESS) { - GELOGE(FAILED, "Register profiling engine failed."); - return FAILED; + int32_t cb_ret = PluginInit(); + if (cb_ret != 0) { + GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); + return cb_ret; } + int32_t device_num = -1; rtError_t rt_ret = rtProfilerStart(model_load_mask, device_num, nullptr); if (rt_ret != RT_ERROR_NONE) { @@ -719,7 +465,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn if (training_trace_mask == PROF_TRAINING_TRACE_MASK) { is_training_trace_ = true; } - is_acl_api_mode_ = true; GELOGI("Prof init success."); #endif return SUCCESS; @@ -730,12 +475,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi std::lock_guard lock(mutex_); is_load_profiling_ = false; is_training_trace_ = false; - is_acl_api_mode_ = false; - int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); - if (ret != SUCCESS) { - GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); - } + // profiling plugin uninit + PluginUnInit(); + int32_t dev_num = -1; rtError_t rt_ret = rtProfilerStop(PROF_MODEL_LOAD_MASK, dev_num, nullptr); if (rt_ret != RT_ERROR_NONE) { @@ -859,7 +602,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt for (int32_t i = 0; i < device_num; i++) { device_id_ptr[i] = static_cast(device_list[i]); } - GELOGD("Runtime config param: 0x%llx, device num: %d.", module, device_num); + GELOGI("Runtime config param: 0x%llx, device num: %d.", module, device_num); rtError_t rt_ret = rtProfilerStart(module, device_num, device_id_ptr.get()); if (rt_ret != RT_ERROR_NONE) { @@ -878,7 +621,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGW("Prof start: load model module is invalid."); } UpdateDeviceIdModuleMap(kProfStart, module, device_list); - GELOGD("Prof start profiling success."); + GELOGI("Prof start profiling success."); #endif return SUCCESS; } @@ -901,7 +644,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt for (int32_t i = 0; i < device_num; i++) { device_id_ptr[i] = static_cast(device_list[i]); } - GELOGD("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); + GELOGI("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); if (rt_ret != RT_ERROR_NONE) { GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); @@ -921,7 +664,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGW("Prof stop: load model module is invalid."); } UpdateDeviceIdModuleMap(kProfStop, module, device_list); - GELOGD("Prof stop profiling success."); + GELOGI("Prof stop profiling success."); #endif return SUCCESS; } @@ -963,47 +706,43 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ProfilingManager::Profilin if (rt_ret != RT_ERROR_NONE) { GELOGE(rt_ret, "Runtime get logic_device_id failed, current logic_device_id:%d", logic_device_id); } - GELOGD("Current logic_device_id:%d", logic_device_id); + GELOGI("Current logic_device_id:%d", logic_device_id); bool execute_model_prof_on = false; auto iter = std::find(device_id_.begin(), device_id_.end(), logic_device_id); if (iter != device_id_.end()) { execute_model_prof_on = true; } - GELOGD("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); + GELOGI("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); return is_execute_profiling_ || execute_model_prof_on; } -/** - * @brief Profiling PluginImpl - */ -// PluginImpl static variable init -Msprof::Engine::Reporter *PluginImpl::reporter_ = nullptr; - -PluginImpl::PluginImpl(const std::string &module) : module_(module) { GELOGI("Create PluginImpl\n"); } - -int PluginImpl::Init(const Msprof::Engine::Reporter *reporter) { - GELOGI("PluginImpl init"); - reporter_ = const_cast(reporter); - return 0; +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() const { + return prof_cb_.msprofReporterCallback( + static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), + static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_INIT), + nullptr, 0); } -int PluginImpl::UnInit() { - GELOGI("PluginImpl Uninit"); - reporter_ = nullptr; - return 0; +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit() const { +#ifdef DAVINCI_SUPPORT_PROFILING + int32_t cb_ret = prof_cb_.msprofReporterCallback( + static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), + static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_UNINIT), + nullptr, 0); + if (cb_ret != 0) { + GELOGW("profiling plugin uninit failed, ret:%d", cb_ret); + } +#endif } -Msprof::Engine::PluginIntf *ProfilingEngineImpl::CreatePlugin() { - GELOGI(" Create Plugin"); - return new (std::nothrow) PluginImpl(GE_PROFILING_MODULE); +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::CallMsprofReport( + ReporterData &reporter_data) const { + return prof_cb_.msprofReporterCallback( + static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), + static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_REPORT), + static_cast(&reporter_data), sizeof(ReporterData)); } -int ProfilingEngineImpl::ReleasePlugin(Msprof::Engine::PluginIntf *plugin) { - if (plugin != nullptr) { - delete plugin; - plugin = nullptr; - } - return 0; -} + } // namespace ge diff --git a/ge/common/profiling/profiling_manager.h b/ge/common/profiling/profiling_manager.h index 66cefc32..c9434a10 100755 --- a/ge/common/profiling/profiling_manager.h +++ b/ge/common/profiling/profiling_manager.h @@ -26,9 +26,7 @@ #include "framework/common/ge_inner_error_codes.h" #include "framework/common/ge_types.h" #include "external/register/register_types.h" -#include "toolchain/prof_engine.h" -#include "toolchain/prof_mgr_core.h" -#include "toolchain/prof_acl_api.h" +#include "toolchain/prof_callback.h" using std::map; using std::string; @@ -43,29 +41,10 @@ struct DeviceSubsInfo { uint64_t module; uint32_t subscribe_count; }; -// register Plugin -class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY PluginImpl : public Msprof::Engine::PluginIntf { - public: - explicit PluginImpl(const std::string &module); - ~PluginImpl() {} - - int Init(const Msprof::Engine::Reporter *reporter); - int UnInit(); - static Msprof::Engine::Reporter *GetPluginReporter() { return reporter_; } - - private: - static Msprof::Engine::Reporter *reporter_; - std::string module_; -}; -// register Engine -class ProfilingEngineImpl : public Msprof::Engine::EngineIntf { - public: - ProfilingEngineImpl() {} - ~ProfilingEngineImpl() {} - - Msprof::Engine::PluginIntf *CreatePlugin(); - int ReleasePlugin(Msprof::Engine::PluginIntf *plugin); +struct MsprofCallback { + MsprofCtrlCallback msprofCtrlCallback; + MsprofReporterCallback msprofReporterCallback; }; class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { @@ -73,68 +52,50 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { ProfilingManager(); virtual ~ProfilingManager(); static ProfilingManager &Instance(); - ge::Status Init(const Options &options); - ge::Status InitFromOptions(const Options &options); - ge::Status InitFromAclCfg(const std::string &config); - ge::Status StartProfiling(int32_t iter, int32_t device_id); - void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); - ge::Status ProfModelSubscribe(uint64_t module, void *model); - ge::Status ProfModelUnsubscribe(void *model); - ge::Status ProfInit(uint64_t module); - ge::Status ProfFinalize(); - ge::Status ProfStartProfiling(uint64_t module, const std::map &config_para); - ge::Status ProfStopProfiling(uint64_t module, const std::map &config_para); + Status Init(const Options &options); + Status ProfInit(uint64_t module); + Status ProfFinalize(); + Status ProfStartProfiling(uint64_t module, const std::map &config_para); + Status ProfStopProfiling(uint64_t module, const std::map &config_para); + Status ProfModelSubscribe(uint64_t module, void *model); + Status ProfModelUnsubscribe(void *model); void StopProfiling(); - bool ProfilingOpTraceOn() const { return is_op_trace_; } - bool ProfilingLoadFlag() const { return is_load_; } bool ProfilingTrainingTraceOn() const { return is_training_trace_; } bool ProfilingModelLoadOn() const { return is_load_profiling_; } bool ProfilingModelExecuteOn() const; bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // only used by command pattern - bool IsAclApiMode() const { return is_acl_api_mode_; } - int32_t GetOpTraceIterNum() const { return op_trace_iter_num_; } void ReportProfilingData(uint32_t model_id, const std::vector &task_desc_info, - const std::vector &compute_graph_desc_info, - bool check_device); - void Report(const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, - Msprof::Engine::ReporterData &reporter_data); + const std::vector &compute_graph_desc_info); void ProfilingTaskDescInfo(uint32_t model_id, const std::vector &task_desc_info, const int32_t &device_id); void ProfilingGraphDescInfo(uint32_t model_id, const std::vector &compute_graph_desc_info, const int32_t &device_id); - void SetProfilingConfig(const string &profiling_cfg); - vector GetProfilingDeviceId() const { return device_id_; } - void PluginUnInit(const std::string &module) const; + Status PluginInit() const; + void PluginUnInit() const; + Status CallMsprofReport(ReporterData &reporter_data) const; + struct MsprofCallback &GetMsprofCallback() { return prof_cb_; } + void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; } + void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; } private: - ge::Status ParseFeaturesFromAclCfg(const Json &feature); - ge::Status ProfParseParam(const std::map &config_para, int32_t &device_num, - vector &device_list); - ge::Status ProfParseDeviceId(const std::map &config_para, + Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf); + Status ProfParseParam(const std::map &config_para, int32_t &device_num, + vector &device_list); + Status ProfParseDeviceId(const std::map &config_para, vector &device_list); uint64_t GetProfilingModule(); + void GraphDescReport(const int32_t &device_id, const string &data); void UpdateDeviceIdModuleMap(string prof_type, uint64_t module, const vector &device_list); - bool is_load_profiling_ = false; - bool is_execute_profiling_ = false; - bool is_op_trace_ = false; - bool is_load_ = false; - bool is_training_trace_ = false; - bool is_acl_api_mode_ = false; - int32_t op_trace_iter_num_ = 0; - string job_id_; - string prof_dir_; + void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); + + bool is_load_profiling_; + bool is_execute_profiling_; + bool is_training_trace_; vector device_id_; - vector op_trace_conf_; - vector profiling_opts_; - vector prof_handle_vec_; - string recv_profiling_config_; - string send_profiling_config_; - string system_trace_conf_; - string task_trace_conf_; - const ProfilingEngineImpl engine_; map device_id_module_map_; // key: device_id, value: profiling on module map subs_dev_module_; // key: device_id, value: profiling on module uint32_t subscribe_count_; std::mutex mutex_; + MsprofCallback prof_cb_; }; } // namespace ge #endif // GE_COMMON_PROFILING_PROFILING_MANAGER_H_ diff --git a/ge/executor/CMakeLists.txt b/ge/executor/CMakeLists.txt index d7dfdc84..29215cc1 100644 --- a/ge/executor/CMakeLists.txt +++ b/ge/executor/CMakeLists.txt @@ -17,6 +17,7 @@ set(SRC_LIST "../common/dump/dump_properties.cc" "../common/dump/dump_manager.cc" "../common/dump/dump_op.cc" + "../common/profiling/ge_profiling.cc" "../graph/load/graph_loader.cc" "../graph/execute/graph_execute.cc" "../omm/csa_interact.cc" diff --git a/ge/executor/ge_executor.cc b/ge/executor/ge_executor.cc index 26b0a323..57ab7800 100755 --- a/ge/executor/ge_executor.cc +++ b/ge/executor/ge_executor.cc @@ -304,7 +304,7 @@ Status GeExecutor::Finalize() { // Stop profiling if (ProfilingManager::Instance().ProfilingOn()) { ProfilingManager::Instance().StopProfiling(); - ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); + ProfilingManager::Instance().PluginUnInit(); } GELOGI("Uninit GeExecutor over."); diff --git a/ge/executor/module.mk b/ge/executor/module.mk index 9566ca64..cc64007e 100644 --- a/ge/executor/module.mk +++ b/ge/executor/module.mk @@ -8,6 +8,7 @@ local_ge_executor_src_files := \ ../common/dump/dump_op.cc \ ../common/ge/plugin_manager.cc \ ../common/ge/op_tiling_manager.cc \ + ../common/profiling/ge_profiling.cc \ ../graph/load/graph_loader.cc \ ../graph/execute/graph_execute.cc \ ../omm/csa_interact.cc \ diff --git a/ge/ge_runner.mk b/ge/ge_runner.mk index 5205a06c..5e700f2b 100644 --- a/ge/ge_runner.mk +++ b/ge/ge_runner.mk @@ -29,6 +29,7 @@ LIBGE_LOCAL_SRC_FILES := \ common/dump/dump_manager.cc \ common/dump/dump_properties.cc \ common/dump/dump_op.cc \ + common/profiling/ge_profiling.cc \ engine_manager/dnnengine_manager.cc \ ge_local_engine/engine/host_cpu_engine.cc \ generator/ge_generator.cc \ @@ -307,7 +308,6 @@ LIBGE_LOCAL_SRC_FILES := \ LIBCLIENT_LOCAL_SRC_FILES := \ proto/ge_api.proto \ client/ge_api.cc \ - client/ge_prof.cc \ RUNNER_LOCAL_C_INCLUDES := \ $(LOCAL_PATH) ./ \ @@ -409,7 +409,6 @@ endif LOCAL_C_INCLUDES := $(RUNNER_LOCAL_C_INCLUDES) LOCAL_SRC_FILES := ../../out/ge/lib64/stub/ge_api.cc \ - ../../out/ge/lib64/stub/ge_prof.cc \ ../../out/ge/lib64/stub/ge_ir_build.cc \ LOCAL_SHARED_LIBRARIES := diff --git a/ge/graph/load/new_model_manager/davinci_model.cc b/ge/graph/load/new_model_manager/davinci_model.cc index d0881d95..18435571 100755 --- a/ge/graph/load/new_model_manager/davinci_model.cc +++ b/ge/graph/load/new_model_manager/davinci_model.cc @@ -712,7 +712,7 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size // collect profiling for ge auto &profiling_manager = ProfilingManager::Instance(); if (profiling_manager.ProfilingModelLoadOn()) { - Status p_ret = ReportProfilingData(!profiling_manager.IsAclApiMode()); + Status p_ret = ReportProfilingData(); if (p_ret != SUCCESS) { GELOGE(p_ret, "Report profiling data failed."); return p_ret; @@ -723,14 +723,14 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size return ret; } -Status DavinciModel::ReportProfilingData(bool check_device) { +Status DavinciModel::ReportProfilingData() { std::vector compute_graph_desc_info; Status ret = GetComputeGraphInfo(compute_graph_desc_info); if (ret != SUCCESS) { GELOGE(ret, "GetComputeGraphInfo failed."); return ret; } - ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info, check_device); + ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info); GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); op_list_.clear(); @@ -2250,10 +2250,8 @@ inline int64_t SumSize(const vector &size_list) { Status DavinciModel::SinkModelProfile() { // profiling plugin must be registered - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); - - Msprof::Engine::ReporterData reporter_data{}; + auto &prof_mgr = ProfilingManager::Instance(); + ReporterData reporter_data; // report model data tag name std::string tag_name; tag_name.append("model_load_info_").append(std::to_string(this->Id())); @@ -2271,32 +2269,32 @@ Status DavinciModel::SinkModelProfile() { reporter_data.deviceId = device_id_; reporter_data.data = (unsigned char *)&name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)name.c_str(); reporter_data.dataLen = name.size(); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); uint32_t model_id = this->Id(); reporter_data.data = (unsigned char *)&model_id; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // Load Start/End Time int64_t start_time = this->GetLoadBeginTime(); reporter_data.data = (unsigned char *)&start_time; reporter_data.dataLen = sizeof(int64_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); int64_t end_time = this->GetLoadEndTime(); reporter_data.data = (unsigned char *)&end_time; reporter_data.dataLen = sizeof(int64_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); int32_t task_num = task_list_.size(); std::multimap op_id_map; @@ -2310,6 +2308,7 @@ Status DavinciModel::SinkModelProfile() { uint32_t op_num = fusion_op_info->original_op_names.size(); uint32_t task_id = task->GetTaskID(); if (op_num > 0) { + GELOGI("task.id = %u, opNum = %u", task_id, op_num); op_id_map.insert(std::make_pair(fusion_op_info->op_index, task_id)); } } @@ -2352,39 +2351,39 @@ Status DavinciModel::SinkModelProfile() { int32_t fusion_op_name_len = fusion_op_name.size() == 0 ? 1 : fusion_op_name.size(); reporter_data.data = (unsigned char *)&fusion_op_name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)fusion_op_name.c_str(); reporter_data.dataLen = fusion_op_name_len; - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // original op name before fusion reporter_data.data = (unsigned char *)&op_num; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); for (uint32_t k = 0; k < op_num; k++) { std::string op_name = fusion_op_info->original_op_names[k]; int32_t op_name_len = op_name.size() == 0 ? 1 : op_name.size(); reporter_data.data = (unsigned char *)&op_name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)op_name.c_str(); reporter_data.dataLen = op_name_len; - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); } // stream id info uint32_t streamId = task->GetStreamId(); reporter_data.data = (unsigned char *)&streamId; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // memory info struct memoryInfo memory_info; @@ -2400,22 +2399,22 @@ Status DavinciModel::SinkModelProfile() { memory_info.weight_size + memory_info.input_size + memory_info.output_size + memory_info.workspace_size; reporter_data.data = (unsigned char *)&memory_info; reporter_data.dataLen = sizeof(struct memoryInfo); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // task info reporter_data.data = (unsigned char *)&task_count; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); Range task_range = op_id_map.equal_range(op_id); for (CIT idx = task_range.first; idx != task_range.second; ++idx) { uint32_t task_id = idx->second; reporter_data.data = (unsigned char *)&task_id; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); } } } @@ -2424,10 +2423,8 @@ Status DavinciModel::SinkModelProfile() { Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { // profiling plugin must be registered - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); - - Msprof::Engine::ReporterData reporter_data{}; + auto &prof_mgr = ProfilingManager::Instance(); + ReporterData reporter_data; // report model data tag name std::string tag_name; tag_name.append("model_time_info_") @@ -2450,33 +2447,33 @@ Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { size_t name_len = name.size(); reporter_data.data = (unsigned char *)&name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)name.c_str(); reporter_data.dataLen = name.size(); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // request id uint64_t request_id = current_data.request_id; reporter_data.data = (unsigned char *)&request_id; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); // thread id int32_t thread_id = GetDataInputTid(); reporter_data.data = (unsigned char *)&thread_id; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); // time info time_info_.modelId = this->Id(); reporter_data.data = (unsigned char *)&time_info_; reporter_data.dataLen = sizeof(struct timeInfo); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); return SUCCESS; @@ -2809,71 +2806,32 @@ void *DavinciModel::Run(DavinciModel *model) { } GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_PRE_PROC_END)); GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_START)); - if (ProfilingManager::Instance().ProfilingOpTraceOn()) { - GELOGI("GetOpTraceIterNum:%d", ProfilingManager::Instance().GetOpTraceIterNum()); - for (int32_t i = 0; i < ProfilingManager::Instance().GetOpTraceIterNum(); i++) { - if (!ProfilingManager::Instance().ProfilingLoadFlag()) { - vector prof_device_id_vec = ProfilingManager::Instance().GetProfilingDeviceId(); - for (size_t j = 0; j < prof_device_id_vec.size(); ++j) { - // just profiling, no need to check value - (void)ProfilingManager::Instance().StartProfiling(i, prof_device_id_vec[j]); - } - } - - GELOGI("rtModelExecute start."); - rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; - (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - continue); // [No need to check value] - GELOGI("rtModelExecute end"); - - GELOGI("rtStreamSynchronize start."); - rt_ret = rtStreamSynchronize(model->rt_model_stream_); - if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { - GELOGI("The model with multiple datasets aborts normally."); - } else { - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; - (void)model->ReturnResult(current_data.index, false, seq_end_flag, data_wrapper->GetOutput()); - continue); // [No need to check value] - } - - GELOGI("rtStreamSynchronize end."); - (void)ProfilingManager::Instance().StopProfiling(); // just profiling, no need to check value - } - } else { - GE_TIMESTAMP_START(rtModelExecute); - GELOGI("rtModelExecute start."); - rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; - (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); - continue); - GELOGI("rtModelExecute end"); - GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); - - GE_TIMESTAMP_START(rtStreamSynchronize); - GELOGI("rtStreamSynchronize start."); - rt_ret = rtStreamSynchronize(model->rt_model_stream_); - if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { - seq_end_flag = true; - } - if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { - GELOGI("The model with multiple datasets aborts normally."); - } else { - GE_IF_BOOL_EXEC( - rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); - (void)model->ReturnResult(current_data.index, false, seq_end_flag, - data_wrapper->GetOutput()); // [No need to check value] - CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); - continue); - } - - GELOGI("rtStreamSynchronize end."); - GE_IF_BOOL_EXEC(model->is_first_execute_, - GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); - } - + GE_TIMESTAMP_START(rtModelExecute); + GELOGI("rtModelExecute start."); + rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; + (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); + CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); + continue); + GELOGI("rtModelExecute end"); + GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); + + GE_TIMESTAMP_START(rtStreamSynchronize); + GELOGI("rtStreamSynchronize start."); + rt_ret = rtStreamSynchronize(model->rt_model_stream_); + if (rt_ret == RT_ERROR_END_OF_SEQUENCE) { + seq_end_flag = true; + } + GE_IF_BOOL_EXEC( + rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); + (void)model->ReturnResult(current_data.index, false, seq_end_flag, + data_wrapper->GetOutput()); // [No need to check value] + CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); + continue); + GELOGI("rtStreamSynchronize end."); + GE_IF_BOOL_EXEC(model->is_first_execute_, + GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); + GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_AFTER_PROC_START)); GE_TIMESTAMP_START(ReturnResult3); diff --git a/ge/graph/load/new_model_manager/davinci_model.h b/ge/graph/load/new_model_manager/davinci_model.h index 650f19eb..19888e1f 100755 --- a/ge/graph/load/new_model_manager/davinci_model.h +++ b/ge/graph/load/new_model_manager/davinci_model.h @@ -440,7 +440,7 @@ class DavinciModel { Status SinkTimeProfile(const InputData ¤t_data); - Status ReportProfilingData(bool check_device = true); + Status ReportProfilingData(); void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) { data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id); diff --git a/ge/graph/load/new_model_manager/model_manager.cc b/ge/graph/load/new_model_manager/model_manager.cc index 0b55d150..b595ac39 100755 --- a/ge/graph/load/new_model_manager/model_manager.cc +++ b/ge/graph/load/new_model_manager/model_manager.cc @@ -40,9 +40,7 @@ const int kCmdParSize = 2; const int kDumpCmdPairSize = 2; const std::size_t kProfCmdParaMaxSize = 1000; const std::size_t kProfStartCmdParaSize = 2; -const std::string kCmdTypeProfile = "profile"; const std::string kCmdTypeDump = "dump"; -const std::string kCmdTypeProfiling = "profiling"; const std::string kCmdTypeProfInit = "prof_init"; const std::string kCmdTypeProfFinalize = "prof_finalize"; const std::string kCmdTypeProfStart = "prof_start"; @@ -632,8 +630,7 @@ Status ModelManager::Stop(uint32_t model_id) { /// Status ModelManager::HandleCommand(const Command &command) { static const std::map> cmds = { - {kCmdTypeProfile, HandleProfileCommand}, {kCmdTypeDump, HandleDumpCommand}, - {kCmdTypeProfiling, HandleAclProfilingCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, + {kCmdTypeDump, HandleDumpCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, {kCmdTypeProfFinalize, HandleProfFinalizeCommand}, {kCmdTypeProfStart, HandleProfStartCommand}, {kCmdTypeProfStop, HandleProfStopCommand}, {kCmdTypeProfModelSubscribe, HandleProfModelSubscribeCommand}, @@ -648,21 +645,6 @@ Status ModelManager::HandleCommand(const Command &command) { } } -Status ModelManager::HandleAclProfilingCommand(const Command &command) { - if (command.cmd_params.size() < kCmdParSize) { - GELOGE(PARAM_INVALID, "When the cmd_type is 'profiling', the size of cmd_params must larger than 2."); - return PARAM_INVALID; - } - - std::string map_key = command.cmd_params[0]; - std::string value = command.cmd_params[1]; - if (map_key == PROFILE_CONFIG) { - ProfilingManager::Instance().SetProfilingConfig(value); - } - - return SUCCESS; -} - Status ModelManager::GetModelByCmd(const Command &command, std::shared_ptr &davinci_model) { if (command.cmd_params.size() < kCmdParSize) { @@ -809,29 +791,6 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { return SUCCESS; } -Status ModelManager::HandleProfileCommand(const Command &command) { - if (command.cmd_params.size() < kCmdParSize) { - GELOGE(PARAM_INVALID, "When the cmd_type is 'profile', the size of cmd_params must larger than 2."); - return PARAM_INVALID; - } - - std::string map_key = command.cmd_params[0]; - std::string value = command.cmd_params[1]; - - GELOGI("Profiling mode, Command key:%s , value:%s ", map_key.c_str(), value.c_str()); - - auto iter = PROFILE_COMPONENT_MAP.find(map_key); - if (iter != PROFILE_COMPONENT_MAP.end()) { - std::string property_value = (value == "on") ? "1" : "0"; - PropertiesManager::Instance().SetPropertyValue(iter->second, property_value); - } - - if ((map_key == PROFILER_JOBCTX || map_key == PROFILER_TARGET_PATH || map_key == RTS_PROFILE_PATH)) { - PropertiesManager::Instance().SetPropertyValue(map_key, value); - } - return SUCCESS; -} - static Status ParserPara(const Command &command, const string &dump_key, string &dump_value) { auto iter = std::find(command.cmd_params.begin(), command.cmd_params.end(), dump_key); if (iter != command.cmd_params.end()) { diff --git a/ge/graph/load/new_model_manager/model_manager.h b/ge/graph/load/new_model_manager/model_manager.h index dc3a5219..fc98d9c2 100755 --- a/ge/graph/load/new_model_manager/model_manager.h +++ b/ge/graph/load/new_model_manager/model_manager.h @@ -169,8 +169,6 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { /// @brief comment handle function /// ge::Status HandleCommand(const Command &command); - static ge::Status HandleAclProfilingCommand(const Command &command); - static ge::Status HandleProfileCommand(const Command &command); static ge::Status HandleDumpCommand(const Command &command); static ge::Status HandleProfModelSubscribeCommand(const Command &command); static ge::Status HandleProfModelUnsubscribeCommand(const Command &command); diff --git a/ge/hybrid/executor/worker/execution_engine.cc b/ge/hybrid/executor/worker/execution_engine.cc index e6729352..b984eec3 100755 --- a/ge/hybrid/executor/worker/execution_engine.cc +++ b/ge/hybrid/executor/worker/execution_engine.cc @@ -260,8 +260,7 @@ Status NodeDoneCallback::ProfilingReport() { } auto &profiling_manager = ProfilingManager::Instance(); - profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info, - !profiling_manager.IsAclApiMode()); + profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info); return SUCCESS; } diff --git a/ge/init/gelib.cc b/ge/init/gelib.cc index 306a804a..92700179 100755 --- a/ge/init/gelib.cc +++ b/ge/init/gelib.cc @@ -485,11 +485,9 @@ Status GELib::Finalize() { void GELib::ShutDownProfiling() { std::lock_guard lock(status_mutex_); - if (!ProfilingManager::Instance().ProfilingOpTraceOn() && ProfilingManager::Instance().ProfilingOn()) { - ProfilingManager::Instance().StopProfiling(); - } if (ProfilingManager::Instance().ProfilingOn()) { - ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); + ProfilingManager::Instance().StopProfiling(); + ProfilingManager::Instance().PluginUnInit(); } } diff --git a/ge/single_op/single_op.cc b/ge/single_op/single_op.cc index 6e51b6ff..a2652b67 100755 --- a/ge/single_op/single_op.cc +++ b/ge/single_op/single_op.cc @@ -72,8 +72,7 @@ Status ProfilingTaskInfo(OpTask *op_task) { std::vector compute_graph_info; auto &profiling_manager = ProfilingManager::Instance(); - profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info, - !profiling_manager.IsAclApiMode()); + profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info); return SUCCESS; } } // namespace diff --git a/ge/stub/gen_stubapi.py b/ge/stub/gen_stubapi.py index d19b44a6..1476d505 100644 --- a/ge/stub/gen_stubapi.py +++ b/ge/stub/gen_stubapi.py @@ -71,7 +71,7 @@ max_code_len_per_line = 100 when DEBUG on """ white_list_for_debug = ["attr_value.h", "operator.h", "tensor.h", "graph.h", "operator_factory.h", - "ge_ir_build.h", "ge_api.h", "ge_prof.h", "tensorflow_parser.h", "caffe_parser.h"] + "ge_ir_build.h", "ge_api.h", "tensorflow_parser.h", "caffe_parser.h"] include_dir_key_words = ["ge", "graph", "parser"] DEBUG = True diff --git a/inc/framework/common/profiling/ge_profiling.h b/inc/framework/common/profiling/ge_profiling.h new file mode 100644 index 00000000..65a0eebb --- /dev/null +++ b/inc/framework/common/profiling/ge_profiling.h @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INC_FRAMEWORK_COMMON_GE_PROFILING_H_ +#define INC_FRAMEWORK_COMMON_GE_PROFILING_H_ + +#include "ge/ge_api_error_codes.h" +#include "toolchain/prof_callback.h" + +#define MAX_DEV_NUM (64) +enum ProfCommandHandleType { + kProfCommandhandleInit = 0, + kProfCommandhandleStart, + kProfCommandhandleStop, + kProfCommandhandleFinalize, + kProfCommandhandleModelSubscribe, + kProfCommandhandleModelUnsubscribe +}; + +struct ProfCommandHandleData { + unit64_t profSwitch; + uint32_t devNums; // length of device id list + uint32_t devIdList[MAX_DEV_NUM]; + uint32_t modelId; +} + +ge::Status RegProfCtrlCallback(MsprofCtrlCallback func); +ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func); +ge::Status RegProfReporterCallback(MsprofReporterCallback func); +ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len); + +#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_H_ \ No newline at end of file diff --git a/inc/framework/common/profiling/ge_profiling_cb.h b/inc/framework/common/profiling/ge_profiling_cb.h new file mode 100644 index 00000000..025232ff --- /dev/null +++ b/inc/framework/common/profiling/ge_profiling_cb.h @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ +#define INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ + +#include "profiling/ge_profiling.h" + +bool IsInitialize(); + +#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ \ No newline at end of file