From 07f5327b18593f9308ddf315278b40069fdd3bd9 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 15:52:41 +0800 Subject: [PATCH 01/24] profiling AR version 1 --- ge/CMakeLists.txt | 13 +- ge/client/module.mk | 1 - ge/common/profiling/ge_profiling.cc | 176 ++++++ ge/common/profiling/ge_profiling_cb.cc | 26 + ge/common/profiling/profiling_manager.cc | 513 +++++------------- ge/common/profiling/profiling_manager.h | 97 +--- ge/executor/CMakeLists.txt | 1 + ge/executor/ge_executor.cc | 2 +- ge/executor/module.mk | 1 + ge/ge_runner.mk | 3 +- .../load/new_model_manager/davinci_model.cc | 180 +++--- .../load/new_model_manager/davinci_model.h | 2 +- .../load/new_model_manager/model_manager.cc | 43 +- .../load/new_model_manager/model_manager.h | 2 - ge/hybrid/executor/worker/execution_engine.cc | 3 +- ge/init/gelib.cc | 6 +- ge/single_op/single_op.cc | 3 +- ge/stub/gen_stubapi.py | 2 +- inc/framework/common/profiling/ge_profiling.h | 45 ++ .../common/profiling/ge_profiling_cb.h | 24 + 20 files changed, 510 insertions(+), 633 deletions(-) create mode 100644 ge/common/profiling/ge_profiling.cc create mode 100644 ge/common/profiling/ge_profiling_cb.cc create mode 100644 inc/framework/common/profiling/ge_profiling.h create mode 100644 inc/framework/common/profiling/ge_profiling_cb.h diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index bd0b0c39..7079e432 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -60,6 +60,8 @@ set(TRAIN_SRC_LIST "common/dump/dump_manager.cc" "common/dump/dump_properties.cc" "common/dump/dump_op.cc" + "common/profiling/ge_profiling.cc" + "common/profiling/ge_profiling_cb.cc" "engine_manager/dnnengine_manager.cc" "ge_local_engine/engine/host_cpu_engine.cc" "generator/ge_generator.cc" @@ -332,7 +334,6 @@ set(TRAIN_SRC_LIST "hybrid/hybrid_davinci_model.cc" "executor/ge_executor.cc" "client/ge_api.cc" - "client/ge_prof.cc" "analyzer/analyzer.cc" "ir_build/ge_ir_build.cc" "ir_build/atc_ir_common.cc" @@ -649,7 +650,7 @@ target_link_libraries(ge_runner $ ge_memory adump_server - msprofiler + msprofiler_fwk static_mmpa -Wl,--no-as-needed graph @@ -658,7 +659,6 @@ target_link_libraries(ge_runner register c_sec slog - msprof runtime resource error_manager @@ -777,13 +777,12 @@ target_link_libraries(opensrc_ascendcl PRIVATE register_static error_manager_static adump_server - msprofiler + msprofiler_fwk -Wl,--no-whole-archive -Wl,--no-as-needed c_sec runtime slog - msprof ascend_hal_stub -Wl,--as-needed -lrt @@ -799,12 +798,10 @@ set_target_properties(opensrc_ascendcl PROPERTIES add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc - ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc COMMAND echo "Generating stub files." && ${HI_PYTHON} ${CMAKE_CURRENT_LIST_DIR}/stub/gen_stubapi.py ${GE_CODE_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} && mv ge_ir_build.cc stub_ge_ir_build.cc && mv ge_api.cc stub_ge_api.cc - && mv ge_prof.cc stub_ge_prof.cc && echo "Generating stub files end." #WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} #DEPENDS stub/gen_stubapi.py ${TOP_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} @@ -813,7 +810,6 @@ add_custom_command( add_custom_target(ge_stub DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc - ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc ) ################################################################## @@ -855,7 +851,6 @@ target_include_directories(atc_stub_ge_compiler PRIVATE ############ stub/libge_runner.so ############ add_library(fwk_stub_ge_runner SHARED stub_ge_api.cc - stub_ge_prof.cc stub_ge_ir_build.cc ) diff --git a/ge/client/module.mk b/ge/client/module.mk index 6ac69d31..c942d550 100644 --- a/ge/client/module.mk +++ b/ge/client/module.mk @@ -4,7 +4,6 @@ LOCAL_PATH := $(call my-dir) COMMON_LOCAL_SRC_FILES := \ proto/ge_api.proto \ ge_api.cc \ - ge_prof.cc \ COMMON_LOCAL_C_INCLUDES := \ diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc new file mode 100644 index 00000000..8da6b12a --- /dev/null +++ b/ge/common/profiling/ge_profiling.cc @@ -0,0 +1,176 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/profiling/ge_profiling.h" +#include "runtime/base.h" +#include "common/profiling/profiling_manager.h" +#include "framework/common/debug/ge_log.h" +#include "framework/common/debug/log.h" +#include "graph/load/graph_loader.h" +#include "init/gelib.h" +#include "framework/common/ge_inner_error_codes.h" + +namespace { +const uint32_t kDeviceListIndex = 3; +const std::string kDeviceNums = "devNums"; +const std::string kDeviceIdList = "devIdList"; +const std::string kProfilingInit = "prof_init"; +const std::string kProfilingFinalize = "prof_finalize"; +const std::string kProfilingStart = "prof_start"; +const std::string kProfilingStop = "prof_stop"; +const std::string kProfModelSubscribe = "prof_model_subscribe"; +const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; +const std::string kRtSetDeviceRegName = "profiling"; + +const std::map kProfCommandTypeMap = { + {kProfCommandhandleInit, kProfilingInit}, + {kProfCommandhandleStart, kProfilingStart}, + {kProfCommandhandleStop, kProfilingStop}, + {kProfCommandhandleFinalize, kProfilingFinalize}, + {kProfCommandhandleModelSubscribe, kProfModelSubscribe}, + {kProfCommandhandleModelUnsubscribe, kProfModelUnsubscribe}}; +} // namespace + +bool TransProfConfigToParam(const MsprofCommandHandle &profCommand, vector &prof_config_params) { + prof_config_params.clear(); + prof_config_params.emplace_back(kDeviceNums); + prof_config_params.emplace_back(std::to_string(profCommand.devNums)); + prof_config_params.emplace_back(kDeviceIdList); + std::string devID = ""; + if (profCommand.devNums == 0) { + GELOGW("The device num is invalid."); + return false; + } + for (uint32_t i = 0; i < profCommand.devNums; i++) { + devID.append(std::to_string(profCommand.devIdList[i])); + if (i != profCommand.devNums - 1) { + devID.append(","); + } + } + + prof_config_params.push_back(devID); + return true; +} + +bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { + if (deviceid_list == nullptr) { + GELOGE(ge::PARAM_INVALID, "deviceIdList is nullptr"); + return false; + } + if (device_nums == 0 || device_nums > MAX_DEV_NUM) { + GELOGE(ge::PARAM_INVALID, "The device nums is invalid."); + return false; + } + + // real device num + int32_t dev_count = 0; + rtError_t rt_err = rtGetDeviceCount(&dev_count); + if (rt_err != RT_ERROR_NONE) { + GELOGE(ge::INTERNAL_ERROR, "Get the Device count fail."); + return false; + } + + if (device_nums > static_cast(dev_count)) { + GELOGE(ge::PARAM_INVALID, "Device num(%u) is not in range 1 ~ %d.", device_nums, dev_count); + return false; + } + + std::unordered_set record; + for (size_t i = 0; i < device_nums; ++i) { + uint32_t dev_id = deviceid_list[i]; + if (dev_id >= static_cast(dev_count)) { + GELOGE(ge::PARAM_INVALID, "Device id %u is not in range 0 ~ %d(exclude %d)", dev_id, dev_count, dev_count); + return false; + } + if (record.count(dev_id) > 0) { + GELOGE(ge::PARAM_INVALID, "Device id %u is duplicatedly set", dev_id); + return false; + } + record.insert(dev_id); + } + return true; +} + +ge::Status RegProfCtrlCallback(MsprofCtrlCallback func) { + if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { + GELOGW("Msprof ctrl callback is exist, just ignore it."); + } else { + ge::ProfilingManager::Instance().SetMsprofCtrlCallback(func); + } + return ge::SUCCESS; +} + +ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { + // Pass MsprofSetDeviceCallback to runtime + ge::Status rt_ret = rtRegDeviceStateCallback(kRtSetDeviceRegName.c_str(), static_cast(func)); + if (rt_ret != ge::SUCCESS) { + GELOGE(rt_ret, "Pass MsprofSetDeviceCallback to runtime failed!"); + return rt_ret; + } + return ge::SUCCESS; +} + +ge::Status RegProfReporterCallback(MsprofReporterCallback func) { + if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { + GELOGW("Msprof ctrl callback is exist, just ignore it."); + } else { + ge::ProfilingManager::Instance().SetMsprofReporterCallback(func); + } + // Pass MsprofReporterCallback to runtime + ge::Status rt_ret = rtSetMsprofReporterCallback(func); + if (rt_ret != ge::SUCCESS) { + GELOGE(rt_ret, "Pass MsprofReporterCallback to runtime failed!!"); + return rt_ret; + } + // Pass MsprofReporterCallback to hccl in opskernel so initialize + + return ge::SUCCESS; +} + +ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len) { + GE_CHECK_NOTNULL(data); + MsprofCommandHandle *prof_config_param = (MsprofCommandHandle *)data; + if (!isProfConfigValid(prof_config_param->devIdList, prof_config_param->devNums)) { + return ge::FAILED; + } + std::vector prof_params; + if (!TransProfConfigToParam(*prof_config_param, prof_params)) { + GELOGE(ge::PARAM_INVALID, "Transfer profilerConfig to string vector failed"); + return ge::PARAM_INVALID; + } + auto iter = kProfCommandTypeMap.find(type); + if (iter == kProfCommandTypeMap.end()) { + GELOGW("The prof comand type is invalid."); + return ge::PARAM_INVALID; + } + ge::GraphLoader graph_loader; + ge::Command command; + command.cmd_params.clear(); + command.cmd_type = iter->second; + command.cmd_params = prof_params; + command.module_index = prof_config_param->profSwitch; + GELOGI("GE commandhandle execute, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), + prof_params[kDeviceListIndex].c_str(), command.module_index); + ge::Status ret = graph_loader.CommandHandle(command); + if (ret != ge::SUCCESS) { + GELOGE(ret, "Handle profiling command failed"); + return ge::FAILED; + } + + GELOGI("Successfully execute profiling command 0x%llx.", command.module_index); + return ge::SUCCESS; +} + diff --git a/ge/common/profiling/ge_profiling_cb.cc b/ge/common/profiling/ge_profiling_cb.cc new file mode 100644 index 00000000..230e129d --- /dev/null +++ b/ge/common/profiling/ge_profiling_cb.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/profiling/ge_profiling_cb.h" +#include "init/gelib.h" + +bool IsInitialize() { + std::shared_ptr instance_ptr = ge::GELib::GetInstance(); + if (instance_ptr == nullptr || instance_ptr->InitFlag() == false) { + return false; + } + return true; +} diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index ba03d671..bcf6d366 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -45,7 +45,10 @@ const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; } // namespace namespace ge { -ProfilingManager::ProfilingManager() : subscribe_count_(0) {} +ProfilingManager::ProfilingManager() : is_load_profiling_(false), + is_execute_profiling_(false), + is_training_trace_(false), + subscribe_count_(0) {} ProfilingManager::~ProfilingManager() {} @@ -58,44 +61,25 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In #ifdef DAVINCI_SUPPORT_PROFILING vector().swap(device_id_); subscribe_count_ = 0; - job_id_ = options.job_id; + GELOGI("ProfilingManager::Init job_id:%s", options.job_id.c_str()); - GELOGI("ProfilingManager::Init job_id:%s", job_id_.c_str()); - - - - Status ret; - if (!recv_profiling_config_.empty()) { - GELOGI("Profiling json config from acl:%s", recv_profiling_config_.c_str()); - ret = InitFromAclCfg(recv_profiling_config_); - } else { - ret = InitFromOptions(options); - if (ret == SUCCESS && is_load_profiling_) { - device_id_.push_back(options.device_id); - } - } + struct MsprofGeOptions prof_conf = { 0 }; + Status ret = InitFromOptions(options, prof_conf); if (ret != SUCCESS) { GELOGE(ret, "Failed to init profiling."); return ret; } if (is_load_profiling_) { - // register Framework to profiling - int result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); - if (result != 0) { - GELOGE(FAILED, "Register profiling engine failed."); + int32_t cb_ret = prof_cb_.msprofCtrlCallback( + static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), + static_cast(&prof_conf), sizeof(MsprofGeOptions)); + if (cb_ret != 0) { + GELOGE(FAILED, "call msprofCtrlCallback failed, type:%u, return:%d", + static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), cb_ret); return FAILED; } - // profiling startup first time - GELOGI("Begin to init profiling, device num %zu", device_id_.size()); - for (size_t i = 0; i < device_id_.size(); ++i) { - ret = StartProfiling(0, device_id_[i]); - if (ret != SUCCESS) { - GELOGW("Profiling start failed on device %d.", device_id_[i]); - continue; - } - GELOGI("Profiling init succ on device %d.", device_id_[i]); - } + GELOGI("Profiling init success"); } else { GELOGI("The profiling is off, skip the initialization"); } @@ -103,264 +87,55 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In return SUCCESS; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromAclCfg( - const std::string &config) { +ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOptions &prof_conf) { #ifdef DAVINCI_SUPPORT_PROFILING - try { - is_load_profiling_ = false; - is_execute_profiling_ = false; - profiling_opts_.clear(); - op_trace_conf_.clear(); - Json start_prof_conf = Json::parse(config); - Json &prof_conf = start_prof_conf[kStartCfg][0]; - job_id_ = prof_conf[kJobID]; - auto iter = prof_conf.find(kProfDir); - if (iter != prof_conf.end()) { - prof_dir_ = prof_conf[kProfDir]; - } - Json &device_id = prof_conf[kDeviceID]; - if (device_id.size() != 0) { - vector().swap(device_id_); - bool is_all = false; - for (size_t i = 0; i < device_id.size(); i++) { - std::string device_id_str = device_id[i].get(); - if (device_id_str == "all") { - is_all = true; - break; - } - device_id_.push_back(std::stoi(device_id_str)); - } - if (is_all) { - int32_t count = 0; - rtError_t rt_err = rtGetDeviceCount(&count); - if (rt_err != RT_ERROR_NONE) { - GELOGE(FAILED, "Call rtGetDeviceCount to get device failed."); - } + // enable profiling by env + char env_profiling_mode[MMPA_MAX_PATH] = { 0x00 }; + is_load_profiling_ = false; + is_execute_profiling_ = false; - vector().swap(device_id_); - for (int32_t i = 0; i < count; ++i) { - device_id_.push_back(i); - } - } - } + (void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); + (void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, sizeof(MsprofGeOptions)); - Json &features = prof_conf[kFeatures]; - if (ParseFeaturesFromAclCfg(features) != SUCCESS) { - GELOGE(FAILED, "Parse feature from acl cfg failed."); - return FAILED; - } + if ((env_profiling_mode != nullptr) && (strcmp("true", env_profiling_mode) == 0) + && (strcmp(prof_conf.options, "\0") != 0)) { + // enable profiling by env is_load_profiling_ = true; is_execute_profiling_ = true; - } catch (...) { - GELOGE(FAILED, "Json conf is not invalid !"); - return ge::PARAM_INVALID; - } -#endif - return ge::SUCCESS; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::ParseFeaturesFromAclCfg( - const Json &features) { -#ifdef DAVINCI_SUPPORT_PROFILING - try { - for (size_t i = 0; i < features.size(); ++i) { - const Json &feature = features[i]; - if ((feature.find(kName) == feature.end()) || feature[kName].is_null()) { - continue; - } - const std::string &name = feature[kName]; - if (name == "op_trace") { - const Json &conf = feature[kConf]; - const Json &events = conf[0][kEvents]; - const std::string &ai_core_events = events[0][kAiCoreEvents]; - GELOGI("Op trace config from acl ai_core_events:%s", ai_core_events.c_str()); - is_op_trace_ = true; - ProfMgrConf prof_mgr_conf; - int result = ProfMgrGetConf(ai_core_events, &prof_mgr_conf); - if (result != 0) { - GELOGE(FAILED, "ProfMgrGetConf failed."); - return FAILED; - } - op_trace_conf_ = prof_mgr_conf.conf; - op_trace_iter_num_ = static_cast(op_trace_conf_.size()); - GELOGI("Op trace profiling iter num %d,", op_trace_iter_num_); - } else if (name == "task_trace") { - is_op_trace_ = false; - if (feature.find(kConf) != feature.end()) { - const Json &conf = feature[kConf]; - std::stringstream task_trace_conf; - task_trace_conf << conf; - task_trace_conf_ = task_trace_conf.str(); - } - GELOGI("Task trace config from acl"); - } else if (name == "system_trace") { - is_op_trace_ = false; - const Json &conf = feature[kConf]; - std::stringstream system_trace_conf; - system_trace_conf << conf; - system_trace_conf_ = system_trace_conf.str(); - GELOGI("System trace config from acl"); - } - profiling_opts_.push_back(name); - } - } catch (...) { - GELOGE(ge::PARAM_INVALID, "Json conf feature is not invalid !"); - return ge::PARAM_INVALID; - } -#endif - return ge::SUCCESS; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromOptions(const Options &options) { -#ifdef DAVINCI_SUPPORT_PROFILING - // enable profiling support two ways: env and front end - char profiling_mode_temp[MMPA_MAX_PATH] = { 0x00 }; - char prof_options_temp[MMPA_MAX_PATH] = { 0x00 }; - (void)mmGetEnv("PROFILING_MODE", profiling_mode_temp, MMPA_MAX_PATH); - (void)mmGetEnv("PROFILING_OPTIONS", prof_options_temp, MMPA_MAX_PATH); - const char *profiling_mode = profiling_mode_temp; - const char *prof_options = prof_options_temp; - if ((profiling_mode == nullptr) || (strcmp("true", profiling_mode) != 0) || (prof_options == nullptr)) { - is_load_profiling_ = false; - is_execute_profiling_ = false; + GELOGI("The profiling in env is %s, %s", env_profiling_mode, prof_conf.options); } else { - std::string prof_options_str = std::string(prof_options); - profiling_opts_ = StringUtils::Split(prof_options_str, ':'); - is_load_profiling_ = true; - is_execute_profiling_ = true; - GELOGI("The profiling in env is %s, %s", profiling_mode, prof_options); - } - if (!is_load_profiling_) { - const std::string enable_profiling = "1"; - if (options.profiling_mode != enable_profiling || options.profiling_options.empty()) { - is_load_profiling_ = false; - is_execute_profiling_ = false; + if (options.profiling_mode != "1" || options.profiling_options.empty()) { return SUCCESS; - } else { - profiling_opts_ = StringUtils::Split(options.profiling_options, ':'); - is_load_profiling_ = true; - is_execute_profiling_ = true; - GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), options.profiling_options.c_str()); } - } - // features:'training_trace', 'task_trace' or 'op_trace' etc - if (!profiling_opts_.empty()) { - if (profiling_opts_[0] == "op_trace") { - is_op_trace_ = true; - // op trace get conf - ProfMgrConf prof_mgr_conf; - int result = ProfMgrGetConf("", &prof_mgr_conf); - if (result != 0) { - GELOGE(FAILED, "ProfMgrGetConf failed."); - return FAILED; - } - op_trace_conf_ = prof_mgr_conf.conf; - op_trace_iter_num_ = static_cast(op_trace_conf_.size()); - GELOGI("op trace profiling iter num %d,", op_trace_iter_num_); - } else { - is_op_trace_ = false; - op_trace_iter_num_ = 1; + // enable profiling by ge option + if (memcpy_s(prof_conf.options, sizeof(prof_conf.options), options.profiling_options.c_str(), + sizeof(options.profiling_options.c_str())) != EOK) { + GELOGE(INTERNAL_ERROR, "copy profiling_options failed."); + return INTERNAL_ERROR; } + is_load_profiling_ = true; + is_execute_profiling_ = true; + GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), prof_conf.options); } -#endif - return ge::SUCCESS; -} -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::StartProfiling(int32_t iter_num, - int32_t device_id) { -#ifdef DAVINCI_SUPPORT_PROFILING - if (!profiling_opts_.empty()) { - GELOGI("Start profiling index is %d", iter_num); - // current one docker only use one device - Json p_device; - - try { - // profiling need physical_device_id - p_device[kDeviceID] = std::to_string(device_id); - p_device[kJobID] = job_id_; - p_device[kTraceID] = std::to_string(GetContext().TraceId()); - if (!prof_dir_.empty()) { - p_device[kProfDir] = prof_dir_; - GELOGI("Prof dir: %s.", prof_dir_.c_str()); - } - - Json features; - if (is_op_trace_) { - Json f; - f[kName] = "op_trace"; - Json conf; - if (op_trace_conf_.size() <= static_cast(iter_num)) { - GELOGE(FAILED, "Op trace iter num is invalid!"); - return FAILED; - } - Json events; - events[0] = nlohmann::json::parse(op_trace_conf_[iter_num]); - conf[0][kEvents] = events; - f[kConf] = conf; - features[0] = f; - if (iter_num == 0) { - is_load_ = true; - } - } else { - for (std::vector::size_type i = 0; i < profiling_opts_.size(); i++) { - Json f; - if (profiling_opts_[i] == "system_trace") { - f[kConf] = nlohmann::json::parse(system_trace_conf_); - } else if (profiling_opts_[i] == "task_trace") { - if (!task_trace_conf_.empty()) { - f[kConf] = nlohmann::json::parse(task_trace_conf_); - } - } - f[kName] = profiling_opts_[i]; - features[i] = f; - } - is_load_ = true; - } - p_device[kFeatures] = features; - // only one device, but sProfMgrStartUp API require for device list - Json devices; - devices[0] = p_device; - - Json start_cfg; - start_cfg[kStartCfg] = devices; - - // convert json to string - std::stringstream ss; - ss << start_cfg; - send_profiling_config_ = ss.str(); - GELOGI("Profiling config %s\n", send_profiling_config_.c_str()); - } catch (...) { - GELOGE(FAILED, "Op trace json conf is not invalid !"); - return FAILED; - } + if (!is_load_profiling_) { + return SUCCESS; + } - // runtime startup for profiling - uint64_t module = GetProfilingModule(); - int32_t device_num = 1; - uint32_t device_id_rt = static_cast(device_id); - GE_CHK_RT_RET(rtProfilerStart(module, device_num, &device_id_rt)); - - // call profiling startup API - ProfMgrCfg prof_cfg = {send_profiling_config_}; - void *prof_handle = ProfMgrStartUp(&prof_cfg); - if (prof_handle == nullptr) { - GELOGW("ProfMgrStartUp failed on device %d ", device_id); - return FAILED; - } - GELOGD("StartProfiling, prof_handle: %p", prof_handle); - prof_handle_vec_.push_back(prof_handle); + if (memcpy_s(prof_conf.jobId, sizeof(prof_conf.jobId), options.job_id.c_str(), + sizeof(options.job_id.c_str())) != EOK) { + GELOGE(INTERNAL_ERROR, "copy job_id failed."); + return INTERNAL_ERROR; } + + // get traceId from options + prof_conf.traceId = GetContext().TraceId(); #endif - return SUCCESS; + return ge::SUCCESS; } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProfiling() { #ifdef DAVINCI_SUPPORT_PROFILING - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - if (reporter != nullptr) { - int ret = reporter->Flush(); - GELOGI("Report data end, ret is %d", ret); - } uint64_t module = GetProfilingModule(); int32_t device_num = static_cast(device_id_.size()); auto device_id_ptr = std::unique_ptr(new (std::nothrow) uint32_t[device_num]); @@ -376,15 +151,14 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); } - for (size_t i = 0; i < prof_handle_vec_.size(); ++i) { - int result = ProfMgrStop(prof_handle_vec_[i]); - if (result != 0) { - GELOGW("ProfMgr stop return fail:%d, handle:%p", result, prof_handle_vec_[i]); - } + // stop profiling + int32_t cb_ret = prof_cb_.msprofCtrlCallback(static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), + nullptr, 0); + if (cb_ret != 0) { + GELOGW("call msprofCtrlCallback failed, type:%u, return:%d", + static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), cb_ret); + return; } - vector().swap(prof_handle_vec_); - is_load_ = false; - recv_profiling_config_ = ""; GELOGI("Stop Profiling success."); #endif } @@ -392,12 +166,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingTaskDescInfo( uint32_t model_id, const std::vector &task_desc_info, const int32_t &device_id) { #ifdef DAVINCI_SUPPORT_PROFILING - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - if (reporter == nullptr) { - GELOGI("Profiling report is nullptr!"); - return; - } - std::string data; for (const auto &task : task_desc_info) { std::string model_name = task.model_name; @@ -412,7 +180,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin .append(std::to_string(stream_id)).append(" ") .append(std::to_string(model_id)).append("\n")); - Msprof::Engine::ReporterData reporter_data{}; + ReporterData reporter_data; reporter_data.deviceId = device_id; reporter_data.data = (unsigned char *)data.c_str(); reporter_data.dataLen = data.size(); @@ -422,9 +190,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin return; } - ret = reporter->Report(&reporter_data); - if (ret != SUCCESS) { - GELOGE(ret, "Reporter data of task_desc_info fail!"); + int32_t cb_ret = CallMsprofReport(reporter_data); + if (cb_ret != 0) { + GELOGE(cb_ret, "Reporter data of task_desc_info failed, ret:%d", cb_ret); return; } } @@ -436,9 +204,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingGraphDescInfo( uint32_t model_id, const std::vector &compute_graph_desc_info, const int32_t &device_id) { #ifdef DAVINCI_SUPPORT_PROFILING - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return;); - std::string data; for (const auto &graph : compute_graph_desc_info) { data.append("model_name:") @@ -493,64 +258,52 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin } data.append(" model_id:").append(std::to_string(model_id)); - data.append("\n"); - Msprof::Engine::ReporterData reporter_data{}; - Report(device_id, data, *reporter, reporter_data); - + GraphDescReport(device_id, data); data.clear(); } #endif } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Report( - const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, - Msprof::Engine::ReporterData &reporter_data) { +void ProfilingManager::GraphDescReport(const int32_t &device_id, const string &data) { #ifdef DAVINCI_SUPPORT_PROFILING + ReporterData reporter_data; + int ret = -1; + int32_t cb_ret = -1; size_t index = data.size() / kReportMaxLen; if (index >= 1) { reporter_data.deviceId = device_id; - int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); + ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); for (size_t i = 0; i < index; ++i) { reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * i; reporter_data.dataLen = kReportMaxLen; - ret = reporter.Report(&reporter_data); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); + cb_ret = CallMsprofReport(reporter_data); + GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); } reporter_data.dataLen = data.size() - kReportMaxLen * index; if (reporter_data.dataLen != 0) { reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * index; - ret = reporter.Report(&reporter_data); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); + cb_ret = CallMsprofReport(reporter_data); + GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); } } else { reporter_data.deviceId = device_id; reporter_data.data = (unsigned char *)data.c_str(); reporter_data.dataLen = data.size(); - int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); + ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); - ret = reporter.Report(&reporter_data); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); - } -#endif -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit(const std::string &module) const { -#ifdef DAVINCI_SUPPORT_PROFILING - int ret = Msprof::Engine::UnInit(module); - if (ret != SUCCESS) { - GELOGE(ret, "profiling plugin uninit failed, ret:%d", ret); + cb_ret = CallMsprofReport(reporter_data); + GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); } #endif } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportProfilingData( uint32_t model_id, const std::vector &task_desc_info, - const std::vector &compute_graph_desc_info, - bool check_device) { + const std::vector &compute_graph_desc_info) { #ifdef DAVINCI_SUPPORT_PROFILING int32_t logic_device_id = 0; rtError_t rt_ret = rtGetDevice(&logic_device_id); @@ -559,13 +312,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr return; } GELOGD("current logic_device_id:%d", logic_device_id); - if (check_device) { - auto ret = std::find(device_id_.begin(), device_id_.end(), logic_device_id); - if (ret == device_id_.end()) { - GELOGE(FAILED, "get valid phy_device_id failed, profiling report failed."); - return; - } - } GELOGD("start ProfilingTaskDescInfo."); ProfilingTaskDescInfo(model_id, task_desc_info, logic_device_id); GELOGD("start ProfilingGraphDescInfo."); @@ -574,11 +320,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr #endif } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::SetProfilingConfig( - const std::string &profiling_cfg) { - recv_profiling_config_ = profiling_cfg; -} - FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetProfilingModule() { uint64_t module = PROF_MODEL_EXECUTE_MASK | PROF_RUNTIME_API_MASK | @@ -594,9 +335,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetP return module; } -void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, - uint32_t device_id, - uint64_t module) { +void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module) { #ifdef DAVINCI_SUPPORT_PROFILING if (prof_type == kProfModelSubscribe) { if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { @@ -608,9 +347,13 @@ void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, subs_dev_module_[device_id] = dev_info; } } else if (prof_type == kProfModelUnsubscribe) { - if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { - if (subs_dev_module_[device_id].subscribe_count > 0) { - subs_dev_module_[device_id].subscribe_count--; + auto iter = subs_dev_module_.find(device_id); + if (iter != subs_dev_module_.end()) { + if (iter->second.subscribe_count > 0) { + iter->second.subscribe_count--; + } + if (iter->second.subscribe_count == 0) { + subs_dev_module_.erase(iter); } } } else { @@ -626,10 +369,11 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo uint64_t model_load_mask = module & PROF_MODEL_LOAD_MASK; if ((subscribe_count_ == 0) && (model_load_mask == PROF_MODEL_LOAD_MASK)) { // register framework to profiling - int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); - if (result != SUCCESS) { - GELOGE(FAILED, "Register profiling engine failed."); - return FAILED; + // register Framework to profiling + int32_t cb_ret = PluginInit(); + if (cb_ret != 0) { + GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); + return cb_ret; } GELOGI("Prof subscribe: model load profiling on."); } @@ -647,7 +391,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo UpdateSubscribeDeviceModuleMap(kProfModelSubscribe, device[0], module); // Report profiling data - Status p_ret = davinci_model->ReportProfilingData(false); + Status p_ret = davinci_model->ReportProfilingData(); if (p_ret != SUCCESS) { GELOGE(p_ret, "Report profiling data failed."); return p_ret; @@ -672,6 +416,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo auto iter = subs_dev_module_.find(device[0]); if (iter != subs_dev_module_.end()) { if (subs_dev_module_[device[0]].subscribe_count == 1) { + // The same device_id, only stop at last time rtError_t rt_ret = rtProfilerStop(subs_dev_module_[device[0]].module, dev_num, device); if (rt_ret != RT_ERROR_NONE) { GELOGE(FAILED, "Runtime profiler stop failed."); @@ -679,15 +424,15 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo } } UpdateSubscribeDeviceModuleMap(kProfModelUnsubscribe, device[0], subs_dev_module_[device[0]].module); + } else { + GELOGE(FAILED, "The device_id:%u has not been subscribed, do not need to cancel.", device[0]); + return FAILED; } subscribe_count_--; if (subscribe_count_ == 0) { - int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); - if (ret != SUCCESS) { - GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); - return ret; - } + // profiling plugin uninit at last subscription + PluginUnInit(); } #endif return SUCCESS; @@ -700,11 +445,12 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn if (model_load_mask == PROF_MODEL_LOAD_MASK) { // register Framework to profiling - int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); - if (result != SUCCESS) { - GELOGE(FAILED, "Register profiling engine failed."); - return FAILED; + int32_t cb_ret = PluginInit(); + if (cb_ret != 0) { + GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); + return cb_ret; } + int32_t device_num = -1; rtError_t rt_ret = rtProfilerStart(model_load_mask, device_num, nullptr); if (rt_ret != RT_ERROR_NONE) { @@ -719,7 +465,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn if (training_trace_mask == PROF_TRAINING_TRACE_MASK) { is_training_trace_ = true; } - is_acl_api_mode_ = true; GELOGI("Prof init success."); #endif return SUCCESS; @@ -730,12 +475,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi std::lock_guard lock(mutex_); is_load_profiling_ = false; is_training_trace_ = false; - is_acl_api_mode_ = false; - int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); - if (ret != SUCCESS) { - GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); - } + // profiling plugin uninit + PluginUnInit(); + int32_t dev_num = -1; rtError_t rt_ret = rtProfilerStop(PROF_MODEL_LOAD_MASK, dev_num, nullptr); if (rt_ret != RT_ERROR_NONE) { @@ -859,7 +602,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt for (int32_t i = 0; i < device_num; i++) { device_id_ptr[i] = static_cast(device_list[i]); } - GELOGD("Runtime config param: 0x%llx, device num: %d.", module, device_num); + GELOGI("Runtime config param: 0x%llx, device num: %d.", module, device_num); rtError_t rt_ret = rtProfilerStart(module, device_num, device_id_ptr.get()); if (rt_ret != RT_ERROR_NONE) { @@ -878,7 +621,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGW("Prof start: load model module is invalid."); } UpdateDeviceIdModuleMap(kProfStart, module, device_list); - GELOGD("Prof start profiling success."); + GELOGI("Prof start profiling success."); #endif return SUCCESS; } @@ -901,7 +644,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt for (int32_t i = 0; i < device_num; i++) { device_id_ptr[i] = static_cast(device_list[i]); } - GELOGD("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); + GELOGI("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); if (rt_ret != RT_ERROR_NONE) { GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); @@ -921,7 +664,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGW("Prof stop: load model module is invalid."); } UpdateDeviceIdModuleMap(kProfStop, module, device_list); - GELOGD("Prof stop profiling success."); + GELOGI("Prof stop profiling success."); #endif return SUCCESS; } @@ -963,47 +706,43 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ProfilingManager::Profilin if (rt_ret != RT_ERROR_NONE) { GELOGE(rt_ret, "Runtime get logic_device_id failed, current logic_device_id:%d", logic_device_id); } - GELOGD("Current logic_device_id:%d", logic_device_id); + GELOGI("Current logic_device_id:%d", logic_device_id); bool execute_model_prof_on = false; auto iter = std::find(device_id_.begin(), device_id_.end(), logic_device_id); if (iter != device_id_.end()) { execute_model_prof_on = true; } - GELOGD("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); + GELOGI("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); return is_execute_profiling_ || execute_model_prof_on; } -/** - * @brief Profiling PluginImpl - */ -// PluginImpl static variable init -Msprof::Engine::Reporter *PluginImpl::reporter_ = nullptr; - -PluginImpl::PluginImpl(const std::string &module) : module_(module) { GELOGI("Create PluginImpl\n"); } - -int PluginImpl::Init(const Msprof::Engine::Reporter *reporter) { - GELOGI("PluginImpl init"); - reporter_ = const_cast(reporter); - return 0; +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() const { + return prof_cb_.msprofReporterCallback( + static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), + static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_INIT), + nullptr, 0); } -int PluginImpl::UnInit() { - GELOGI("PluginImpl Uninit"); - reporter_ = nullptr; - return 0; +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit() const { +#ifdef DAVINCI_SUPPORT_PROFILING + int32_t cb_ret = prof_cb_.msprofReporterCallback( + static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), + static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_UNINIT), + nullptr, 0); + if (cb_ret != 0) { + GELOGW("profiling plugin uninit failed, ret:%d", cb_ret); + } +#endif } -Msprof::Engine::PluginIntf *ProfilingEngineImpl::CreatePlugin() { - GELOGI(" Create Plugin"); - return new (std::nothrow) PluginImpl(GE_PROFILING_MODULE); +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::CallMsprofReport( + ReporterData &reporter_data) const { + return prof_cb_.msprofReporterCallback( + static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), + static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_REPORT), + static_cast(&reporter_data), sizeof(ReporterData)); } -int ProfilingEngineImpl::ReleasePlugin(Msprof::Engine::PluginIntf *plugin) { - if (plugin != nullptr) { - delete plugin; - plugin = nullptr; - } - return 0; -} + } // namespace ge diff --git a/ge/common/profiling/profiling_manager.h b/ge/common/profiling/profiling_manager.h index 66cefc32..c9434a10 100755 --- a/ge/common/profiling/profiling_manager.h +++ b/ge/common/profiling/profiling_manager.h @@ -26,9 +26,7 @@ #include "framework/common/ge_inner_error_codes.h" #include "framework/common/ge_types.h" #include "external/register/register_types.h" -#include "toolchain/prof_engine.h" -#include "toolchain/prof_mgr_core.h" -#include "toolchain/prof_acl_api.h" +#include "toolchain/prof_callback.h" using std::map; using std::string; @@ -43,29 +41,10 @@ struct DeviceSubsInfo { uint64_t module; uint32_t subscribe_count; }; -// register Plugin -class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY PluginImpl : public Msprof::Engine::PluginIntf { - public: - explicit PluginImpl(const std::string &module); - ~PluginImpl() {} - - int Init(const Msprof::Engine::Reporter *reporter); - int UnInit(); - static Msprof::Engine::Reporter *GetPluginReporter() { return reporter_; } - - private: - static Msprof::Engine::Reporter *reporter_; - std::string module_; -}; -// register Engine -class ProfilingEngineImpl : public Msprof::Engine::EngineIntf { - public: - ProfilingEngineImpl() {} - ~ProfilingEngineImpl() {} - - Msprof::Engine::PluginIntf *CreatePlugin(); - int ReleasePlugin(Msprof::Engine::PluginIntf *plugin); +struct MsprofCallback { + MsprofCtrlCallback msprofCtrlCallback; + MsprofReporterCallback msprofReporterCallback; }; class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { @@ -73,68 +52,50 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { ProfilingManager(); virtual ~ProfilingManager(); static ProfilingManager &Instance(); - ge::Status Init(const Options &options); - ge::Status InitFromOptions(const Options &options); - ge::Status InitFromAclCfg(const std::string &config); - ge::Status StartProfiling(int32_t iter, int32_t device_id); - void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); - ge::Status ProfModelSubscribe(uint64_t module, void *model); - ge::Status ProfModelUnsubscribe(void *model); - ge::Status ProfInit(uint64_t module); - ge::Status ProfFinalize(); - ge::Status ProfStartProfiling(uint64_t module, const std::map &config_para); - ge::Status ProfStopProfiling(uint64_t module, const std::map &config_para); + Status Init(const Options &options); + Status ProfInit(uint64_t module); + Status ProfFinalize(); + Status ProfStartProfiling(uint64_t module, const std::map &config_para); + Status ProfStopProfiling(uint64_t module, const std::map &config_para); + Status ProfModelSubscribe(uint64_t module, void *model); + Status ProfModelUnsubscribe(void *model); void StopProfiling(); - bool ProfilingOpTraceOn() const { return is_op_trace_; } - bool ProfilingLoadFlag() const { return is_load_; } bool ProfilingTrainingTraceOn() const { return is_training_trace_; } bool ProfilingModelLoadOn() const { return is_load_profiling_; } bool ProfilingModelExecuteOn() const; bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // only used by command pattern - bool IsAclApiMode() const { return is_acl_api_mode_; } - int32_t GetOpTraceIterNum() const { return op_trace_iter_num_; } void ReportProfilingData(uint32_t model_id, const std::vector &task_desc_info, - const std::vector &compute_graph_desc_info, - bool check_device); - void Report(const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, - Msprof::Engine::ReporterData &reporter_data); + const std::vector &compute_graph_desc_info); void ProfilingTaskDescInfo(uint32_t model_id, const std::vector &task_desc_info, const int32_t &device_id); void ProfilingGraphDescInfo(uint32_t model_id, const std::vector &compute_graph_desc_info, const int32_t &device_id); - void SetProfilingConfig(const string &profiling_cfg); - vector GetProfilingDeviceId() const { return device_id_; } - void PluginUnInit(const std::string &module) const; + Status PluginInit() const; + void PluginUnInit() const; + Status CallMsprofReport(ReporterData &reporter_data) const; + struct MsprofCallback &GetMsprofCallback() { return prof_cb_; } + void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; } + void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; } private: - ge::Status ParseFeaturesFromAclCfg(const Json &feature); - ge::Status ProfParseParam(const std::map &config_para, int32_t &device_num, - vector &device_list); - ge::Status ProfParseDeviceId(const std::map &config_para, + Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf); + Status ProfParseParam(const std::map &config_para, int32_t &device_num, + vector &device_list); + Status ProfParseDeviceId(const std::map &config_para, vector &device_list); uint64_t GetProfilingModule(); + void GraphDescReport(const int32_t &device_id, const string &data); void UpdateDeviceIdModuleMap(string prof_type, uint64_t module, const vector &device_list); - bool is_load_profiling_ = false; - bool is_execute_profiling_ = false; - bool is_op_trace_ = false; - bool is_load_ = false; - bool is_training_trace_ = false; - bool is_acl_api_mode_ = false; - int32_t op_trace_iter_num_ = 0; - string job_id_; - string prof_dir_; + void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); + + bool is_load_profiling_; + bool is_execute_profiling_; + bool is_training_trace_; vector device_id_; - vector op_trace_conf_; - vector profiling_opts_; - vector prof_handle_vec_; - string recv_profiling_config_; - string send_profiling_config_; - string system_trace_conf_; - string task_trace_conf_; - const ProfilingEngineImpl engine_; map device_id_module_map_; // key: device_id, value: profiling on module map subs_dev_module_; // key: device_id, value: profiling on module uint32_t subscribe_count_; std::mutex mutex_; + MsprofCallback prof_cb_; }; } // namespace ge #endif // GE_COMMON_PROFILING_PROFILING_MANAGER_H_ diff --git a/ge/executor/CMakeLists.txt b/ge/executor/CMakeLists.txt index d7dfdc84..29215cc1 100644 --- a/ge/executor/CMakeLists.txt +++ b/ge/executor/CMakeLists.txt @@ -17,6 +17,7 @@ set(SRC_LIST "../common/dump/dump_properties.cc" "../common/dump/dump_manager.cc" "../common/dump/dump_op.cc" + "../common/profiling/ge_profiling.cc" "../graph/load/graph_loader.cc" "../graph/execute/graph_execute.cc" "../omm/csa_interact.cc" diff --git a/ge/executor/ge_executor.cc b/ge/executor/ge_executor.cc index 26b0a323..57ab7800 100755 --- a/ge/executor/ge_executor.cc +++ b/ge/executor/ge_executor.cc @@ -304,7 +304,7 @@ Status GeExecutor::Finalize() { // Stop profiling if (ProfilingManager::Instance().ProfilingOn()) { ProfilingManager::Instance().StopProfiling(); - ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); + ProfilingManager::Instance().PluginUnInit(); } GELOGI("Uninit GeExecutor over."); diff --git a/ge/executor/module.mk b/ge/executor/module.mk index 9566ca64..cc64007e 100644 --- a/ge/executor/module.mk +++ b/ge/executor/module.mk @@ -8,6 +8,7 @@ local_ge_executor_src_files := \ ../common/dump/dump_op.cc \ ../common/ge/plugin_manager.cc \ ../common/ge/op_tiling_manager.cc \ + ../common/profiling/ge_profiling.cc \ ../graph/load/graph_loader.cc \ ../graph/execute/graph_execute.cc \ ../omm/csa_interact.cc \ diff --git a/ge/ge_runner.mk b/ge/ge_runner.mk index 5205a06c..5e700f2b 100644 --- a/ge/ge_runner.mk +++ b/ge/ge_runner.mk @@ -29,6 +29,7 @@ LIBGE_LOCAL_SRC_FILES := \ common/dump/dump_manager.cc \ common/dump/dump_properties.cc \ common/dump/dump_op.cc \ + common/profiling/ge_profiling.cc \ engine_manager/dnnengine_manager.cc \ ge_local_engine/engine/host_cpu_engine.cc \ generator/ge_generator.cc \ @@ -307,7 +308,6 @@ LIBGE_LOCAL_SRC_FILES := \ LIBCLIENT_LOCAL_SRC_FILES := \ proto/ge_api.proto \ client/ge_api.cc \ - client/ge_prof.cc \ RUNNER_LOCAL_C_INCLUDES := \ $(LOCAL_PATH) ./ \ @@ -409,7 +409,6 @@ endif LOCAL_C_INCLUDES := $(RUNNER_LOCAL_C_INCLUDES) LOCAL_SRC_FILES := ../../out/ge/lib64/stub/ge_api.cc \ - ../../out/ge/lib64/stub/ge_prof.cc \ ../../out/ge/lib64/stub/ge_ir_build.cc \ LOCAL_SHARED_LIBRARIES := diff --git a/ge/graph/load/new_model_manager/davinci_model.cc b/ge/graph/load/new_model_manager/davinci_model.cc index d0881d95..18435571 100755 --- a/ge/graph/load/new_model_manager/davinci_model.cc +++ b/ge/graph/load/new_model_manager/davinci_model.cc @@ -712,7 +712,7 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size // collect profiling for ge auto &profiling_manager = ProfilingManager::Instance(); if (profiling_manager.ProfilingModelLoadOn()) { - Status p_ret = ReportProfilingData(!profiling_manager.IsAclApiMode()); + Status p_ret = ReportProfilingData(); if (p_ret != SUCCESS) { GELOGE(p_ret, "Report profiling data failed."); return p_ret; @@ -723,14 +723,14 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size return ret; } -Status DavinciModel::ReportProfilingData(bool check_device) { +Status DavinciModel::ReportProfilingData() { std::vector compute_graph_desc_info; Status ret = GetComputeGraphInfo(compute_graph_desc_info); if (ret != SUCCESS) { GELOGE(ret, "GetComputeGraphInfo failed."); return ret; } - ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info, check_device); + ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info); GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); op_list_.clear(); @@ -2250,10 +2250,8 @@ inline int64_t SumSize(const vector &size_list) { Status DavinciModel::SinkModelProfile() { // profiling plugin must be registered - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); - - Msprof::Engine::ReporterData reporter_data{}; + auto &prof_mgr = ProfilingManager::Instance(); + ReporterData reporter_data; // report model data tag name std::string tag_name; tag_name.append("model_load_info_").append(std::to_string(this->Id())); @@ -2271,32 +2269,32 @@ Status DavinciModel::SinkModelProfile() { reporter_data.deviceId = device_id_; reporter_data.data = (unsigned char *)&name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)name.c_str(); reporter_data.dataLen = name.size(); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); uint32_t model_id = this->Id(); reporter_data.data = (unsigned char *)&model_id; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // Load Start/End Time int64_t start_time = this->GetLoadBeginTime(); reporter_data.data = (unsigned char *)&start_time; reporter_data.dataLen = sizeof(int64_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); int64_t end_time = this->GetLoadEndTime(); reporter_data.data = (unsigned char *)&end_time; reporter_data.dataLen = sizeof(int64_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); int32_t task_num = task_list_.size(); std::multimap op_id_map; @@ -2310,6 +2308,7 @@ Status DavinciModel::SinkModelProfile() { uint32_t op_num = fusion_op_info->original_op_names.size(); uint32_t task_id = task->GetTaskID(); if (op_num > 0) { + GELOGI("task.id = %u, opNum = %u", task_id, op_num); op_id_map.insert(std::make_pair(fusion_op_info->op_index, task_id)); } } @@ -2352,39 +2351,39 @@ Status DavinciModel::SinkModelProfile() { int32_t fusion_op_name_len = fusion_op_name.size() == 0 ? 1 : fusion_op_name.size(); reporter_data.data = (unsigned char *)&fusion_op_name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)fusion_op_name.c_str(); reporter_data.dataLen = fusion_op_name_len; - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // original op name before fusion reporter_data.data = (unsigned char *)&op_num; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); for (uint32_t k = 0; k < op_num; k++) { std::string op_name = fusion_op_info->original_op_names[k]; int32_t op_name_len = op_name.size() == 0 ? 1 : op_name.size(); reporter_data.data = (unsigned char *)&op_name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)op_name.c_str(); reporter_data.dataLen = op_name_len; - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); } // stream id info uint32_t streamId = task->GetStreamId(); reporter_data.data = (unsigned char *)&streamId; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // memory info struct memoryInfo memory_info; @@ -2400,22 +2399,22 @@ Status DavinciModel::SinkModelProfile() { memory_info.weight_size + memory_info.input_size + memory_info.output_size + memory_info.workspace_size; reporter_data.data = (unsigned char *)&memory_info; reporter_data.dataLen = sizeof(struct memoryInfo); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // task info reporter_data.data = (unsigned char *)&task_count; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); Range task_range = op_id_map.equal_range(op_id); for (CIT idx = task_range.first; idx != task_range.second; ++idx) { uint32_t task_id = idx->second; reporter_data.data = (unsigned char *)&task_id; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); } } } @@ -2424,10 +2423,8 @@ Status DavinciModel::SinkModelProfile() { Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { // profiling plugin must be registered - Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); - GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); - - Msprof::Engine::ReporterData reporter_data{}; + auto &prof_mgr = ProfilingManager::Instance(); + ReporterData reporter_data; // report model data tag name std::string tag_name; tag_name.append("model_time_info_") @@ -2450,33 +2447,33 @@ Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { size_t name_len = name.size(); reporter_data.data = (unsigned char *)&name_len; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); reporter_data.data = (unsigned char *)name.c_str(); reporter_data.dataLen = name.size(); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", - this->Id()); + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, + "Reporter data fail, model id:%u.", this->Id()); // request id uint64_t request_id = current_data.request_id; reporter_data.data = (unsigned char *)&request_id; reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); // thread id int32_t thread_id = GetDataInputTid(); reporter_data.data = (unsigned char *)&thread_id; reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); // time info time_info_.modelId = this->Id(); reporter_data.data = (unsigned char *)&time_info_; reporter_data.dataLen = sizeof(struct timeInfo); - GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, + GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); return SUCCESS; @@ -2809,71 +2806,32 @@ void *DavinciModel::Run(DavinciModel *model) { } GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_PRE_PROC_END)); GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_START)); - if (ProfilingManager::Instance().ProfilingOpTraceOn()) { - GELOGI("GetOpTraceIterNum:%d", ProfilingManager::Instance().GetOpTraceIterNum()); - for (int32_t i = 0; i < ProfilingManager::Instance().GetOpTraceIterNum(); i++) { - if (!ProfilingManager::Instance().ProfilingLoadFlag()) { - vector prof_device_id_vec = ProfilingManager::Instance().GetProfilingDeviceId(); - for (size_t j = 0; j < prof_device_id_vec.size(); ++j) { - // just profiling, no need to check value - (void)ProfilingManager::Instance().StartProfiling(i, prof_device_id_vec[j]); - } - } - - GELOGI("rtModelExecute start."); - rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; - (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - continue); // [No need to check value] - GELOGI("rtModelExecute end"); - - GELOGI("rtStreamSynchronize start."); - rt_ret = rtStreamSynchronize(model->rt_model_stream_); - if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { - GELOGI("The model with multiple datasets aborts normally."); - } else { - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; - (void)model->ReturnResult(current_data.index, false, seq_end_flag, data_wrapper->GetOutput()); - continue); // [No need to check value] - } - - GELOGI("rtStreamSynchronize end."); - (void)ProfilingManager::Instance().StopProfiling(); // just profiling, no need to check value - } - } else { - GE_TIMESTAMP_START(rtModelExecute); - GELOGI("rtModelExecute start."); - rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; - (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); - continue); - GELOGI("rtModelExecute end"); - GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); - - GE_TIMESTAMP_START(rtStreamSynchronize); - GELOGI("rtStreamSynchronize start."); - rt_ret = rtStreamSynchronize(model->rt_model_stream_); - if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { - seq_end_flag = true; - } - if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { - GELOGI("The model with multiple datasets aborts normally."); - } else { - GE_IF_BOOL_EXEC( - rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); - (void)model->ReturnResult(current_data.index, false, seq_end_flag, - data_wrapper->GetOutput()); // [No need to check value] - CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); - continue); - } - - GELOGI("rtStreamSynchronize end."); - GE_IF_BOOL_EXEC(model->is_first_execute_, - GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); - } - + GE_TIMESTAMP_START(rtModelExecute); + GELOGI("rtModelExecute start."); + rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; + (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); + CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); + continue); + GELOGI("rtModelExecute end"); + GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); + + GE_TIMESTAMP_START(rtStreamSynchronize); + GELOGI("rtStreamSynchronize start."); + rt_ret = rtStreamSynchronize(model->rt_model_stream_); + if (rt_ret == RT_ERROR_END_OF_SEQUENCE) { + seq_end_flag = true; + } + GE_IF_BOOL_EXEC( + rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); + (void)model->ReturnResult(current_data.index, false, seq_end_flag, + data_wrapper->GetOutput()); // [No need to check value] + CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); + continue); + GELOGI("rtStreamSynchronize end."); + GE_IF_BOOL_EXEC(model->is_first_execute_, + GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); + GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_AFTER_PROC_START)); GE_TIMESTAMP_START(ReturnResult3); diff --git a/ge/graph/load/new_model_manager/davinci_model.h b/ge/graph/load/new_model_manager/davinci_model.h index 650f19eb..19888e1f 100755 --- a/ge/graph/load/new_model_manager/davinci_model.h +++ b/ge/graph/load/new_model_manager/davinci_model.h @@ -440,7 +440,7 @@ class DavinciModel { Status SinkTimeProfile(const InputData ¤t_data); - Status ReportProfilingData(bool check_device = true); + Status ReportProfilingData(); void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) { data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id); diff --git a/ge/graph/load/new_model_manager/model_manager.cc b/ge/graph/load/new_model_manager/model_manager.cc index 0b55d150..b595ac39 100755 --- a/ge/graph/load/new_model_manager/model_manager.cc +++ b/ge/graph/load/new_model_manager/model_manager.cc @@ -40,9 +40,7 @@ const int kCmdParSize = 2; const int kDumpCmdPairSize = 2; const std::size_t kProfCmdParaMaxSize = 1000; const std::size_t kProfStartCmdParaSize = 2; -const std::string kCmdTypeProfile = "profile"; const std::string kCmdTypeDump = "dump"; -const std::string kCmdTypeProfiling = "profiling"; const std::string kCmdTypeProfInit = "prof_init"; const std::string kCmdTypeProfFinalize = "prof_finalize"; const std::string kCmdTypeProfStart = "prof_start"; @@ -632,8 +630,7 @@ Status ModelManager::Stop(uint32_t model_id) { /// Status ModelManager::HandleCommand(const Command &command) { static const std::map> cmds = { - {kCmdTypeProfile, HandleProfileCommand}, {kCmdTypeDump, HandleDumpCommand}, - {kCmdTypeProfiling, HandleAclProfilingCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, + {kCmdTypeDump, HandleDumpCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, {kCmdTypeProfFinalize, HandleProfFinalizeCommand}, {kCmdTypeProfStart, HandleProfStartCommand}, {kCmdTypeProfStop, HandleProfStopCommand}, {kCmdTypeProfModelSubscribe, HandleProfModelSubscribeCommand}, @@ -648,21 +645,6 @@ Status ModelManager::HandleCommand(const Command &command) { } } -Status ModelManager::HandleAclProfilingCommand(const Command &command) { - if (command.cmd_params.size() < kCmdParSize) { - GELOGE(PARAM_INVALID, "When the cmd_type is 'profiling', the size of cmd_params must larger than 2."); - return PARAM_INVALID; - } - - std::string map_key = command.cmd_params[0]; - std::string value = command.cmd_params[1]; - if (map_key == PROFILE_CONFIG) { - ProfilingManager::Instance().SetProfilingConfig(value); - } - - return SUCCESS; -} - Status ModelManager::GetModelByCmd(const Command &command, std::shared_ptr &davinci_model) { if (command.cmd_params.size() < kCmdParSize) { @@ -809,29 +791,6 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { return SUCCESS; } -Status ModelManager::HandleProfileCommand(const Command &command) { - if (command.cmd_params.size() < kCmdParSize) { - GELOGE(PARAM_INVALID, "When the cmd_type is 'profile', the size of cmd_params must larger than 2."); - return PARAM_INVALID; - } - - std::string map_key = command.cmd_params[0]; - std::string value = command.cmd_params[1]; - - GELOGI("Profiling mode, Command key:%s , value:%s ", map_key.c_str(), value.c_str()); - - auto iter = PROFILE_COMPONENT_MAP.find(map_key); - if (iter != PROFILE_COMPONENT_MAP.end()) { - std::string property_value = (value == "on") ? "1" : "0"; - PropertiesManager::Instance().SetPropertyValue(iter->second, property_value); - } - - if ((map_key == PROFILER_JOBCTX || map_key == PROFILER_TARGET_PATH || map_key == RTS_PROFILE_PATH)) { - PropertiesManager::Instance().SetPropertyValue(map_key, value); - } - return SUCCESS; -} - static Status ParserPara(const Command &command, const string &dump_key, string &dump_value) { auto iter = std::find(command.cmd_params.begin(), command.cmd_params.end(), dump_key); if (iter != command.cmd_params.end()) { diff --git a/ge/graph/load/new_model_manager/model_manager.h b/ge/graph/load/new_model_manager/model_manager.h index dc3a5219..fc98d9c2 100755 --- a/ge/graph/load/new_model_manager/model_manager.h +++ b/ge/graph/load/new_model_manager/model_manager.h @@ -169,8 +169,6 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { /// @brief comment handle function /// ge::Status HandleCommand(const Command &command); - static ge::Status HandleAclProfilingCommand(const Command &command); - static ge::Status HandleProfileCommand(const Command &command); static ge::Status HandleDumpCommand(const Command &command); static ge::Status HandleProfModelSubscribeCommand(const Command &command); static ge::Status HandleProfModelUnsubscribeCommand(const Command &command); diff --git a/ge/hybrid/executor/worker/execution_engine.cc b/ge/hybrid/executor/worker/execution_engine.cc index e6729352..b984eec3 100755 --- a/ge/hybrid/executor/worker/execution_engine.cc +++ b/ge/hybrid/executor/worker/execution_engine.cc @@ -260,8 +260,7 @@ Status NodeDoneCallback::ProfilingReport() { } auto &profiling_manager = ProfilingManager::Instance(); - profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info, - !profiling_manager.IsAclApiMode()); + profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info); return SUCCESS; } diff --git a/ge/init/gelib.cc b/ge/init/gelib.cc index 306a804a..92700179 100755 --- a/ge/init/gelib.cc +++ b/ge/init/gelib.cc @@ -485,11 +485,9 @@ Status GELib::Finalize() { void GELib::ShutDownProfiling() { std::lock_guard lock(status_mutex_); - if (!ProfilingManager::Instance().ProfilingOpTraceOn() && ProfilingManager::Instance().ProfilingOn()) { - ProfilingManager::Instance().StopProfiling(); - } if (ProfilingManager::Instance().ProfilingOn()) { - ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); + ProfilingManager::Instance().StopProfiling(); + ProfilingManager::Instance().PluginUnInit(); } } diff --git a/ge/single_op/single_op.cc b/ge/single_op/single_op.cc index 6e51b6ff..a2652b67 100755 --- a/ge/single_op/single_op.cc +++ b/ge/single_op/single_op.cc @@ -72,8 +72,7 @@ Status ProfilingTaskInfo(OpTask *op_task) { std::vector compute_graph_info; auto &profiling_manager = ProfilingManager::Instance(); - profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info, - !profiling_manager.IsAclApiMode()); + profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info); return SUCCESS; } } // namespace diff --git a/ge/stub/gen_stubapi.py b/ge/stub/gen_stubapi.py index d19b44a6..1476d505 100644 --- a/ge/stub/gen_stubapi.py +++ b/ge/stub/gen_stubapi.py @@ -71,7 +71,7 @@ max_code_len_per_line = 100 when DEBUG on """ white_list_for_debug = ["attr_value.h", "operator.h", "tensor.h", "graph.h", "operator_factory.h", - "ge_ir_build.h", "ge_api.h", "ge_prof.h", "tensorflow_parser.h", "caffe_parser.h"] + "ge_ir_build.h", "ge_api.h", "tensorflow_parser.h", "caffe_parser.h"] include_dir_key_words = ["ge", "graph", "parser"] DEBUG = True diff --git a/inc/framework/common/profiling/ge_profiling.h b/inc/framework/common/profiling/ge_profiling.h new file mode 100644 index 00000000..65a0eebb --- /dev/null +++ b/inc/framework/common/profiling/ge_profiling.h @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INC_FRAMEWORK_COMMON_GE_PROFILING_H_ +#define INC_FRAMEWORK_COMMON_GE_PROFILING_H_ + +#include "ge/ge_api_error_codes.h" +#include "toolchain/prof_callback.h" + +#define MAX_DEV_NUM (64) +enum ProfCommandHandleType { + kProfCommandhandleInit = 0, + kProfCommandhandleStart, + kProfCommandhandleStop, + kProfCommandhandleFinalize, + kProfCommandhandleModelSubscribe, + kProfCommandhandleModelUnsubscribe +}; + +struct ProfCommandHandleData { + unit64_t profSwitch; + uint32_t devNums; // length of device id list + uint32_t devIdList[MAX_DEV_NUM]; + uint32_t modelId; +} + +ge::Status RegProfCtrlCallback(MsprofCtrlCallback func); +ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func); +ge::Status RegProfReporterCallback(MsprofReporterCallback func); +ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len); + +#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_H_ \ No newline at end of file diff --git a/inc/framework/common/profiling/ge_profiling_cb.h b/inc/framework/common/profiling/ge_profiling_cb.h new file mode 100644 index 00000000..025232ff --- /dev/null +++ b/inc/framework/common/profiling/ge_profiling_cb.h @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ +#define INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ + +#include "profiling/ge_profiling.h" + +bool IsInitialize(); + +#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ \ No newline at end of file From 62b3e8eeb4ad9ee4610dd717bd9a5481c6b65438 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 16:59:21 +0800 Subject: [PATCH 02/24] profiling AR version2 --- ge/CMakeLists.txt | 4 +- ge/common/profiling/ge_profiling.cc | 3 +- ge/common/profiling/profiling_manager.cc | 148 ++++++++++++++++------- ge/common/profiling/profiling_manager.h | 6 +- ge/graph/build/task_generator.cc | 20 +-- 5 files changed, 120 insertions(+), 61 deletions(-) diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index 7079e432..e36b45d9 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -648,9 +648,11 @@ target_include_directories(ge_runner PRIVATE target_link_libraries(ge_runner $ + -Wl,--whole-archive + msprofiler_fwk + -Wl,--no-whole-archive ge_memory adump_server - msprofiler_fwk static_mmpa -Wl,--no-as-needed graph diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index 8da6b12a..bab699cc 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -125,8 +125,9 @@ ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { ge::Status RegProfReporterCallback(MsprofReporterCallback func) { if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { - GELOGW("Msprof ctrl callback is exist, just ignore it."); + GELOGW("Msprof reporter callback is exist, just ignore it."); } else { + GELOGI("GE register Msprof reporter callback."); ge::ProfilingManager::Instance().SetMsprofReporterCallback(func); } // Pass MsprofReporterCallback to runtime diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index bcf6d366..fd56f15d 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -24,16 +24,9 @@ #include "graph/load/new_model_manager/davinci_model.h" namespace { -const char *const kJobID = "jobID"; -const char *const kDeviceID = "deviceID"; -const char *const kStartCfg = "startCfg"; -const char *const kFeatures = "features"; -const char *const kConf = "conf"; -const char *const kEvents = "events"; -const char *const kAiCoreEvents = "ai_core_events"; -const char *const kName = "name"; -const char *const kTraceID = "traceId"; -const char *const kProfDir = "resultPath"; +const char *const kTrainingTrace = "training_trace"; +const char *const kFpPoint = "fp_point"; +const char *const kBpPoint = "bp_point"; const size_t kReportMaxLen = 2048; const int32_t kMaxDeviceNum = 256; const std::string kConfigNumsdev = "devNums"; @@ -70,7 +63,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In return ret; } - if (is_load_profiling_) { + if (is_execute_profiling_) { int32_t cb_ret = prof_cb_.msprofCtrlCallback( static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), static_cast(&prof_conf), sizeof(MsprofGeOptions)); @@ -91,37 +84,42 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt #ifdef DAVINCI_SUPPORT_PROFILING // enable profiling by env char env_profiling_mode[MMPA_MAX_PATH] = { 0x00 }; - is_load_profiling_ = false; + is_load_profiling_ = false; // Change in ProfInit is_execute_profiling_ = false; - (void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); - (void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, sizeof(MsprofGeOptions)); - - if ((env_profiling_mode != nullptr) && (strcmp("true", env_profiling_mode) == 0) - && (strcmp(prof_conf.options, "\0") != 0)) { - // enable profiling by env - is_load_profiling_ = true; - is_execute_profiling_ = true; - GELOGI("The profiling in env is %s, %s", env_profiling_mode, prof_conf.options); - } else { - if (options.profiling_mode != "1" || options.profiling_options.empty()) { - return SUCCESS; - } + if (options.profiling_mode == "1" && !options.profiling_options.empty()) { // enable profiling by ge option - if (memcpy_s(prof_conf.options, sizeof(prof_conf.options), options.profiling_options.c_str(), + if (memcpy_s(prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX, options.profiling_options.c_str(), sizeof(options.profiling_options.c_str())) != EOK) { GELOGE(INTERNAL_ERROR, "copy profiling_options failed."); return INTERNAL_ERROR; } - is_load_profiling_ = true; is_execute_profiling_ = true; GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), prof_conf.options); + } else { + (void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); + (void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX); + // The env is invalid + if ((env_profiling_mode == nullptr) || (strcmp("true", env_profiling_mode) != 0) + || (strcmp(prof_conf.options, "\0") == 0)) { + return SUCCESS; + } + // enable profiling by env + is_execute_profiling_ = true; + GELOGI("The profiling in env is %s, %s", env_profiling_mode, prof_conf.options); } - if (!is_load_profiling_) { + if (!is_execute_profiling_) { return SUCCESS; } + // Parse json str for bp fp + Status ret = ParseOptions(prof_conf.options); + if (ret != ge::SUCCESS) { + GELOGE(ge::PARAM_INVALID, "Parse taining trace param failed."); + return ge::PARAM_INVALID; + } + if (memcpy_s(prof_conf.jobId, sizeof(prof_conf.jobId), options.job_id.c_str(), sizeof(options.job_id.c_str())) != EOK) { GELOGE(INTERNAL_ERROR, "copy job_id failed."); @@ -134,23 +132,55 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt return ge::SUCCESS; } +ge::Status ProfilingManager::ParseOptions(const std::string &options) { + if (options.empty()) { + GELOGE(ge::PARAM_INVALID, "Profiling options is empty.") + return ge::PARAM_INVALID; + } + try { + Json prof_options = Json::parse(options); + const std::string training_trace = prof_options[kTrainingTrace]; + if (training_trace.empty()) { + GELOGI("Training trace will not take effect."); + return ge::SUCCESS; + } + GELOGI("GE profiling training trace:%s", training_trace.c_str()); + if (training_trace != "on") { + GELOGE(ge::PARAM_INVALID, "Training trace param:%s is invalid.", training_trace.c_str()); + return ge::PARAM_INVALID; + } + fp_point = prof_options[kFpPoint]; + bp_point = prof_options[kBpPoint]; + if (!fp_point_.empty() && !bp_point_.empty()) { + GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); + } + } catch (...) { + GELOGE(FAILED, "Json prof_conf options is invalid."); + return ge::PARAM_INVALID; + } + return ge::SUCCESS; +} + FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProfiling() { #ifdef DAVINCI_SUPPORT_PROFILING uint64_t module = GetProfilingModule(); + // The following if case will not be executed in normal case, inc case of ProfStopProfiling is abnormal int32_t device_num = static_cast(device_id_.size()); - auto device_id_ptr = std::unique_ptr(new (std::nothrow) uint32_t[device_num]); - if (device_id_ptr == nullptr) { - GELOGE(FAILED, "Stop profiling: device id ptr is null."); - return; - } - for (int32_t i = 0; i < device_num; i++) { - device_id_ptr[i] = static_cast(device_id_[i]); - } - rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); - if (rt_ret != RT_ERROR_NONE) { - GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); + if (device_num != 0) { + auto device_id_ptr = std::unique_ptr(new (std::nothrow) uint32_t[device_num]); + if (device_id_ptr == nullptr) { + GELOGE(FAILED, "Stop profiling: device id ptr is null."); + return; + } + for (int32_t i = 0; i < device_num; i++) { + device_id_ptr[i] = static_cast(device_id_[i]); + } + rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); + if (rt_ret != RT_ERROR_NONE) { + GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); + } } - + // stop profiling int32_t cb_ret = prof_cb_.msprofCtrlCallback(static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), nullptr, 0); @@ -475,6 +505,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi std::lock_guard lock(mutex_); is_load_profiling_ = false; is_training_trace_ = false; + is_execute_profiling_ = false; // profiling plugin uninit PluginUnInit(); @@ -714,7 +745,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ProfilingManager::Profilin execute_model_prof_on = true; } GELOGI("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); - return is_execute_profiling_ || execute_model_prof_on; + return execute_model_prof_on; } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() const { @@ -744,5 +775,40 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::CallMs static_cast(&reporter_data), sizeof(ReporterData)); } +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpPoint( + std::string &fp_point, std::string &bp_point) { + // Env or options mode, fp_point_/bp_point_ have initiliazed on profiling init + if (!fp_point_.empty() && !bp_point_.empty()) { + GELOGI("Bp Fp have been initialized in env or options"); + fp_point = fp_point_; + bp_point = bp_point_; + GELOGI("Bp Fp have been initailized in env or options, bp_point: %s, fp_point: %s", bp_point.c_str(), fp_point.c_str()); + return; + } + // ProfApi mode and training trace is set + try { + char env_profiling_options[MSPROF_OPTIONS_DEF_LEN_MAX] = { 0x00 }; + INT32 ret = mmGetEnv("PROFILING_OPTIONS", env_profiling_options, MSPROF_OPTIONS_DEF_LEN_MAX); + if (ret != EN_OK) { + GELOGI("PROFILING_OPTIONS env is not exist."); + return; + } + GELOGI("Parse env PROFILING_OPTIONS:%s.", env_profiling_options); + Json prof_options = Json::parse(env_profiling_options); + + fp_point_ = prof_options[kFpPoint]; + bp_point_ = prof_options[kBpPoint]; + + fp_point = fp_point_; + bp_point = bp_point_; + if (!fp_point_.empty() && !bp_point_.empty()) { + GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); + } + } catch (...) { + GELOGE(FAILED, "Json prof options is invalid."); + return ge::PARAM_INVALID; + } +} + } // namespace ge diff --git a/ge/common/profiling/profiling_manager.h b/ge/common/profiling/profiling_manager.h index c9434a10..c9ceed92 100755 --- a/ge/common/profiling/profiling_manager.h +++ b/ge/common/profiling/profiling_manager.h @@ -63,7 +63,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { bool ProfilingTrainingTraceOn() const { return is_training_trace_; } bool ProfilingModelLoadOn() const { return is_load_profiling_; } bool ProfilingModelExecuteOn() const; - bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // only used by command pattern + bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // is_execute_profiling_ only used by ge option and env void ReportProfilingData(uint32_t model_id, const std::vector &task_desc_info, const std::vector &compute_graph_desc_info); void ProfilingTaskDescInfo(uint32_t model_id, const std::vector &task_desc_info, @@ -76,8 +76,10 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { struct MsprofCallback &GetMsprofCallback() { return prof_cb_; } void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; } void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; } + void GetFpBpPoint(std::string &fp_point, std::string &bp_point); private: Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf); + Status ParseOptions(const std::string &options); Status ProfParseParam(const std::map &config_para, int32_t &device_num, vector &device_list); Status ProfParseDeviceId(const std::map &config_para, @@ -96,6 +98,8 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { uint32_t subscribe_count_; std::mutex mutex_; MsprofCallback prof_cb_; + std::string fp_point_; + std::string bp_point_; }; } // namespace ge #endif // GE_COMMON_PROFILING_PROFILING_MANAGER_H_ diff --git a/ge/graph/build/task_generator.cc b/ge/graph/build/task_generator.cc index 41607f1f..b506f945 100755 --- a/ge/graph/build/task_generator.cc +++ b/ge/graph/build/task_generator.cc @@ -49,8 +49,6 @@ const char *const kIsLastNode = "is_last_node"; const char *const kIsInputVar = "INPUT_IS_VAR"; const char *const kIsOutputVar = "OUTPUT_IS_VAR"; const char *const kProfilingMode = "PROFILING_MODE"; -const char *const kProfilingFpPoint = "FP_POINT"; -const char *const kProfilingBpPoint = "BP_POINT"; const uint32_t kProfilingArStep = 2; const uint64_t kProfilingFpStartLogid = 1; const uint64_t kProfilingBpEndLogid = 2; @@ -810,35 +808,23 @@ Status TaskGenerator::GetFpBpIndex(const ComputeGraphPtr &graph, ProfilingPoint vector &all_reduce_nodes, std::string &fp_point_str, std::string &bp_point_str) const { - if (ge::GetContext().GetOption(OPTION_EXEC_PROFILING_FPPONIT_OPTIONS, fp_point_str) == SUCCESS && - ge::GetContext().GetOption(OPTION_EXEC_PROFILING_BPPONIT_OPTIONS, bp_point_str) == SUCCESS && - !fp_point_str.empty() && !bp_point_str.empty()) { - return SUCCESS; - } + ProfilingManager::Instance().GetFpBpPoint(fp_point_str, bp_point_str); Status ret = SUCCESS; - const char *fp_point = std::getenv(kProfilingFpPoint); - if (fp_point == nullptr) { + if (fp_point_str.empty()) { ret = AutoFindFpOpIndex(graph, profiling_point); if (ret != SUCCESS) { GELOGW("First forward profiling op_index not set and FindFpOpIndex failed."); return FAILED; } - } else { - fp_point_str = string(fp_point); - GELOGI("Get fp_point_str from env %s", fp_point_str.c_str()); } - const char *bp_point = std::getenv(kProfilingBpPoint); - if (bp_point == nullptr) { + if (bp_point_str.empty()) { ret = AutoFindBpOpIndex(graph, profiling_point, all_reduce_nodes); if (ret != SUCCESS) { GELOGW("Last backward profiling op_index not set and FindBpOpIndex failed."); return FAILED; } - } else { - bp_point_str = string(bp_point); - GELOGI("Get bp_point_str from env %s", bp_point_str.c_str()); } return SUCCESS; From 612c82f405cdde94b26e538d58942e94dda5d9fd Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 17:41:02 +0800 Subject: [PATCH 03/24] Profiling AR version 3 --- ge/common/profiling/ge_profiling.cc | 74 ++++++++++++++++-------- ge/common/profiling/profiling_manager.cc | 56 ++++++++++++++++-- ge/common/profiling/profiling_manager.h | 17 ++++++ 3 files changed, 119 insertions(+), 28 deletions(-) diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index bab699cc..4fc61ca0 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -22,6 +22,7 @@ #include "graph/load/graph_loader.h" #include "init/gelib.h" #include "framework/common/ge_inner_error_codes.h" +#include "opskernel_manager/ops_kernel_builder_manager.h" namespace { const uint32_t kDeviceListIndex = 3; @@ -44,7 +45,7 @@ const std::map kProfCommandTypeMap = { {kProfCommandhandleModelUnsubscribe, kProfModelUnsubscribe}}; } // namespace -bool TransProfConfigToParam(const MsprofCommandHandle &profCommand, vector &prof_config_params) { +bool TransProfConfigToParam(const ProfCommandHandleData &profCommand, vector &prof_config_params) { prof_config_params.clear(); prof_config_params.emplace_back(kDeviceNums); prof_config_params.emplace_back(std::to_string(profCommand.devNums)); @@ -71,7 +72,7 @@ bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { return false; } if (device_nums == 0 || device_nums > MAX_DEV_NUM) { - GELOGE(ge::PARAM_INVALID, "The device nums is invalid."); + GELOGE(ge::PARAM_INVALID, "The device nums: %u is invalid.", device_nums); return false; } @@ -105,16 +106,26 @@ bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { } ge::Status RegProfCtrlCallback(MsprofCtrlCallback func) { + if (func == nullptr) { + GELOGE(ge::PARAM_INVALID, "Msprof ctrl callback is nullptr."); + return ge::PARAM_INVALID; + } if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { GELOGW("Msprof ctrl callback is exist, just ignore it."); } else { + GELOGI("GE register Msprof ctrl callback."); ge::ProfilingManager::Instance().SetMsprofCtrlCallback(func); } return ge::SUCCESS; } ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { + if (func == nullptr) { + GELOGE(ge::PARAM_INVALID, "MsprofSetDeviceCallback callback is nullptr."); + return ge::PARAM_INVALID; + } // Pass MsprofSetDeviceCallback to runtime + GELOGI("GE pass setdevice callback to runtime."); ge::Status rt_ret = rtRegDeviceStateCallback(kRtSetDeviceRegName.c_str(), static_cast(func)); if (rt_ret != ge::SUCCESS) { GELOGE(rt_ret, "Pass MsprofSetDeviceCallback to runtime failed!"); @@ -124,54 +135,71 @@ ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { } ge::Status RegProfReporterCallback(MsprofReporterCallback func) { + if (func == nullptr) { + GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); + return ge::PARAM_INVALID; + } if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { GELOGW("Msprof reporter callback is exist, just ignore it."); } else { GELOGI("GE register Msprof reporter callback."); ge::ProfilingManager::Instance().SetMsprofReporterCallback(func); + // Pass MsprofReporterCallback to runtime + ge::Status rt_ret = rtSetMsprofReporterCallback(func); + if (rt_ret != ge::SUCCESS) { + GELOGE(rt_ret, "Pass MsprofReporterCallback to runtime failed!!"); + return rt_ret; + } + // Pass MsprofReporterCallback to hccl in opskernel so initialize + rt_ret = OpskernelBuilderManager::Instance().RegProfReporterCallback(func); + if (rt_ret != ge::SUCCESS) { + GELOGE(rt_ret, "Pass MsprofReporterCallback to hccl failed."); + return rt_ret; + } } - // Pass MsprofReporterCallback to runtime - ge::Status rt_ret = rtSetMsprofReporterCallback(func); - if (rt_ret != ge::SUCCESS) { - GELOGE(rt_ret, "Pass MsprofReporterCallback to runtime failed!!"); - return rt_ret; - } - // Pass MsprofReporterCallback to hccl in opskernel so initialize - return ge::SUCCESS; } ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len) { - GE_CHECK_NOTNULL(data); - MsprofCommandHandle *prof_config_param = (MsprofCommandHandle *)data; - if (!isProfConfigValid(prof_config_param->devIdList, prof_config_param->devNums)) { - return ge::FAILED; - } - std::vector prof_params; - if (!TransProfConfigToParam(*prof_config_param, prof_params)) { - GELOGE(ge::PARAM_INVALID, "Transfer profilerConfig to string vector failed"); - return ge::PARAM_INVALID; + if (type != kProfCommandhandleFinalize) { + GE_CHECK_NOTNULL(data); } + MsprofCommandHandleData *prof_config_param = (MsprofCommandHandleData *)data; auto iter = kProfCommandTypeMap.find(type); if (iter == kProfCommandTypeMap.end()) { GELOGW("The prof comand type is invalid."); return ge::PARAM_INVALID; } + std::vector prof_params; + if (type == kProfCommandhandleStart || type == kProfCommandhandleStop) { + if (!isProfConfigValid(prof_config_param->devIdList, prof_config_param->devNums)) { + return ge::FAILED; + } + + if (!TransProfConfigToParam(*prof_config_param, prof_params)) { + GELOGE(ge::PARAM_INVALID, "Transfer profilerConfig to string vector failed"); + return ge::PARAM_INVALID; + } + } ge::GraphLoader graph_loader; ge::Command command; command.cmd_params.clear(); command.cmd_type = iter->second; command.cmd_params = prof_params; - command.module_index = prof_config_param->profSwitch; - GELOGI("GE commandhandle execute, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), - prof_params[kDeviceListIndex].c_str(), command.module_index); + if (type != kProfCommandhandleFinalize) { + command.module_index = prof_config_param->profSwitch; + } + GELOGI("GE commandhandle execute, Command Type: %d, data type config: 0x%llx", type, command.module_index); + if (type == kProfCommandhandleStart || type == kProfCommandhandleStop) { + GELOGI("Profiling device nums:%s , deviceID:[%s]", prof_params[0].c_str(), prof_params[kDeviceListIndex].c_str()); + } ge::Status ret = graph_loader.CommandHandle(command); if (ret != ge::SUCCESS) { GELOGE(ret, "Handle profiling command failed"); return ge::FAILED; } - GELOGI("Successfully execute profiling command 0x%llx.", command.module_index); + GELOGI("Successfully execute profiling command type: %d, command 0x%llx.", type, command.module_index); return ge::SUCCESS; } diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index fd56f15d..bfc747b5 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -22,6 +22,7 @@ #include "graph/ge_context.h" #include "runtime/base.h" #include "graph/load/new_model_manager/davinci_model.h" +#include "opskernel_manager/ops_kernel_builder_manager.h" namespace { const char *const kTrainingTrace = "training_trace"; @@ -41,7 +42,10 @@ namespace ge { ProfilingManager::ProfilingManager() : is_load_profiling_(false), is_execute_profiling_(false), is_training_trace_(false), - subscribe_count_(0) {} + subscribe_count_(0) { + prof_cb_.msprofCtrlCallback = nullptr; + prof_cb_.msprofReporterCallback = nullptr; +} ProfilingManager::~ProfilingManager() {} @@ -64,6 +68,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In } if (is_execute_profiling_) { + if (prof_cb_.msprofCtrlCallback == nullptr) { + GELOGE(ge::PARAM_INVALID, "MsprofCtrlCallback callback is nullptr."); + return ge::PARAM_INVALID; + } int32_t cb_ret = prof_cb_.msprofCtrlCallback( static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), static_cast(&prof_conf), sizeof(MsprofGeOptions)); @@ -116,7 +124,7 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt // Parse json str for bp fp Status ret = ParseOptions(prof_conf.options); if (ret != ge::SUCCESS) { - GELOGE(ge::PARAM_INVALID, "Parse taining trace param failed."); + GELOGE(ge::PARAM_INVALID, "Parse training trace param failed."); return ge::PARAM_INVALID; } @@ -182,6 +190,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf } // stop profiling + if (prof_cb_.msprofCtrlCallback == nullptr) { + GELOGE(ge::PARAM_INVALID, "MsprofCtrlCallback callback is nullptr."); + return ge::PARAM_INVALID; + } int32_t cb_ret = prof_cb_.msprofCtrlCallback(static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), nullptr, 0); if (cb_ret != 0) { @@ -210,7 +222,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin .append(std::to_string(stream_id)).append(" ") .append(std::to_string(model_id)).append("\n")); - ReporterData reporter_data; + ReporterData reporter_data{}; reporter_data.deviceId = device_id; reporter_data.data = (unsigned char *)data.c_str(); reporter_data.dataLen = data.size(); @@ -298,7 +310,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin void ProfilingManager::GraphDescReport(const int32_t &device_id, const string &data) { #ifdef DAVINCI_SUPPORT_PROFILING - ReporterData reporter_data; + ReporterData reporter_data{}; int ret = -1; int32_t cb_ret = -1; size_t index = data.size() / kReportMaxLen; @@ -487,6 +499,11 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn GELOGE(FAILED, "Runtime profiler start failed."); return FAILED; } + Status hccl_ret = OpskernelBuilderManager::Instance().ProfStart(model_load_mask); + if (hccl_ret != SUCCESS) { + GELOGE(FAILED, "Hccl profiler start failed."); + return FAILED; + } is_load_profiling_ = true; GELOGI("Prof init: model load profiling on."); } @@ -517,6 +534,11 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi return FAILED; } + Status hccl_ret = OpskernelBuilderManager::Instance().ProfStop(PROF_MODEL_LOAD_MASK); + if (hccl_ret != SUCCESS) { + GELOGE(FAILED, "Hccl profiler stop failed."); + return FAILED; + } for (auto device_id_module : device_id_module_map_) { if (device_id_module.second != 0) { uint32_t device_id = static_cast(device_id_module.first); @@ -640,6 +662,12 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGE(FAILED, "Runtime profiler config proc failed."); return FAILED; } + + Status hccl_ret = OpskernelBuilderManager::Instance().ProfStart(module); + if (hccl_ret != SUCCESS) { + GELOGE(FAILED, "Hccl profiler start failed."); + return FAILED; + } if ((module & PROF_MODEL_EXECUTE_MASK) == PROF_MODEL_EXECUTE_MASK) { for (int32_t i = 0; i < device_num; i++) { if (std::find(device_id_.begin(), device_id_.end(), device_list[i]) == device_id_.end()) { @@ -681,6 +709,11 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); return FAILED; } + Status hccl_ret = OpskernelBuilderManager::Instance().ProfStop(module); + if (hccl_ret != SUCCESS) { + GELOGE(FAILED, "Hccl profiler stop failed."); + return FAILED; + } uint64_t execute_model_mask = module & PROF_MODEL_EXECUTE_MASK; if (execute_model_mask == PROF_MODEL_EXECUTE_MASK) { for (int32_t i = 0; i < device_num; i++) { @@ -749,6 +782,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ProfilingManager::Profilin } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() const { + if (prof_cb_.msprofReporterCallback == nullptr) { + GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); + return ge::PARAM_INVALID; + } return prof_cb_.msprofReporterCallback( static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_INIT), @@ -757,6 +794,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::Plugin FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit() const { #ifdef DAVINCI_SUPPORT_PROFILING + if (prof_cb_.msprofReporterCallback == nullptr) { + GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); + return ge::PARAM_INVALID; + } int32_t cb_ret = prof_cb_.msprofReporterCallback( static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_UNINIT), @@ -769,6 +810,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUn FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::CallMsprofReport( ReporterData &reporter_data) const { + if (prof_cb_.msprofReporterCallback == nullptr) { + GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); + return ge::PARAM_INVALID; + } return prof_cb_.msprofReporterCallback( static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_REPORT), @@ -806,8 +851,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpP } } catch (...) { GELOGE(FAILED, "Json prof options is invalid."); - return ge::PARAM_INVALID; + return; } + return; } diff --git a/ge/common/profiling/profiling_manager.h b/ge/common/profiling/profiling_manager.h index c9ceed92..5fa4fac4 100755 --- a/ge/common/profiling/profiling_manager.h +++ b/ge/common/profiling/profiling_manager.h @@ -35,6 +35,23 @@ using Json = nlohmann::json; namespace { const std::string GE_PROFILING_MODULE = "Framework"; + // DataTypeConfig MASK + #define PROF_ACL_API_MASK 0x0001 + #define PROF_TASK_TIME_MASK 0x0002 + #define PROF_AICORE_METRICS_MASK 0x0004 + #define PROF_AICPU_TRACE_MASK 0x0008 + #define PROF_MODEL_EXECUTE_MASK 0x0010 + #define PROF_RUNTIME_API_MASK 0x0020 + #define PROF_RUNTIME_TRACE_MASK 0x0040 + #define PROF_SCHEDULE_TIMELINE_MASK 0x0080 + #define PROF_SCHEDULE_TRACE_MASK 0x0100 + #define PROF_AIVECTORCORE_METRICS_MASK 0x0200 + #define PROF_SUBTASK_TIME_MASK 0x0400 + #define PROF_TRAINING_TRACE_MASK 0x0800 + #define PROF_HCCL_TRACE_MASK 0x1000 + #define PROF_DATA_PROCESS_MASK 0x2000 + #define PROF_MODEL_LOAD_MASK 0x8000000000000000 + } // namespace namespace ge { struct DeviceSubsInfo { From 09e82a80daad693addcd56d5be64d43407b765f9 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 19:36:21 +0800 Subject: [PATCH 04/24] Profiling AR version4 --- ge/common/profiling/ge_profiling.cc | 6 ++--- ge/common/profiling/profiling_manager.cc | 22 +++++++++---------- .../load/new_model_manager/davinci_model.cc | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index 4fc61ca0..b6708f7a 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -139,7 +139,7 @@ ge::Status RegProfReporterCallback(MsprofReporterCallback func) { GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); return ge::PARAM_INVALID; } - if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { + if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofReporterCallback != nullptr) { GELOGW("Msprof reporter callback is exist, just ignore it."); } else { GELOGI("GE register Msprof reporter callback."); @@ -151,7 +151,7 @@ ge::Status RegProfReporterCallback(MsprofReporterCallback func) { return rt_ret; } // Pass MsprofReporterCallback to hccl in opskernel so initialize - rt_ret = OpskernelBuilderManager::Instance().RegProfReporterCallback(func); + rt_ret = OpsKernelBuilderManager::Instance().RegProfReporterCallBack(func); if (rt_ret != ge::SUCCESS) { GELOGE(rt_ret, "Pass MsprofReporterCallback to hccl failed."); return rt_ret; @@ -164,7 +164,7 @@ ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t le if (type != kProfCommandhandleFinalize) { GE_CHECK_NOTNULL(data); } - MsprofCommandHandleData *prof_config_param = (MsprofCommandHandleData *)data; + ProfCommandHandleData *prof_config_param = (ProfCommandHandleData *)data; auto iter = kProfCommandTypeMap.find(type); if (iter == kProfCommandTypeMap.end()) { GELOGW("The prof comand type is invalid."); diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index bfc747b5..222cde6a 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -76,7 +76,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), static_cast(&prof_conf), sizeof(MsprofGeOptions)); if (cb_ret != 0) { - GELOGE(FAILED, "call msprofCtrlCallback failed, type:%u, return:%d", + GELOGE(FAILED, "Call msprofCtrlCallback failed, type:%u, return:%d", static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), cb_ret); return FAILED; } @@ -110,7 +110,7 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt // The env is invalid if ((env_profiling_mode == nullptr) || (strcmp("true", env_profiling_mode) != 0) || (strcmp(prof_conf.options, "\0") == 0)) { - return SUCCESS; + return SUCCESS; } // enable profiling by env is_execute_profiling_ = true; @@ -157,8 +157,8 @@ ge::Status ProfilingManager::ParseOptions(const std::string &options) { GELOGE(ge::PARAM_INVALID, "Training trace param:%s is invalid.", training_trace.c_str()); return ge::PARAM_INVALID; } - fp_point = prof_options[kFpPoint]; - bp_point = prof_options[kBpPoint]; + fp_point_ = prof_options[kFpPoint]; + bp_point_ = prof_options[kBpPoint]; if (!fp_point_.empty() && !bp_point_.empty()) { GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); } @@ -175,7 +175,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf // The following if case will not be executed in normal case, inc case of ProfStopProfiling is abnormal int32_t device_num = static_cast(device_id_.size()); if (device_num != 0) { - auto device_id_ptr = std::unique_ptr(new (std::nothrow) uint32_t[device_num]); + auto device_id_ptr = std::unique_ptr(new (std::nothrow) uint32_t[device_num]); if (device_id_ptr == nullptr) { GELOGE(FAILED, "Stop profiling: device id ptr is null."); return; @@ -499,7 +499,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn GELOGE(FAILED, "Runtime profiler start failed."); return FAILED; } - Status hccl_ret = OpskernelBuilderManager::Instance().ProfStart(model_load_mask); + Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStart(model_load_mask); if (hccl_ret != SUCCESS) { GELOGE(FAILED, "Hccl profiler start failed."); return FAILED; @@ -534,7 +534,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi return FAILED; } - Status hccl_ret = OpskernelBuilderManager::Instance().ProfStop(PROF_MODEL_LOAD_MASK); + Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStop(PROF_MODEL_LOAD_MASK); if (hccl_ret != SUCCESS) { GELOGE(FAILED, "Hccl profiler stop failed."); return FAILED; @@ -663,7 +663,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt return FAILED; } - Status hccl_ret = OpskernelBuilderManager::Instance().ProfStart(module); + Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStart(module); if (hccl_ret != SUCCESS) { GELOGE(FAILED, "Hccl profiler start failed."); return FAILED; @@ -709,7 +709,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); return FAILED; } - Status hccl_ret = OpskernelBuilderManager::Instance().ProfStop(module); + Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStop(module); if (hccl_ret != SUCCESS) { GELOGE(FAILED, "Hccl profiler stop failed."); return FAILED; @@ -796,7 +796,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUn #ifdef DAVINCI_SUPPORT_PROFILING if (prof_cb_.msprofReporterCallback == nullptr) { GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); - return ge::PARAM_INVALID; + return; } int32_t cb_ret = prof_cb_.msprofReporterCallback( static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), @@ -827,7 +827,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpP GELOGI("Bp Fp have been initialized in env or options"); fp_point = fp_point_; bp_point = bp_point_; - GELOGI("Bp Fp have been initailized in env or options, bp_point: %s, fp_point: %s", bp_point.c_str(), fp_point.c_str()); + GELOGI("Bp Fp have been initialized in env or options. bp_point: %s, fp_point: %s", bp_point.c_str(), fp_point.c_str()); return; } // ProfApi mode and training trace is set diff --git a/ge/graph/load/new_model_manager/davinci_model.cc b/ge/graph/load/new_model_manager/davinci_model.cc index 18435571..bade866f 100755 --- a/ge/graph/load/new_model_manager/davinci_model.cc +++ b/ge/graph/load/new_model_manager/davinci_model.cc @@ -2251,7 +2251,7 @@ inline int64_t SumSize(const vector &size_list) { Status DavinciModel::SinkModelProfile() { // profiling plugin must be registered auto &prof_mgr = ProfilingManager::Instance(); - ReporterData reporter_data; + ReporterData reporter_data{}; // report model data tag name std::string tag_name; tag_name.append("model_load_info_").append(std::to_string(this->Id())); @@ -2424,7 +2424,7 @@ Status DavinciModel::SinkModelProfile() { Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { // profiling plugin must be registered auto &prof_mgr = ProfilingManager::Instance(); - ReporterData reporter_data; + ReporterData reporter_data{}; // report model data tag name std::string tag_name; tag_name.append("model_time_info_") From 2e87815ba5382d6b3af45a5f39215a431aac39f4 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 19:56:58 +0800 Subject: [PATCH 05/24] Profiling AR version 5 --- ge/init/gelib.cc | 6 +++- .../ops_kernel_builder_manager.cc | 28 +++++++++++++++++++ .../ops_kernel_builder_manager.h | 6 ++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/ge/init/gelib.cc b/ge/init/gelib.cc index 92700179..20859c07 100755 --- a/ge/init/gelib.cc +++ b/ge/init/gelib.cc @@ -145,6 +145,10 @@ Status GELib::InnerInitialize(const map &options) { return initOpsBuilderStatus; } + if (is_train_mode_ || (options_.device_id != kDefaultDeviceIdForInfer)) { + GE_CHK_RT_RET(rtSetDevice(options.device_id)); + } + GELOGI("sessionManager initial."); GE_TIMESTAMP_START(SessionManagerInitialize); Status initSmStatus = sessionManager_.Initialize(options); @@ -338,7 +342,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status GELib::InitSystemWithOpt // set device id GELOGI("set logical device id:%u", options.device_id); GetContext().SetCtxDeviceId(static_cast(options.device_id)); - GE_CHK_RT_RET(rtSetDevice(options.device_id)); + // GE_CHK_RT_RET(rtSetDevice(options.device_id)); // In the scenario that the automatic add fusion is set, but there is no cleanaddr operator, // maybe need to check it diff --git a/ge/opskernel_manager/ops_kernel_builder_manager.cc b/ge/opskernel_manager/ops_kernel_builder_manager.cc index e0001fcd..30c3ffcc 100644 --- a/ge/opskernel_manager/ops_kernel_builder_manager.cc +++ b/ge/opskernel_manager/ops_kernel_builder_manager.cc @@ -167,4 +167,32 @@ Status OpsKernelBuilderManager::GenerateTask(const Node &node, GELOGD("Done invoking GenerateTask successfully"); return SUCCESS; } + +Status OpsKernelBuilderManager::RegProfReporterCallBack(void *func) const { + GE_CHECK_NOTNULL(func); + for (auto it = ops_kernel_builders_.begin(); it != ops_kernel_builders_.end(); it++) { + GE_CHK_STATUS_RET(it->second->RegProfReporterCallBack(func), + "Failed to invoke RegProfReporterCallBack."); + } + GELOGD("Done invoking RegProfReporterCallBack successfully"); + return SUCCESS; +} + +Status OpsKernelBuilderManager::ProfStart(uint64_t prof_config) const { + for (auto it = ops_kernel_builders_.begin(); it != ops_kernel_builders_.end(); it++) { + GE_CHK_STATUS_RET(it->second->ProfStart(prof_config), + "Failed to invoke ProfStart."); + } + GELOGD("Done invoking ProfStart successfully"); + return SUCCESS; +} + +Status OpsKernelBuilderManager::ProfStop((uint64_t prof_config) const { + for (auto it = ops_kernel_builders_.begin(); it != ops_kernel_builders_.end(); it++) { + GE_CHK_STATUS_RET(it->second->ProfStop(prof_config), + "Failed to invoke ProfStop."); + } + GELOGD("Done invoking ProfStop successfully"); + return SUCCESS; +} } // namespace ge \ No newline at end of file diff --git a/ge/opskernel_manager/ops_kernel_builder_manager.h b/ge/opskernel_manager/ops_kernel_builder_manager.h index 7a95ddfa..bcd05fe0 100644 --- a/ge/opskernel_manager/ops_kernel_builder_manager.h +++ b/ge/opskernel_manager/ops_kernel_builder_manager.h @@ -46,6 +46,12 @@ class OpsKernelBuilderManager { Status GenerateTask(const Node &node, RunContext &context, std::vector &tasks) const; + Status RegProfReporterCallBack(void *func) const; + + Status ProfStart(uint64_t prof_config) const; + + Status ProfStop((uint64_t prof_config) const; + private: OpsKernelBuilderManager() = default; static Status GetLibPaths(const std::map &options, std::string &lib_paths); From bbae4fcf946eb39259d5e7512fdf835742c1819b Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 20:02:18 +0800 Subject: [PATCH 06/24] Profiling remove ge_prof.h/.c --- ge/client/ge_prof.cc | 369 -------------------------------------- inc/external/ge/ge_prof.h | 102 ----------- 2 files changed, 471 deletions(-) delete mode 100644 ge/client/ge_prof.cc delete mode 100644 inc/external/ge/ge_prof.h diff --git a/ge/client/ge_prof.cc b/ge/client/ge_prof.cc deleted file mode 100644 index ede38430..00000000 --- a/ge/client/ge_prof.cc +++ /dev/null @@ -1,369 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ge/ge_prof.h" -#include "ge/ge_api.h" -#include "init/gelib.h" -#include "common/debug/log.h" -#include "framework/common/debug/ge_log.h" -#include "common/profiling/profiling_manager.h" -#include "graph/load/graph_loader.h" -#include "toolchain/prof_acl_api.h" - -using std::map; -using std::string; -using std::vector; - -namespace { -const uint32_t kMaxDeviceNum = 64; -const uint32_t kDeviceListIndex = 3; -const std::string kProfilingInit = "prof_init"; -const std::string kProfilingFinalize = "prof_finalize"; -const std::string kProfilingStart = "prof_start"; -const std::string kProfilingStop = "prof_stop"; -const std::string kDeviceNums = "devNums"; -const std::string kDeviceIdList = "devIdList"; -const std::string kAicoreMetrics = "aicoreMetrics"; - -const std::map kProfAicoreMetricsToString = { - {ge::kAicoreArithmaticThroughput, "AICORE_ARITHMATIC_THROUGHPUT"}, - {ge::kAicorePipeline, "AICORE_PIPELINE"}, - {ge::kAicoreSynchronization, "AICORE_SYNCHRONIZATION"}, - {ge::kAicoreMemory, "AICORE_MEMORY"}, - {ge::kAicoreInternalMemory, "AICORE_INTERNAL_MEMORY"}, - {ge::kAicoreStall, "AICORE_STALL"}}; -} // namespace - -static bool g_graph_prof_init_ = false; -static std::mutex g_prof_mutex_; - -namespace ge { -struct aclgrphProfConfig { - ProfConfig config; -}; - -Status aclgrphProfInit(const char *profiler_path, uint32_t length) { - GELOGT(TRACE_INIT, "Graph prof init start"); - - std::shared_ptr instance_ptr = ge::GELib::GetInstance(); - if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); - return FAILED; - } - - std::lock_guard lock(g_prof_mutex_); - if (g_graph_prof_init_) { - GELOGW("Multi graph profiling initializations."); - return GE_PROF_MULTI_INIT; - } - - Status ret = CheckPath(profiler_path, length); - if (ret != SUCCESS) { - GELOGE(ret, "Profiling config path is invalid."); - return ret; - } - // if command mode is set, just return - if (ProfilingManager::Instance().ProfilingOn()) { - GELOGW("Graph prof init failed, cause profiling command pattern is running."); - return GE_PROF_MODE_CONFLICT; - } - - ret = ProfInit(profiler_path); - if (ret != SUCCESS) { - GELOGE(ret, "ProfInit init fail"); - return ret; - } - - GraphLoader graph_loader; - Command command; - command.cmd_params.clear(); - command.cmd_type = kProfilingInit; - command.module_index = PROF_MODEL_LOAD; - ret = graph_loader.CommandHandle(command); - if (ret != SUCCESS) { - GELOGE(ret, "Handle profiling command %s failed, config = %s", kProfilingInit.c_str(), profiler_path); - return ret; - } - if (!g_graph_prof_init_) { - g_graph_prof_init_ = true; - GELOGI("Profiling init successfully."); - } - - GELOGI("Successfully execute GraphProfInit."); - return SUCCESS; -} - -Status aclgrphProfFinalize() { - std::shared_ptr instance_ptr = ge::GELib::GetInstance(); - if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); - return FAILED; - } - std::lock_guard lock(g_prof_mutex_); - // if command mode is set, just return - if (ProfilingManager::Instance().ProfilingOn()) { - GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); - return GE_PROF_MODE_CONFLICT; - } - - if (!g_graph_prof_init_) { - GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); - return GE_PROF_NOT_INIT; - } - GraphLoader graph_loader; - Command command; - command.cmd_params.clear(); - command.cmd_type = kProfilingFinalize; - Status ret = graph_loader.CommandHandle(command); - if (ret != SUCCESS) { - GELOGE(ret, "Handle profiling command %s failed.", kProfilingFinalize.c_str()); - return ret; - } - - ret = ProfFinalize(); - if (ret != SUCCESS) { - GELOGE(ret, "Finalize profiling failed, result = %d", ret); - } - - if (ret == SUCCESS) { - g_graph_prof_init_ = false; - GELOGI("Successfully execute GraphProfFinalize."); - } - return ret; -} - -bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector &prof_config_params) { - prof_config_params.clear(); - prof_config_params.emplace_back(kDeviceNums); - prof_config_params.emplace_back(std::to_string(profiler_config->config.devNums)); - prof_config_params.emplace_back(kDeviceIdList); - std::string devID = ""; - if (profiler_config->config.devNums == 0) { - GELOGW("The device num is invalid."); - return false; - } - for (uint32_t i = 0; i < profiler_config->config.devNums; i++) { - devID.append(std::to_string(profiler_config->config.devIdList[i])); - if (i != profiler_config->config.devNums - 1) { - devID.append(","); - } - } - - prof_config_params.push_back(devID); - prof_config_params.push_back(kAicoreMetrics); - auto iter = - kProfAicoreMetricsToString.find(static_cast(profiler_config->config.aicoreMetrics)); - if (iter == kProfAicoreMetricsToString.end()) { - GELOGW("The prof aicore metrics is invalid."); - return false; - } - prof_config_params.push_back(iter->second); - return true; -} - -bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { - if (deviceid_list == nullptr) { - GELOGE(PARAM_INVALID, "deviceIdList is nullptr"); - return false; - } - if (device_nums == 0 || device_nums > kMaxDeviceNum) { - GELOGE(PARAM_INVALID, "The device nums is invalid."); - return false; - } - - // real device num - int32_t dev_count = 0; - rtError_t rt_err = rtGetDeviceCount(&dev_count); - if (rt_err != RT_ERROR_NONE) { - GELOGE(INTERNAL_ERROR, "Get the Device count fail."); - return false; - } - - if (device_nums > static_cast(dev_count)) { - GELOGE(PARAM_INVALID, "Device num(%u) is not in range 1 ~ %d.", device_nums, dev_count); - return false; - } - - std::unordered_set record; - for (size_t i = 0; i < device_nums; ++i) { - uint32_t dev_id = deviceid_list[i]; - if (dev_id >= static_cast(dev_count)) { - GELOGE(PARAM_INVALID, "Device id %u is not in range 0 ~ %d(exclude %d)", dev_id, dev_count, dev_count); - return false; - } - if (record.count(dev_id) > 0) { - GELOGE(PARAM_INVALID, "Device id %u is duplicatedly set", dev_id); - return false; - } - record.insert(dev_id); - } - return true; -} - -aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t device_nums, - ProfilingAicoreMetrics aicore_metrics, ProfAicoreEvents *aicore_events, - uint64_t data_type_config) { - if (!isProfConfigValid(deviceid_list, device_nums)) { - return nullptr; - } - aclgrphProfConfig *config = new (std::nothrow) aclgrphProfConfig(); - if (config == nullptr) { - GELOGE(INTERNAL_ERROR, "new aclgrphProfConfig fail"); - return nullptr; - } - config->config.devNums = device_nums; - if (memcpy_s(config->config.devIdList, sizeof(config->config.devIdList), deviceid_list, - device_nums * sizeof(uint32_t)) != EOK) { - GELOGE(INTERNAL_ERROR, "copy devID failed. size = %u", device_nums); - delete config; - return nullptr; - } - - config->config.aicoreMetrics = static_cast(aicore_metrics); - config->config.dataTypeConfig = data_type_config; - GELOGI("Successfully create prof config."); - return config; -} - -Status aclgrphProfDestroyConfig(aclgrphProfConfig *profiler_config) { - if (profiler_config == nullptr) { - GELOGE(PARAM_INVALID, "destroy profilerConfig failed, profilerConfig must not be nullptr"); - return PARAM_INVALID; - } - - delete profiler_config; - GELOGI("Successfully destroy prof config."); - return SUCCESS; -} - -Status aclgrphProfStart(aclgrphProfConfig *profiler_config) { - if (profiler_config == nullptr) { - GELOGE(PARAM_INVALID, "aclgrphProfConfig is invalid."); - return FAILED; - } - std::shared_ptr instance_ptr = ge::GELib::GetInstance(); - if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); - return FAILED; - } - - std::lock_guard lock(g_prof_mutex_); - // if command mode is set, just return - if (ProfilingManager::Instance().ProfilingOn()) { - GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); - return GE_PROF_MODE_CONFLICT; - } - if (!g_graph_prof_init_) { - GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); - return GE_PROF_NOT_INIT; - } - - Status ret = ProfStartProfiling(&profiler_config->config); - if (ret != SUCCESS) { - GELOGE(ret, "Start profiling failed, prof result = %d", ret); - return FAILED; - } - - std::vector prof_params; - if (!TransProfConfigToParam(profiler_config, prof_params)) { - GELOGE(PARAM_INVALID, "Transfer profilerConfig to string vector failed"); - return PARAM_INVALID; - } - - GraphLoader graph_loader; - Command command; - command.cmd_params.clear(); - command.cmd_type = kProfilingStart; - command.cmd_params = prof_params; - command.module_index = profiler_config->config.dataTypeConfig; - GELOGI("Profiling will start, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), - prof_params[kDeviceListIndex].c_str(), command.module_index); - ret = graph_loader.CommandHandle(command); - if (ret != SUCCESS) { - GELOGE(ret, "Handle profiling command failed"); - return FAILED; - } - - GELOGI("Successfully execute GraphProfStartProfiling."); - - return SUCCESS; -} - -Status aclgrphProfStop(aclgrphProfConfig *profiler_config) { - if (profiler_config == nullptr) { - GELOGE(PARAM_INVALID, "aclgrphProfConfig is invalid."); - return FAILED; - } - std::shared_ptr instance_ptr = ge::GELib::GetInstance(); - if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); - return FAILED; - } - - std::lock_guard lock(g_prof_mutex_); - // if command mode is set, just return - if (ProfilingManager::Instance().ProfilingOn()) { - GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); - return GE_PROF_MODE_CONFLICT; - } - if (!g_graph_prof_init_) { - GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); - return GE_PROF_NOT_INIT; - } - - for (uint32_t i = 0; i < profiler_config->config.devNums; i++) { - uint64_t data_type_config; - Status status = ProfGetDataTypeConfig(profiler_config->config.devIdList[i], data_type_config); - if (status != SUCCESS) { - GELOGE(status, "Prof get data type config failed, prof result = %d", status); - return status; - } - if (data_type_config != profiler_config->config.dataTypeConfig) { - GELOGE(FAILED, "data type config verify failed"); - return FAILED; - } - } - - std::vector prof_params; - if (!TransProfConfigToParam(profiler_config, prof_params)) { - GELOGE(PARAM_INVALID, "Transfer profilerConfig to string vector failed"); - return PARAM_INVALID; - } - - GraphLoader graph_loader; - Command command; - command.cmd_params.clear(); - command.cmd_type = kProfilingStop; - command.cmd_params = prof_params; - command.module_index = profiler_config->config.dataTypeConfig; - GELOGI("Profiling will stop, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), - prof_params[kDeviceListIndex].c_str(), command.module_index); - Status ret = graph_loader.CommandHandle(command); - if (ret != SUCCESS) { - GELOGE(ret, "Handle profiling command failed"); - return FAILED; - } - - ret = ProfStopProfiling(&profiler_config->config); - if (ret != SUCCESS) { - GELOGE(ret, "Stop profiling failed, prof result = %d", ret); - return ret; - } - - GELOGI("Successfully execute GraphProfStopProfiling."); - return SUCCESS; -} -} // namespace ge diff --git a/inc/external/ge/ge_prof.h b/inc/external/ge/ge_prof.h deleted file mode 100644 index 658cea76..00000000 --- a/inc/external/ge/ge_prof.h +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright 2019-2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef INC_EXTERNAL_GE_GE_PROF_H_ -#define INC_EXTERNAL_GE_GE_PROF_H_ - -#include -#include -#include - -#include "ge/ge_api_error_codes.h" - -namespace ge { -enum ProfDataTypeConfig { - kProfTaskTime = 0x0002, - kProfAiCoreMetrics = 0x0004, - kProfAicpuTrace = 0x0008, - kProfTrainingTrace = 0x0800, - kProfHcclTrace = 0x1000 -}; - -enum ProfilingAicoreMetrics { - kAicoreArithmaticThroughput = 0, - kAicorePipeline = 1, - kAicoreSynchronization = 2, - kAicoreMemory = 3, - kAicoreInternalMemory = 4, - kAicoreStall = 5 -}; - -typedef struct ProfAicoreEvents ProfAicoreEvents; -typedef struct aclgrphProfConfig aclgrphProfConfig; - -/// -/// @ingroup AscendCL -/// @brief Initialize the profiling and set profiling configuration path -/// @param [in] profiler_path: configuration path of profiling -/// @param [in] length: length of configuration path -/// @return Status result of function -/// -Status aclgrphProfInit(const char *profiler_path, uint32_t length); - -/// -/// @ingroup AscendCL -/// @brief Finalize profiling -/// @return Status result of function -/// -Status aclgrphProfFinalize(); - -/// -/// @ingroup AscendCL -/// @brief Create data of type aclgrphProfConfig -/// @param [in] deviceid_list: device id list -/// @param [in] device_nums: device numbers -/// @param [in] aicore_metrics: type of aicore metrics -/// @param [in] aicore_events: pointer to aicore events be reserved, only support NULL now -/// @param [in] data_type_config: modules need profiling -/// @return Status result of function -/// -aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t device_nums, - ProfilingAicoreMetrics aicore_metrics, ProfAicoreEvents *aicore_events, - uint64_t data_type_config); - -/// -/// @ingroup AscendCL -/// @brief Destroy data of type aclgrphProfConfig -/// @param [in] profiler_config: config of profiling -/// @return Status result of function -/// -Status aclgrphProfDestroyConfig(aclgrphProfConfig *profiler_config); - -/// -/// @ingroup AscendCL -/// @brief Start profiling of modules which is configured by profiler config -/// @param [in] profiler_config: config of profiling -/// @return Status result of function -/// -Status aclgrphProfStart(aclgrphProfConfig *profiler_config); - -/// -/// @ingroup AscendCL -/// @brief Stop profiling of modules which is configured by profiler config -/// @param [in] profiler_config: config of profiling -/// @return Status result of function -/// -Status aclgrphProfStop(aclgrphProfConfig *profiler_config); -} // namespace ge - -#endif // INC_EXTERNAL_GE_GE_PROF_H_ From d54a6ea9c9f75a9dd7a569dde6febc504bb0de44 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 20:15:11 +0800 Subject: [PATCH 07/24] Profiling AR version 6 --- ge/common/profiling/profiling_manager.cc | 3 +-- inc/framework/common/profiling/ge_profiling.h | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index 222cde6a..1a0a1b2f 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -192,7 +192,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf // stop profiling if (prof_cb_.msprofCtrlCallback == nullptr) { GELOGE(ge::PARAM_INVALID, "MsprofCtrlCallback callback is nullptr."); - return ge::PARAM_INVALID; + return; } int32_t cb_ret = prof_cb_.msprofCtrlCallback(static_cast(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), nullptr, 0); @@ -824,7 +824,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpP std::string &fp_point, std::string &bp_point) { // Env or options mode, fp_point_/bp_point_ have initiliazed on profiling init if (!fp_point_.empty() && !bp_point_.empty()) { - GELOGI("Bp Fp have been initialized in env or options"); fp_point = fp_point_; bp_point = bp_point_; GELOGI("Bp Fp have been initialized in env or options. bp_point: %s, fp_point: %s", bp_point.c_str(), fp_point.c_str()); diff --git a/inc/framework/common/profiling/ge_profiling.h b/inc/framework/common/profiling/ge_profiling.h index 65a0eebb..10b53d6d 100644 --- a/inc/framework/common/profiling/ge_profiling.h +++ b/inc/framework/common/profiling/ge_profiling.h @@ -31,11 +31,11 @@ enum ProfCommandHandleType { }; struct ProfCommandHandleData { - unit64_t profSwitch; + uint64_t profSwitch; uint32_t devNums; // length of device id list uint32_t devIdList[MAX_DEV_NUM]; uint32_t modelId; -} +}; ge::Status RegProfCtrlCallback(MsprofCtrlCallback func); ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func); From b6ee361521fdea9d9b9b625db2656262e7d8fdf6 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 20:30:57 +0800 Subject: [PATCH 08/24] Profiling AR version 7 --- ge/common/profiling/profiling_manager.cc | 2 ++ ge/graph/load/new_model_manager/davinci_model.cc | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index 1a0a1b2f..5419c8e8 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -588,6 +588,7 @@ Status ProfilingManager::ProfParseDeviceId(const std::map return FAILED; } catch (std::out_of_range &) { GELOGE(FAILED, "Device num: %s is out of range.", iter->second.c_str()); + return FAILED; } catch (...) { GELOGE(FAILED, "Device num: %s cannot change to int.", iter->second.c_str()); return FAILED; diff --git a/ge/graph/load/new_model_manager/davinci_model.cc b/ge/graph/load/new_model_manager/davinci_model.cc index bade866f..bc755e07 100755 --- a/ge/graph/load/new_model_manager/davinci_model.cc +++ b/ge/graph/load/new_model_manager/davinci_model.cc @@ -2819,15 +2819,20 @@ void *DavinciModel::Run(DavinciModel *model) { GE_TIMESTAMP_START(rtStreamSynchronize); GELOGI("rtStreamSynchronize start."); rt_ret = rtStreamSynchronize(model->rt_model_stream_); - if (rt_ret == RT_ERROR_END_OF_SEQUENCE) { + if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { seq_end_flag = true; } - GE_IF_BOOL_EXEC( + if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { + GELOGI("The model with multiple datasets aborts normally."); + } else { + GE_IF_BOOL_EXEC( rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); (void)model->ReturnResult(current_data.index, false, seq_end_flag, data_wrapper->GetOutput()); // [No need to check value] CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); continue); + } + GELOGI("rtStreamSynchronize end."); GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); From 8ab72ebdb9d71eb29be9682e594b842d5c0c2d55 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 22:28:45 +0800 Subject: [PATCH 09/24] Profiling version 8 --- ge/opskernel_manager/ops_kernel_builder_manager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/opskernel_manager/ops_kernel_builder_manager.h b/ge/opskernel_manager/ops_kernel_builder_manager.h index bcd05fe0..a2d3e565 100644 --- a/ge/opskernel_manager/ops_kernel_builder_manager.h +++ b/ge/opskernel_manager/ops_kernel_builder_manager.h @@ -50,7 +50,7 @@ class OpsKernelBuilderManager { Status ProfStart(uint64_t prof_config) const; - Status ProfStop((uint64_t prof_config) const; + Status ProfStop(uint64_t prof_config) const; private: OpsKernelBuilderManager() = default; From 95676781681cfd1611646afb8cd183a60d5f11a2 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 22:37:29 +0800 Subject: [PATCH 10/24] Profiling AR version 9 --- ge/common/profiling/ge_profiling.cc | 2 +- ge/common/profiling/profiling_manager.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index b6708f7a..b06a8448 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -151,7 +151,7 @@ ge::Status RegProfReporterCallback(MsprofReporterCallback func) { return rt_ret; } // Pass MsprofReporterCallback to hccl in opskernel so initialize - rt_ret = OpsKernelBuilderManager::Instance().RegProfReporterCallBack(func); + rt_ret = ge::OpsKernelBuilderManager::Instance().RegProfReporterCallBack(func); if (rt_ret != ge::SUCCESS) { GELOGE(rt_ret, "Pass MsprofReporterCallback to hccl failed."); return rt_ret; diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index 5419c8e8..5cb08dd0 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -142,7 +142,7 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt ge::Status ProfilingManager::ParseOptions(const std::string &options) { if (options.empty()) { - GELOGE(ge::PARAM_INVALID, "Profiling options is empty.") + GELOGE(ge::PARAM_INVALID, "Profiling options is empty."); return ge::PARAM_INVALID; } try { From 216a6f3b6a887ce3e800d4344ddacb4493ff3c85 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 22:47:28 +0800 Subject: [PATCH 11/24] Profiling AR version 10 --- ge/common/profiling/ge_profiling.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index b06a8448..67e6b619 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -151,7 +151,7 @@ ge::Status RegProfReporterCallback(MsprofReporterCallback func) { return rt_ret; } // Pass MsprofReporterCallback to hccl in opskernel so initialize - rt_ret = ge::OpsKernelBuilderManager::Instance().RegProfReporterCallBack(func); + rt_ret = ge::OpsKernelBuilderManager::Instance().RegProfReporterCallBack(reinterpret_cast(func)); if (rt_ret != ge::SUCCESS) { GELOGE(rt_ret, "Pass MsprofReporterCallback to hccl failed."); return rt_ret; From 4f898216bd2b269d0be6fb956510a225d5ed5e8d Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Thu, 10 Dec 2020 22:54:40 +0800 Subject: [PATCH 12/24] Profiling AR version 11 --- ge/opskernel_manager/ops_kernel_builder_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/opskernel_manager/ops_kernel_builder_manager.cc b/ge/opskernel_manager/ops_kernel_builder_manager.cc index 30c3ffcc..4361df44 100644 --- a/ge/opskernel_manager/ops_kernel_builder_manager.cc +++ b/ge/opskernel_manager/ops_kernel_builder_manager.cc @@ -187,7 +187,7 @@ Status OpsKernelBuilderManager::ProfStart(uint64_t prof_config) const { return SUCCESS; } -Status OpsKernelBuilderManager::ProfStop((uint64_t prof_config) const { +Status OpsKernelBuilderManager::ProfStop(uint64_t prof_config) const { for (auto it = ops_kernel_builders_.begin(); it != ops_kernel_builders_.end(); it++) { GE_CHK_STATUS_RET(it->second->ProfStop(prof_config), "Failed to invoke ProfStop."); From c3be30b9ce1b422a6b536d8790fc689ed8aaebb2 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 11 Dec 2020 10:58:17 +0800 Subject: [PATCH 13/24] Profiling AR VERSION11 remove task_id --- ge/common/profiling/profiling_manager.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index 5cb08dd0..df36d5f1 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -133,9 +133,6 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt GELOGE(INTERNAL_ERROR, "copy job_id failed."); return INTERNAL_ERROR; } - - // get traceId from options - prof_conf.traceId = GetContext().TraceId(); #endif return ge::SUCCESS; } From 46179fe5e2721ca30a8ea8a6e787d7f3c346e043 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 11 Dec 2020 14:12:13 +0800 Subject: [PATCH 14/24] update deviceid --- ge/init/gelib.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/init/gelib.cc b/ge/init/gelib.cc index 20859c07..03028e53 100755 --- a/ge/init/gelib.cc +++ b/ge/init/gelib.cc @@ -146,7 +146,7 @@ Status GELib::InnerInitialize(const map &options) { } if (is_train_mode_ || (options_.device_id != kDefaultDeviceIdForInfer)) { - GE_CHK_RT_RET(rtSetDevice(options.device_id)); + GE_CHK_RT_RET(rtSetDevice(options_.device_id)); } GELOGI("sessionManager initial."); From 77ff5be0a0e86d61514d78753b06477c5a3b91dd Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 11 Dec 2020 20:24:51 +0800 Subject: [PATCH 15/24] Profiling AR version11 --- ge/common/profiling/profiling_manager.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index df36d5f1..3adf9065 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -60,7 +60,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In subscribe_count_ = 0; GELOGI("ProfilingManager::Init job_id:%s", options.job_id.c_str()); - struct MsprofGeOptions prof_conf = { 0 }; + struct MsprofGeOptions prof_conf = {{ 0 }}; Status ret = InitFromOptions(options, prof_conf); if (ret != SUCCESS) { GELOGE(ret, "Failed to init profiling."); @@ -108,8 +108,7 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt (void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); (void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX); // The env is invalid - if ((env_profiling_mode == nullptr) || (strcmp("true", env_profiling_mode) != 0) - || (strcmp(prof_conf.options, "\0") == 0)) { + if ((strcmp("true", env_profiling_mode) != 0) || (strcmp(prof_conf.options, "\0") == 0)) { return SUCCESS; } // enable profiling by env From 77c21e71b755f9dafccd5eb69673e0eac28f226c Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 11 Dec 2020 22:09:12 +0800 Subject: [PATCH 16/24] Profiling AR version12 --- ge/common/profiling/profiling_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index 3adf9065..457ee464 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -98,7 +98,7 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt if (options.profiling_mode == "1" && !options.profiling_options.empty()) { // enable profiling by ge option if (memcpy_s(prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX, options.profiling_options.c_str(), - sizeof(options.profiling_options.c_str())) != EOK) { + options.profiling_options.size()) != EOK) { GELOGE(INTERNAL_ERROR, "copy profiling_options failed."); return INTERNAL_ERROR; } From e435b4ea0317c3707c9729555d112e34813bd004 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Fri, 11 Dec 2020 22:29:12 +0800 Subject: [PATCH 17/24] Profiling AR version14 --- ge/common/profiling/profiling_manager.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index 457ee464..c8f728c2 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -103,7 +103,8 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt return INTERNAL_ERROR; } is_execute_profiling_ = true; - GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), prof_conf.options); + GELOGI("The profiling in options is %s, %s. origin option: %s", options.profiling_mode.c_str(), + prof_conf.options, options.profiling_options.c_str()); } else { (void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); (void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX); From 71d2fb4d58f1396ea028fe8985a873755fbc7f16 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sat, 12 Dec 2020 11:38:08 +0800 Subject: [PATCH 18/24] Remove pass reporter callback to hccl --- ge/common/profiling/ge_profiling.cc | 8 +---- ge/common/profiling/profiling_manager.cc | 23 --------------- ge/init/gelib.cc | 6 +--- .../ops_kernel_builder_manager.cc | 29 +------------------ .../ops_kernel_builder_manager.h | 6 ---- 5 files changed, 3 insertions(+), 69 deletions(-) diff --git a/ge/common/profiling/ge_profiling.cc b/ge/common/profiling/ge_profiling.cc index 67e6b619..640f77a1 100644 --- a/ge/common/profiling/ge_profiling.cc +++ b/ge/common/profiling/ge_profiling.cc @@ -22,7 +22,6 @@ #include "graph/load/graph_loader.h" #include "init/gelib.h" #include "framework/common/ge_inner_error_codes.h" -#include "opskernel_manager/ops_kernel_builder_manager.h" namespace { const uint32_t kDeviceListIndex = 3; @@ -150,12 +149,7 @@ ge::Status RegProfReporterCallback(MsprofReporterCallback func) { GELOGE(rt_ret, "Pass MsprofReporterCallback to runtime failed!!"); return rt_ret; } - // Pass MsprofReporterCallback to hccl in opskernel so initialize - rt_ret = ge::OpsKernelBuilderManager::Instance().RegProfReporterCallBack(reinterpret_cast(func)); - if (rt_ret != ge::SUCCESS) { - GELOGE(rt_ret, "Pass MsprofReporterCallback to hccl failed."); - return rt_ret; - } + // Pass MsprofReporterCallback to hccl } return ge::SUCCESS; } diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index c8f728c2..456cb0a4 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -22,7 +22,6 @@ #include "graph/ge_context.h" #include "runtime/base.h" #include "graph/load/new_model_manager/davinci_model.h" -#include "opskernel_manager/ops_kernel_builder_manager.h" namespace { const char *const kTrainingTrace = "training_trace"; @@ -496,11 +495,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn GELOGE(FAILED, "Runtime profiler start failed."); return FAILED; } - Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStart(model_load_mask); - if (hccl_ret != SUCCESS) { - GELOGE(FAILED, "Hccl profiler start failed."); - return FAILED; - } is_load_profiling_ = true; GELOGI("Prof init: model load profiling on."); } @@ -530,12 +524,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi GELOGE(FAILED, "Runtime profiler stop failed."); return FAILED; } - - Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStop(PROF_MODEL_LOAD_MASK); - if (hccl_ret != SUCCESS) { - GELOGE(FAILED, "Hccl profiler stop failed."); - return FAILED; - } for (auto device_id_module : device_id_module_map_) { if (device_id_module.second != 0) { uint32_t device_id = static_cast(device_id_module.first); @@ -661,12 +649,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGE(FAILED, "Runtime profiler config proc failed."); return FAILED; } - - Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStart(module); - if (hccl_ret != SUCCESS) { - GELOGE(FAILED, "Hccl profiler start failed."); - return FAILED; - } if ((module & PROF_MODEL_EXECUTE_MASK) == PROF_MODEL_EXECUTE_MASK) { for (int32_t i = 0; i < device_num; i++) { if (std::find(device_id_.begin(), device_id_.end(), device_list[i]) == device_id_.end()) { @@ -708,11 +690,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); return FAILED; } - Status hccl_ret = OpsKernelBuilderManager::Instance().ProfStop(module); - if (hccl_ret != SUCCESS) { - GELOGE(FAILED, "Hccl profiler stop failed."); - return FAILED; - } uint64_t execute_model_mask = module & PROF_MODEL_EXECUTE_MASK; if (execute_model_mask == PROF_MODEL_EXECUTE_MASK) { for (int32_t i = 0; i < device_num; i++) { diff --git a/ge/init/gelib.cc b/ge/init/gelib.cc index 03028e53..92700179 100755 --- a/ge/init/gelib.cc +++ b/ge/init/gelib.cc @@ -145,10 +145,6 @@ Status GELib::InnerInitialize(const map &options) { return initOpsBuilderStatus; } - if (is_train_mode_ || (options_.device_id != kDefaultDeviceIdForInfer)) { - GE_CHK_RT_RET(rtSetDevice(options_.device_id)); - } - GELOGI("sessionManager initial."); GE_TIMESTAMP_START(SessionManagerInitialize); Status initSmStatus = sessionManager_.Initialize(options); @@ -342,7 +338,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status GELib::InitSystemWithOpt // set device id GELOGI("set logical device id:%u", options.device_id); GetContext().SetCtxDeviceId(static_cast(options.device_id)); - // GE_CHK_RT_RET(rtSetDevice(options.device_id)); + GE_CHK_RT_RET(rtSetDevice(options.device_id)); // In the scenario that the automatic add fusion is set, but there is no cleanaddr operator, // maybe need to check it diff --git a/ge/opskernel_manager/ops_kernel_builder_manager.cc b/ge/opskernel_manager/ops_kernel_builder_manager.cc index 4361df44..37bdcf7a 100644 --- a/ge/opskernel_manager/ops_kernel_builder_manager.cc +++ b/ge/opskernel_manager/ops_kernel_builder_manager.cc @@ -168,31 +168,4 @@ Status OpsKernelBuilderManager::GenerateTask(const Node &node, return SUCCESS; } -Status OpsKernelBuilderManager::RegProfReporterCallBack(void *func) const { - GE_CHECK_NOTNULL(func); - for (auto it = ops_kernel_builders_.begin(); it != ops_kernel_builders_.end(); it++) { - GE_CHK_STATUS_RET(it->second->RegProfReporterCallBack(func), - "Failed to invoke RegProfReporterCallBack."); - } - GELOGD("Done invoking RegProfReporterCallBack successfully"); - return SUCCESS; -} - -Status OpsKernelBuilderManager::ProfStart(uint64_t prof_config) const { - for (auto it = ops_kernel_builders_.begin(); it != ops_kernel_builders_.end(); it++) { - GE_CHK_STATUS_RET(it->second->ProfStart(prof_config), - "Failed to invoke ProfStart."); - } - GELOGD("Done invoking ProfStart successfully"); - return SUCCESS; -} - -Status OpsKernelBuilderManager::ProfStop(uint64_t prof_config) const { - for (auto it = ops_kernel_builders_.begin(); it != ops_kernel_builders_.end(); it++) { - GE_CHK_STATUS_RET(it->second->ProfStop(prof_config), - "Failed to invoke ProfStop."); - } - GELOGD("Done invoking ProfStop successfully"); - return SUCCESS; -} -} // namespace ge \ No newline at end of file +} // namespace ge diff --git a/ge/opskernel_manager/ops_kernel_builder_manager.h b/ge/opskernel_manager/ops_kernel_builder_manager.h index a2d3e565..7a95ddfa 100644 --- a/ge/opskernel_manager/ops_kernel_builder_manager.h +++ b/ge/opskernel_manager/ops_kernel_builder_manager.h @@ -46,12 +46,6 @@ class OpsKernelBuilderManager { Status GenerateTask(const Node &node, RunContext &context, std::vector &tasks) const; - Status RegProfReporterCallBack(void *func) const; - - Status ProfStart(uint64_t prof_config) const; - - Status ProfStop(uint64_t prof_config) const; - private: OpsKernelBuilderManager() = default; static Status GetLibPaths(const std::map &options, std::string &lib_paths); From 1f2d63f2b9d81403aaa650bbb07271a421ab69ff Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sat, 12 Dec 2020 17:07:59 +0800 Subject: [PATCH 19/24] remove all msprof, changed to static msprof --- CMakeLists.txt | 14 +++++--------- ge/CMakeLists.txt | 2 +- ge/client/module.mk | 10 ++++++---- ge/executor/CMakeLists.txt | 1 - ge/executor/module.mk | 4 ---- ge/ge_runner.mk | 5 +---- 6 files changed, 13 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 648bb954..86d0184b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ if (ENABLE_OPEN_SRC) set(STATIC_ACL_LIB ${GE_LIB_PATH}) find_module(slog libslog.so ${GE_LIB_PATH}) find_module(static_mmpa libmmpa.a ${GE_LIB_PATH}) - find_module(msprof libmsprof.so ${GE_LIB_PATH}) + find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH}) find_module(hccl libhccl.so ${GE_LIB_PATH}) find_module(adump_server libadump_server.a ${GE_LIB_PATH}) find_module(runtime libruntime.so ${GE_LIB_PATH}) @@ -80,20 +80,19 @@ if (ENABLE_OPEN_SRC) find_module(error_manager liberror_manager.so ${GE_LIB_PATH}) find_module(ascend_hal_stub libascend_hal.so ${GE_LIB_PATH}) find_module(error_manager_static liberror_manager.a ${GE_LIB_PATH}) - find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH}) + find_module(msprofiler_fwk libmsprofiler_fwk.a ${GE_LIB_PATH}) #find_module(ascendcl_static libascendcl.a ${GE_LIB_PATH}) else() find_module(slog libslog.so ${ASCEND_ATC_DIR}) find_module(static_mmpa libmmpa.a ${ASCEND_ATC_DIR}) find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) if(PLATFORM STREQUAL "train") - find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR}) find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR}) find_module(resource libresource.so ${ASCEND_RUNTIME_DIR}) find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR}) - find_module(msprofiler libmsprofiler.a ${ASCEND_RUNTIME_DIR}) + find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_RUNTIME_DIR}) find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) if(PRODUCT STREQUAL "flr3") message(FATAL_ERROR "This platform is not supported in train mode, build terminated") @@ -108,18 +107,15 @@ if (ENABLE_OPEN_SRC) find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR}) #find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) if(PRODUCT STREQUAL "flr3") - find_module(msprof libmsprof.so ${ASCEND_DRIVER_SHARE_DIR}) elseif(PRODUCT STREQUAL "flr1") find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) - find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) elseif(PRODUCT STREQUAL "flr2") # flr2 ascend_hal_stub limsprof ? else() find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}) - find_module(msprof libmsprof.so ${ASCEND_DRIVER_DIR}) endif() elseif(PLATFORM STREQUAL "all") - find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) + find_module(msprofiler libmsprofiler.a ${ASCEND_DRIVER_COMMON_DIR}) find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) find_module(adump_server libadump_server.a ${ASCEND_ACL_DIR}) find_module(runtime libruntime.so ${ASCEND_ACL_DIR}) @@ -127,7 +123,7 @@ if (ENABLE_OPEN_SRC) find_module(resource libresource.so ${ASCEND_ATC_DIR}) find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR}) - find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR}) + find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_ACL_DIR}) find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) #find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) else() diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index e36b45d9..c2a34aba 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -779,7 +779,7 @@ target_link_libraries(opensrc_ascendcl PRIVATE register_static error_manager_static adump_server - msprofiler_fwk + msprofiler -Wl,--no-whole-archive -Wl,--no-as-needed c_sec diff --git a/ge/client/module.mk b/ge/client/module.mk index c942d550..e9d35418 100644 --- a/ge/client/module.mk +++ b/ge/client/module.mk @@ -68,9 +68,9 @@ LOCAL_SHARED_LIBRARIES := \ libgraph \ libregister \ libge_compiler \ - libge_common \ - libmsprof + libge_common +LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \ LOCAL_LDFLAGS := -lrt -ldl @@ -103,8 +103,10 @@ LOCAL_SHARED_LIBRARIES := \ libregister \ libruntime \ libge_compiler \ - libge_common \ - libmsprof + libge_common + + +LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \ LOCAL_LDFLAGS := -lrt -ldl diff --git a/ge/executor/CMakeLists.txt b/ge/executor/CMakeLists.txt index 29215cc1..cc5c1710 100644 --- a/ge/executor/CMakeLists.txt +++ b/ge/executor/CMakeLists.txt @@ -245,7 +245,6 @@ target_link_libraries(ge_executor_shared PRIVATE mmpa graph register - msprof error_manager ascend_hal_stub ascend_protobuf diff --git a/ge/executor/module.mk b/ge/executor/module.mk index cc64007e..34c2a37e 100644 --- a/ge/executor/module.mk +++ b/ge/executor/module.mk @@ -178,7 +178,6 @@ local_ge_executor_shared_library := \ libmmpa \ libgraph \ libregister \ - libmsprof \ liberror_manager \ local_ge_executor_ldflags := -lrt -ldl \ @@ -235,7 +234,6 @@ LOCAL_SHARED_LIBRARIES := \ libmmpa \ libgraph \ libregister \ - libmsprof \ liberror_manager \ stub/libascend_hal \ @@ -273,7 +271,6 @@ LOCAL_SHARED_LIBRARIES := \ libruntime \ libslog \ libmmpa \ - libmsprof \ LOCAL_LDFLAGS += $(local_ge_executor_ldflags) @@ -305,7 +302,6 @@ LOCAL_SHARED_LIBRARIES := \ libruntime \ libslog \ libmmpa \ - libmsprof \ ifeq ($(device_os),android) LOCAL_LDFLAGS += -ldl diff --git a/ge/ge_runner.mk b/ge/ge_runner.mk index 5e700f2b..820404b3 100644 --- a/ge/ge_runner.mk +++ b/ge/ge_runner.mk @@ -372,7 +372,7 @@ LOCAL_SRC_FILES += $(LIBCLIENT_LOCAL_SRC_FILES) LOCAL_STATIC_LIBRARIES := libge_memory \ libadump_server \ - libmsprofiler \ + libmsprofiler_fwk \ libmmpa \ LOCAL_SHARED_LIBRARIES := \ @@ -382,7 +382,6 @@ LOCAL_SHARED_LIBRARIES := \ libgraph \ libregister \ libge_common \ - libmsprof \ liberror_manager \ LOCAL_LDFLAGS := -lrt -ldl @@ -464,7 +463,6 @@ LOCAL_SHARED_LIBRARIES := \ libc_sec \ libslog \ libmmpa \ - libmsprof \ LOCAL_LDFLAGS := -lrt -ldl @@ -497,7 +495,6 @@ LOCAL_SHARED_LIBRARIES := \ libc_sec \ libslog \ libmmpa \ - libmsprof \ LOCAL_LDFLAGS := -lrt -ldl From 5823abd12fc186aecd7de3617b3b3e75b83cfb84 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sat, 12 Dec 2020 17:32:16 +0800 Subject: [PATCH 20/24] Profiling remove gerunner whole-archive --- ge/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index c2a34aba..f892a74c 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -648,9 +648,7 @@ target_include_directories(ge_runner PRIVATE target_link_libraries(ge_runner $ - -Wl,--whole-archive msprofiler_fwk - -Wl,--no-whole-archive ge_memory adump_server static_mmpa From c3bf97f497abe2686ecd81ec3d261a9468cd5233 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sat, 12 Dec 2020 21:05:47 +0800 Subject: [PATCH 21/24] rename ge_proiling_cb.h to ge_runner_profiling.h --- ge/CMakeLists.txt | 2 +- .../{ge_profiling_cb.cc => ge_runner_profiling.cc} | 2 +- ge/ge_runner.mk | 1 + .../profiling/{ge_profiling_cb.h => ge_runner_profiling.h} | 6 +++--- 4 files changed, 6 insertions(+), 5 deletions(-) rename ge/common/profiling/{ge_profiling_cb.cc => ge_runner_profiling.cc} (94%) rename inc/framework/common/profiling/{ge_profiling_cb.h => ge_runner_profiling.h} (80%) diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index f892a74c..a0044d23 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -61,7 +61,7 @@ set(TRAIN_SRC_LIST "common/dump/dump_properties.cc" "common/dump/dump_op.cc" "common/profiling/ge_profiling.cc" - "common/profiling/ge_profiling_cb.cc" + "common/profiling/ge_runner_profiling.cc" "engine_manager/dnnengine_manager.cc" "ge_local_engine/engine/host_cpu_engine.cc" "generator/ge_generator.cc" diff --git a/ge/common/profiling/ge_profiling_cb.cc b/ge/common/profiling/ge_runner_profiling.cc similarity index 94% rename from ge/common/profiling/ge_profiling_cb.cc rename to ge/common/profiling/ge_runner_profiling.cc index 230e129d..067aafe3 100644 --- a/ge/common/profiling/ge_profiling_cb.cc +++ b/ge/common/profiling/ge_runner_profiling.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "common/profiling/ge_profiling_cb.h" +#include "common/profiling/ge_runner_profiling.h" #include "init/gelib.h" bool IsInitialize() { diff --git a/ge/ge_runner.mk b/ge/ge_runner.mk index 820404b3..c0f59320 100644 --- a/ge/ge_runner.mk +++ b/ge/ge_runner.mk @@ -30,6 +30,7 @@ LIBGE_LOCAL_SRC_FILES := \ common/dump/dump_properties.cc \ common/dump/dump_op.cc \ common/profiling/ge_profiling.cc \ + common/profiling/ge_runner_profiling.cc \ engine_manager/dnnengine_manager.cc \ ge_local_engine/engine/host_cpu_engine.cc \ generator/ge_generator.cc \ diff --git a/inc/framework/common/profiling/ge_profiling_cb.h b/inc/framework/common/profiling/ge_runner_profiling.h similarity index 80% rename from inc/framework/common/profiling/ge_profiling_cb.h rename to inc/framework/common/profiling/ge_runner_profiling.h index 025232ff..d2eff767 100644 --- a/inc/framework/common/profiling/ge_profiling_cb.h +++ b/inc/framework/common/profiling/ge_runner_profiling.h @@ -14,11 +14,11 @@ * limitations under the License. */ -#ifndef INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ -#define INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ +#ifndef INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ +#define INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ #include "profiling/ge_profiling.h" bool IsInitialize(); -#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_CB_H_ \ No newline at end of file +#endif // INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ From b85945357069e0f3d7c611e6c2d208d10345029d Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sun, 13 Dec 2020 16:00:54 +0800 Subject: [PATCH 22/24] update cmakelist target_objects --- ge/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index a0044d23..59b804d8 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -605,7 +605,7 @@ set(INFER_SRC_LIST if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES) ############ libge_runner.so ############ -add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS}) +add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS} $) target_compile_definitions(ge_runner PRIVATE PROTOBUF_INLINE_NOT_IN_HEADERS=0 @@ -648,7 +648,6 @@ target_include_directories(ge_runner PRIVATE target_link_libraries(ge_runner $ - msprofiler_fwk ge_memory adump_server static_mmpa From abd0da89eec882da6f16597df92988cef6c1ca1f Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sun, 13 Dec 2020 16:36:14 +0800 Subject: [PATCH 23/24] add newline at end of file --- inc/framework/common/profiling/ge_profiling.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/framework/common/profiling/ge_profiling.h b/inc/framework/common/profiling/ge_profiling.h index 10b53d6d..e56411c9 100644 --- a/inc/framework/common/profiling/ge_profiling.h +++ b/inc/framework/common/profiling/ge_profiling.h @@ -42,4 +42,4 @@ ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func); ge::Status RegProfReporterCallback(MsprofReporterCallback func); ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len); -#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_H_ \ No newline at end of file +#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_H_ From c9e5c8721d05c59df1184fe9edfccfbee6fd134a Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sun, 13 Dec 2020 19:23:18 +0800 Subject: [PATCH 24/24] update submodule --- metadef | 2 +- parser | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metadef b/metadef index c85822cd..dba83744 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit c85822cd5404e40cb4ff2bfc9483062648c13c57 +Subproject commit dba83744a3ffe3d5f89496e69bb65c50f800c299 diff --git a/parser b/parser index 5bc8c38b..ce574894 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit 5bc8c38b37476e8f4b9391c96e4a2cca59e53d8e +Subproject commit ce574894f13cd94749d1a3964a13e8c97c20434a