m53297601
/
graphengine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add error msg

Browse Source
pull/1401/head
wangxiaotian22 4 years ago
parent 04d17e7a39
commit 3241cb6f29
11 changed files with 344 additions and 3 deletions
Show Stats Download Patch File Download Diff File

2
ge/graph/manager/graph_manager_utils.cc
Unescape View File

@ -84,6 +84,7 @@ Status SubGraphInfo::FreeInOutBuffer() {
rtError_t rt_ret;
rt_ret = rtFreeHost(*iter);
if (rt_ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail when SubGraphInfo %s", __FUNCTION__);
GELOGE(rt_ret, "[GraphManager] subgraph free buffer failed, modelId = %u", model_id_info_.model_id);
buffer_addr_.erase(buffer_addr_.begin(), iter);
return GE_GRAPH_FREE_FAILED;
@ -119,6 +120,7 @@ Status GraphModelListener::OnComputeDone(uint32_t model_id, uint32_t task_id, ui
uint32_t GraphModelListener::GetResultCode() const {
if (!is_finished_) {
REPORT_CALL_ERROR("E19999", "Model not run finish, fail for %s", __FUNCTION__);
GELOGE(INTERNAL_ERROR, "[GraphManager] model not run finish.");
return INTERNAL_ERROR;
}

9
ge/graph/manager/graph_mem_allocator.cc
Unescape View File

@ -49,6 +49,8 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, size_t memory_size
uint8_t *memory_addr = nullptr;
if (rtMalloc(reinterpret_cast<void **>(&memory_addr), memory_size, memory_type_) != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, purpose:%s, size:%zu, device_id:%u, when MemoryAllocator %s",
purpose.c_str(), memory_size, device_id, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR,
"MemoryAllocator::MallocMemory device_id = %u,"
" size= %lu",
@ -66,6 +68,7 @@ Status MemoryAllocator::FreeMemory(uint8_t *memory_addr, uint32_t device_id) con
GELOGI("MemoryAllocator::FreeMemory device_id = %u", device_id);
auto rtRet = rtFree(memory_addr);
if (rtRet != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtFree fail, device_id:%u, when MemoryAllocator %s", device_id, __FUNCTION__);
GELOGE(rtRet, "MemoryAllocator::MallocMemory device_id = %u", device_id);
return RT_ERROR_TO_GE_STATUS(rtRet);
}
@ -84,6 +87,9 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, const string &memo
uint8_t *memory_addr = MallocMemory(purpose, memory_size, device_id);
if (memory_addr == nullptr) {
REPORT_CALL_ERROR("E19999", "Malloc Memory fail, purpose:%s, memory_key:%s, memory_size:%zu, device_id:%u, "
"when MemoryAllocator %s", purpose.c_str(), memory_key.c_str(),
memory_size, device_id, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR,
"MemoryAllocator::MallocMemory failed,"
" memory_key[%s], size = %lu.",
@ -120,6 +126,8 @@ Status MemoryAllocator::FreeMemory(const string &memory_key, uint32_t device_id)
}
if (FreeMemory(it->second.memory_addr_, device_id) != ge::SUCCESS) {
REPORT_CALL_ERROR("E19999", "Free Memory fail, memory_key:%s, device_id:%u, when MemoryAllocator %s",
memory_key.c_str(), device_id, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR,
"MemoryAllocator::FreeMemory rtFree failed,"
" memory_key[%s]",
@ -169,6 +177,7 @@ Status MemManager::Initialize(const std::vector<rtMemType_t> &memory_type) {
memory_allocator_map_[index] = memory_allocator;
GELOGI("Create MemoryAllocator memory type[%u] success.", index);
} else {
REPORT_CALL_ERROR("E19999", "New MemoryAllocator fail, index:%u, when MemoryAllocator %s", index, __FUNCTION__);
GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc MemoryAllocator failed.");
}
} else {

51
ge/graph/manager/graph_var_manager.cc
Unescape View File

@ -39,6 +39,8 @@ VarResource::~VarResource() {
ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTensorDesc &tensor_desc, uint8_t **dev_ptr,
rtMemType_t &memory_type) {
if (dev_ptr == nullptr) {
REPORT_INNER_ERROR("E19999", "Param dev_ptr is nullptr, var_name:%s, session_id:%lu, "
"check invalid when VarResource %s", var_name.c_str(), session_id_, __FUNCTION__);
GELOGE(FAILED, "[GetVarAddr] dev_ptr is null!");
return FAILED;
}
@ -47,6 +49,9 @@ ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTens
auto iter = var_addr_mgr_map_.find(var_key);
if (iter == var_addr_mgr_map_.end()) {
REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, "
"check invalid when VarResource %s", var_key.c_str(), var_name.c_str(),
session_id_, __FUNCTION__);
GELOGE(FAILED, "VarResource::GetVarAddr failed, var_key %s", var_key.c_str());
return FAILED;
}
@ -102,6 +107,9 @@ ge::Status VarResource::SaveVarAddr(const std::string &var_name, const ge::GeTen
return SUCCESS;
}
REPORT_INNER_ERROR("E19999", "var_key:%s conflict in var_addr_mgr_map_, var_name:%s, session_id:%lu, "
"check invalid when VarResource %s", var_key.c_str(), var_name.c_str(),
session_id_, __FUNCTION__);
GELOGE(FAILED, "VarResource::SaveVarAddr, var_key %s save addr conflict", var_key.c_str());
return FAILED;
}
@ -136,6 +144,8 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O
}
if (op_desc == nullptr) {
REPORT_INNER_ERROR("E19999", "Param op_desc is nullptr, var_name:%s, session_id:%lu, check invalid "
"when VarResource %s", var_name.c_str(), session_id_, __FUNCTION__);
GELOGE(FAILED, "[RenewCurVarDesc] renew var desc fail! input opdesc is null!");
return FAILED;
}
@ -152,6 +162,9 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O
cur_var_tensor_desc_map_[var_name] = curr_desc;
auto iter = var_addr_mgr_map_.find(key);
if (iter == var_addr_mgr_map_.end()) {
REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, op:%s(%s), "
"check invalid when VarResource %s", key.c_str(), var_name.c_str(),
session_id_, op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "[RenewCurVarDesc] can't find ele with key [%s]", key.c_str());
return FAILED;
}
@ -271,11 +284,15 @@ Status HbmMemResource::AssignVarMem(const std::string &var_name, uint64_t size,
uint64_t real_size = size;
total_size_ = VarManager::Instance(session_id)->GetVarMemMaxSize();
if (total_size_ < var_mem_size_) {
REPORT_INNER_ERROR("E19999", "VarMemMaxSize:%lu < var_mem_size_:%lu, var_size:%lu, var_name:%s, check invalid"
" when HbmMemResource %s", total_size_, var_mem_size_, size, var_name.c_str(), __FUNCTION__);
GELOGE(PARAM_INVALID, "total_size_: %lu is smaller than var_mem_size_: %lu", total_size_, var_mem_size_);
return PARAM_INVALID;
}
uint64_t free_size = total_size_ - var_mem_size_;
if (free_size < (size + kSessionMemAlignSize * kSessionMemAlignUnit)) {
REPORT_INNER_ERROR("E19999", "free_size:%lu not enough, var_align_size:%lu, var_name:%s, check invalid "
"when HbmMemResource %s", free_size, size, var_name.c_str(), __FUNCTION__);
GELOGE(PARAM_INVALID, "Out of memory : current var size[%lu] exceeds total var size[%lu]",
size + kSessionMemAlignSize * kSessionMemAlignUnit + var_mem_size_, total_size_);
return PARAM_INVALID;
@ -299,6 +316,8 @@ Status HbmMemResource::AssignVarMem(const std::string &var_name, uint64_t size,
Status RdmaMemResource::AssignVarMem(const std::string &var_name, uint64_t size, uint64_t session_id, size_t &address) {
uint8_t *buffer = MemManager::Instance().RdmaPoolInstance(RT_MEMORY_HBM).Malloc(size);
if (buffer == nullptr) {
REPORT_CALL_ERROR("E19999", "malloc rdma memory fail, var_size:%lu, var_name:%s when RdmaMemResource %s",
size, var_name.c_str(), __FUNCTION__);
GELOGE(MEMALLOC_FAILED, "Failed to malloc rdma memory for node %s, size = %lu", var_name.c_str(), size);
return MEMALLOC_FAILED;
}
@ -448,6 +467,8 @@ int64_t VarManager::GetVarMemSize(rtMemType_t memory_type) {
}
if (mem_resource == nullptr) {
REPORT_INNER_ERROR("E19999", "Find no mem_resource in map, memory_type:%d, session_id:%lu when VarManager %s",
memory_type, session_id_, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid.");
return 0;
}
@ -461,6 +482,8 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) {
if (iter == mem_resource_map_.end()) {
mem_resource = MemResource::BuildMemResourceFromType(memory_type);
if (mem_resource == nullptr) {
REPORT_INNER_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu when VarManager %s",
memory_type, session_id_, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type);
return ge::INTERNAL_ERROR;
} else {
@ -471,6 +494,8 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) {
}
if (mem_resource == nullptr) {
REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu when VarManager %s",
memory_type, session_id_, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid.");
return FAILED;
}
@ -489,6 +514,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen
size_t mem_offset = 0;
ge::Status result = TensorUtils::GetSize(tensor_desc, tensor_desc_size);
if (result != ge::SUCCESS) {
REPORT_INNER_ERROR("E19999", "Get size from tensor fail, var_name:%s, memory_type:%d, session_id:%lu, "
"when VarManager %s", var_name.c_str(), memory_type, session_id_, __FUNCTION__);
GELOGE(result, "get size from TensorDesc failed");
return result;
}
@ -498,6 +525,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen
if (it == mem_resource_map_.end()) {
mem_resource = MemResource::BuildMemResourceFromType(memory_type);
if (mem_resource == nullptr) {
REPORT_INNER_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu when VarManager %s",
memory_type, session_id_, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type);
return ge::INTERNAL_ERROR;
} else {
@ -508,6 +537,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen
}
if (mem_resource == nullptr) {
REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu when VarManager %s",
memory_type, session_id_, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid, memory_type = %u.", memory_type);
return ge::INTERNAL_ERROR;
}
@ -517,6 +548,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen
return ge::INTERNAL_ERROR;
}
if (var_resource_ == nullptr) {
REPORT_INNER_ERROR("E19999", "VarManager has not been init, memory_type:%d, session_id:%lu, "
"check invalid when VarManager %s", memory_type, session_id_, __FUNCTION__);
GELOGW("VarManager has not been init.");
return ge::INTERNAL_ERROR;
}
@ -635,6 +668,9 @@ ge::Status VarManager::RenewCurVarDesc(const std::string &var_name, ge::OpDescPt
GELOGD("VarManager::RenewCurVarDesc var_name = %s.", var_name.c_str());
if (var_resource_ == nullptr) {
REPORT_INNER_ERROR("E19999", "VarManager has not been init, op:%s(%s), session_id:%lu, check invalid "
"when VarManager %s", op_desc->GetName().c_str(), op_desc->GetType().c_str(),
session_id_, __FUNCTION__);
GELOGE(ge::INTERNAL_ERROR, "VarManager has not been init.");
return ge::INTERNAL_ERROR;
}
@ -786,6 +822,8 @@ Status VarManager::SetMemoryMallocSize(const map<string, string> &options) {
var_mem_logic_base_ = graph_mem_max_size_ + kGraphMemoryBuffer;
if (var_mem_logic_base_ > kMaxMemorySize) {
REPORT_INNER_ERROR("E19999", "var_login_base:%zu can not exeed limit:%zu, session_id:%lu, check invalid "
"when VarManager %s", var_mem_logic_base_, kMaxMemorySize, session_id_, __FUNCTION__);
GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kMemoryVarLogicBase : %zu can not exceed max memory size : %zu.",
var_mem_logic_base_, kMaxMemorySize);
return ge::GE_GRAPH_OPTIONS_INVALID;
@ -793,6 +831,8 @@ Status VarManager::SetMemoryMallocSize(const map<string, string> &options) {
use_max_mem_size_ = graph_mem_max_size_ + var_mem_max_size_;
if (use_max_mem_size_ > kMaxMemorySize) {
REPORT_INNER_ERROR("E19999", "all mem_use size:%zu can not exeed limit:%zu, session_id:%lu, check invalid "
"when VarManager %s", use_max_mem_size_, kMaxMemorySize, session_id_, __FUNCTION__);
GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kUseMaxMemorySize : %zu can not exceed max memory size : %zu.",
use_max_mem_size_, kMaxMemorySize);
return ge::GE_GRAPH_OPTIONS_INVALID;
@ -803,6 +843,8 @@ Status VarManager::SetMemoryMallocSize(const map<string, string> &options) {
Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) {
if (memory_size.empty()) {
REPORT_INNER_ERROR("E19999", "Param memory_size is empty, session_id:%lu, check invalid when VarManager %s",
session_id_, __FUNCTION__);
GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input is empty.");
return GE_GRAPH_OPTIONS_INVALID;
}
@ -828,15 +870,23 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) {
for (char c : split) {
if (!isdigit(c)) {
REPORT_INNER_ERROR("E19999", "Param memory_size:%s contains non digit, session_id:%lu, check invalid "
"when VarManager %s", memory_size.c_str(), session_id_, __FUNCTION__);
GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input contains non digit.");
return GE_GRAPH_OPTIONS_INVALID;
}
}
uint64_t num = std::strtoul(split.c_str(), nullptr, 0);
GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(result, static_cast<uint32_t>(num)),
REPORT_INNER_ERROR("E19999", "Param memory_size:%s will overflow after multi all, session_id:%lu, "
"check invalid when VarManager %s", memory_size.c_str(),
session_id_, __FUNCTION__);
GELOGE(FAILED, "Input memory size is out of range.");
return FAILED);
if ((num > kMaxMemorySize) || (result * static_cast<size_t>(num) > kMaxMemorySize)) {
REPORT_INNER_ERROR("E19999", "Param memory_size:%s after multi will exceed limit:%lu, session_id:%lu, "
"check invalid when VarManager %s", memory_size.c_str(), kMaxMemorySize,
session_id_, __FUNCTION__);
GELOGE(FAILED, "Input memory size can not exceed max memory size : %zu.", kMaxMemorySize);
return FAILED;
}
@ -940,6 +990,7 @@ VarManager *VarManagerPool::GetVarManager(uint64_t session_id) {
VarManager *var_manager = new (std::nothrow) VarManager(session_id);
if (var_manager == nullptr) {
REPORT_INNER_ERROR("E19999", "New VarManager fali, session_id:%lu, when VarManager %s", session_id, __FUNCTION__);
GELOGE(INTERNAL_ERROR,
"VarManager::Instance find session by "
"session_id[%lu] failed.",

4
ge/graph/manager/host_mem_allocator.cc
Unescape View File

@ -34,6 +34,7 @@ uint8_t *HostMemAllocator::Malloc(size_t size) {
std::lock_guard<std::mutex> lock(mutex_);
std::shared_ptr<AlignedPtr> aligned_ptr = MakeShared<AlignedPtr>(size);
if (aligned_ptr == nullptr) {
REPORT_INNER_ERROR("E19999", "New AlignedPtr fail, when HostMemAllocator %s", __FUNCTION__);
GELOGE(INTERNAL_ERROR, "make shared_ptr for AlignedPtr failed");
return nullptr;
}
@ -44,6 +45,7 @@ uint8_t *HostMemAllocator::Malloc(size_t size) {
Status HostMemAllocator::Free(const void *memory_addr) {
if (memory_addr == nullptr) {
REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, check invalid when HostMemAllocator %s", __FUNCTION__);
GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer");
return GE_GRAPH_FREE_FAILED;
}
@ -51,6 +53,8 @@ Status HostMemAllocator::Free(const void *memory_addr) {
std::lock_guard<std::mutex> lock(mutex_);
auto it = allocated_blocks_.find(memory_addr);
if (it == allocated_blocks_.end()) {
REPORT_INNER_ERROR("E19999", "Memory_addr is not alloc before, check invalid when HostMemAllocator %s",
__FUNCTION__);
GELOGE(PARAM_INVALID, "Invalid memory pointer");
return PARAM_INVALID;
}

9
ge/graph/manager/host_mem_manager.cc
Unescape View File

@ -39,6 +39,8 @@ Status SharedMemAllocator::Allocate(SharedMemInfo &mem_info) {
rtMallocHostSharedMemoryOut output_para;
rtError_t rt_ret = rtMallocHostSharedMemory(&input_para, &output_para);
if (rt_ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtMallocHostSharedMemory fail, ret:0x%X, when SharedMemAllocator %s",
rt_ret, __FUNCTION__);
GELOGE(RT_FAILED, "Call rt api(rtMallocHostSharedMemory) failed, devid:[%u].", device_id);
return GE_GRAPH_MEMORY_ALLOC_FAILED;
}
@ -59,6 +61,8 @@ Status SharedMemAllocator::DeAllocate(SharedMemInfo &mem_info) {
mem_info.host_aligned_ptr->MutableGet(), mem_info.device_address};
rtError_t rt_ret = rtFreeHostSharedMemory(&free_para);
if (rt_ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtFreeHostSharedMemory fail, ret:0x%X, when SharedMemAllocator %s",
rt_ret, __FUNCTION__);
GELOGE(RT_FAILED, "Call rt api(rtFreeHostSharedMemory) failed, ret: 0x%X.", rt_ret);
return RT_FAILED;
}
@ -74,6 +78,7 @@ Status HostMemManager::Initialize() {
std::lock_guard<std::recursive_mutex> lock(mutex_);
allocator_ = std::unique_ptr<SharedMemAllocator>(new (std::nothrow) SharedMemAllocator());
if (allocator_ == nullptr) {
REPORT_CALL_ERROR("E19999", "New SharedMemAllocator fail when SharedMemAllocator %s", __FUNCTION__);
GELOGE(GE_GRAPH_MALLOC_FAILED, "Shared memory allocator init failed!");
return GE_GRAPH_MALLOC_FAILED;
}
@ -94,6 +99,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) {
std::lock_guard<std::recursive_mutex> lock(mutex_);
auto iter = var_memory_base_map_.find(mem_info.op_name);
if (iter != var_memory_base_map_.end()) {
REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_ when HostMemManager %s",
mem_info.op_name.c_str(), __FUNCTION__);
GELOGE(FAILED, "Host shared memory for op %s has been malloced", mem_info.op_name.c_str());
return FAILED;
}
@ -107,6 +114,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) {
Status HostMemManager::QueryVarMemInfo(const string &op_name, uint64_t &base_addr, uint64_t &data_size) {
std::lock_guard<std::recursive_mutex> lock(mutex_);
if (var_memory_base_map_.find(op_name) == var_memory_base_map_.end()) {
REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_ when HostMemManager %s",
op_name.c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Find host base base_addr failed,node name:%s!", op_name.c_str());
return INTERNAL_ERROR;
}

6
ge/graph/manager/memory_api.cc
Unescape View File

@ -50,6 +50,8 @@ Status RdmaRemoteRegister(const std::vector<HostVarInfo> &var_info, rtMemType_t
path.append(file_name);
string canonical_path = RealPath(path.c_str());
if (canonical_path.empty()) {
REPORT_INNER_ERROR("E19999", "canonical_path:%s is empty, check invalid when %s",
canonical_path.c_str(), __FUNCTION__);
GELOGE(FAILED, "Failed to get realpath of %s", path.c_str());
return FAILED;
}
@ -65,12 +67,16 @@ Status RdmaRemoteRegister(const std::vector<HostVarInfo> &var_info, rtMemType_t
auto hcom_remote_mem_register =
(HcclResult(*)(const MemRegisterAddr *, uint32_t))dlsym(handle, "HcomRegRemoteAccessMem");
if (hcom_remote_mem_register == nullptr) {
REPORT_CALL_ERROR("E19999", "Symbol HcomRegRemoteAccessMem can't find in %s, check invalid when %s",
canonical_path.c_str(), __FUNCTION__);
GELOGE(FAILED, "Failed to invoke hcom_remote_mem_register function.");
return FAILED;
}
HcclResult hccl_ret = hcom_remote_mem_register(reg_addrs.get(), table_len);
if (hccl_ret != HCCL_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Call hcom_remote_mem_register failed, ret:%u, when %s",
hccl_ret.c_str(), __FUNCTION__);
GELOGE(HCCL_E_INTERNAL, "Rdma mem register failed, ret: 0x%X", hccl_ret);
return HCCL_E_INTERNAL;
}

11
ge/graph/manager/rdma_pool_allocator.cc
Unescape View File

@ -81,6 +81,8 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) {
auto device_id = GetContext().DeviceId();
GELOGD("Init Rdma Memory with size [%zu] for devid:[%u]", mem_size, device_id);
if (rdma_base_addr_ != nullptr) {
REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid when RdmaPoolAllocator %s",
__FUNCTION__);
GELOGE(GE_MULTI_INIT, "Rdma pool has been malloced");
return GE_MULTI_INIT;
}
@ -100,6 +102,7 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) {
// Init with a base block.
auto *base_block = new (std::nothrow) Block(device_id, mem_size, rdma_base_addr_);
if (base_block == nullptr) {
REPORT_CALL_ERROR("E19999", "New Block failed, device_id:%u, when RdmaPoolAllocator %s", device_id, __FUNCTION__);
GELOGE(GE_GRAPH_MALLOC_FAILED, "Block malloc failed");
return GE_GRAPH_MALLOC_FAILED;
}
@ -118,6 +121,8 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) {
block_bin_.erase(it);
block->allocated = true;
if (block->ptr == nullptr) {
REPORT_INNER_ERROR("E19999", "Rdmapool memory address is nullptr, device_id:%u, check invalid when RdmaPoolAllocator %s",
device_id, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Rdmapool memory address is nullptr.");
return nullptr;
}
@ -150,6 +155,8 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) {
Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) {
GELOGI("Free rdma memory, device id = %u", device_id);
if (memory_addr == nullptr) {
REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, device_id:%u, check invalid when RdmaPoolAllocator %s",
device_id, __FUNCTION__);
GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer");
return GE_GRAPH_FREE_FAILED;
}
@ -157,6 +164,8 @@ Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) {
std::lock_guard<std::recursive_mutex> lock(mutex_);
auto it = allocated_blocks_.find(memory_addr);
if (it == allocated_blocks_.end()) {
REPORT_INNER_ERROR("E19999", "Param memory_addr is not allocated before, device_id:%u, "
"check invalid when RdmaPoolAllocator %s", device_id, __FUNCTION__);
GELOGE(PARAM_INVALID, "Invalid memory pointer");
return PARAM_INVALID;
}
@ -199,6 +208,8 @@ void RdmaPoolAllocator::MergeBlocks(Block *dst, Block *src) {
Status RdmaPoolAllocator::GetBaseAddr(uint64_t &base_addr, uint64_t &mem_size) {
if (rdma_base_addr_ == nullptr) {
REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid when RdmaPoolAllocator %s",
__FUNCTION__);
GELOGE(INTERNAL_ERROR, "Rdma base addr is nullptr.");
return INTERNAL_ERROR;
}

53
ge/graph/manager/trans_var_data_utils.cc
Unescape View File

@ -35,18 +35,24 @@ class RtContextSwitchGuard {
RtContextSwitchGuard(rtCtxMode_t mode, uint32_t device_id) : last_(nullptr), current_(nullptr) {
auto ret = rtCtxGetCurrent(&last_);
if (ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, device_id:%u, ret:0x%X, when %s",
device_id, ret, __FUNCTION__);
GELOGE(RT_FAILED, "Failed to get current context from rt, error-code %d", ret);
return;
}
ret = rtCtxCreate(&current_, mode, static_cast<int32_t>(device_id));
if (ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtCtxCreate failed, device_id:%u, ret:0x%X, when %s",
device_id, ret, __FUNCTION__);
GELOGE(RT_FAILED, "Failed to create new context for device %u, error-code %d", device_id, ret);
return;
}
ret = rtCtxSetCurrent(current_);
if (ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, device_id:%u, ret:0x%X, when %s",
device_id, ret, __FUNCTION__);
GELOGE(RT_FAILED, "Failed to switch context to normal, context %p, device %u", current_, device_id);
return;
}
@ -72,6 +78,8 @@ class RtContextSwitchGuard {
int64_t CalcVarSizeInBytes(const GeTensorDesc &desc) {
int64_t var_size = GetSizeByDataType(desc.GetDataType());
if (var_size <= 0) {
REPORT_INNER_ERROR("E19999", "Data type:%s in desc, it's size:%ld < 0, check invalid when %s",
TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str(), var_size, __FUNCTION__);
GELOGE(PARAM_INVALID, "Failed to calc var data size from data type %s",
TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str());
return -1;
@ -89,6 +97,8 @@ Status CopyVarToDevice(const NodePtr &var, const formats::TransResult &trans_res
auto ret = rtMemcpy(var_addr, trans_result.length, reinterpret_cast<void *>(trans_result.data.get()),
trans_result.length, RT_MEMCPY_HOST_TO_DEVICE);
if (ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, op:%s(%s), size:%lu, ret:0x%X, when %s", var->GetName().c_str(),
var->GetType().c_str(), trans_result.length, ret, __FUNCTION__);
GELOGE(RT_FAILED, "Failed to copy memory to device, size %zu", trans_result.length);
return RT_FAILED;
}
@ -110,6 +120,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt
uint8_t *var_addr = VarManager::Instance(session_id)->GetVarMemoryAddr(var_logic, RT_MEMORY_HBM);
if (var_addr == nullptr) {
REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, op:%s(%s), session_id:%lu, when %s",
RT_MEMORY_HBM, var->GetName().c_str(), var->GetType().c_str(), session_id, __FUNCTION__);
GELOGE(INTERNAL_ERROR,
"Failed to copy var %s from device, cant not get "
"var addr from logic addr %p",
@ -124,6 +136,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt
std::unique_ptr<uint8_t[]> var_host(new(std::nothrow) uint8_t[var_size_bytes]);
if (var_host == nullptr) {
REPORT_CALL_ERROR("E19999", "New host memory failed, size:%ld, op:%s(%s), session_id:%lu, when %s",
var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, __FUNCTION__);
GELOGE(OUT_OF_MEMORY, "Failed to malloc rt-host memory, size %ld", var_size_bytes);
return OUT_OF_MEMORY;
}
@ -131,6 +145,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt
ret = rtMemcpy(reinterpret_cast<void *>(var_host.get()), var_size_bytes, reinterpret_cast<void *>(var_addr),
var_size_bytes, RT_MEMCPY_DEVICE_TO_HOST);
if (ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%ld, op:%s(%s), session_id:%lu, ret:0x%X when %s",
var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, ret, __FUNCTION__);
GELOGE(RT_FAILED,
"Failed to copy var memory from device, var %s, size %ld,"
" rt-error-code %u",
@ -175,6 +191,12 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats
TypeUtils::DataTypeToSerialString(data_type).c_str());
auto ret = formats::TransFormat({src_data, src_format, dst_format, src_shape, dst_shape, data_type}, tmp_result);
if (ret != SUCCESS) {
REPORT_CALL_ERROR("E19999", "Trans format from %s to %s, shape %s to %s failed, data type:%s, ret:%u, when %s",
TypeUtils::FormatToSerialString(src_format).c_str(),
TypeUtils::FormatToSerialString(dst_format).c_str(),
formats::ShapeToString(src_shape).c_str(),
formats::ShapeToString(dst_shape).c_str(),
TypeUtils::DataTypeToSerialString(data_type).c_str(), ret, __FUNCTION__);
GELOGE(INTERNAL_ERROR,
"Failed to trans format from %s to %s, shape %s to %s, "
"data type %s error code %u",
@ -195,6 +217,10 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats
auto ret = formats::TransDataType({src_data, static_cast<size_t>(src_data_size), src_data_type, dst_data_type},
tmp_result);
if (ret != SUCCESS) {
REPORT_CALL_ERROR("E19999", "Trans data type from %s to %s failed, input shape %s, data size %ld, ret:%u, "
"when %s", TypeUtils::DataTypeToSerialString(src_data_type).c_str(),
TypeUtils::DataTypeToSerialString(dst_data_type).c_str(),
formats::ShapeToString(input_shape).c_str(), src_data_size, ret, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to trans data type from %s to %s, input shape %s, data size %ld, error code %u",
TypeUtils::DataTypeToSerialString(src_data_type).c_str(),
TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), formats::ShapeToString(input_shape).c_str(),
@ -202,6 +228,8 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats
return ret;
}
} else {
REPORT_INNER_ERROR("E19999", "Trans var data failed, the trans type %s does not supported, check invalid when %s",
trans_info.node_type.c_str(), __FUNCTION__);
GELOGE(UNSUPPORTED, "Failed to trans var data, the trans type %s does not supported",
trans_info.node_type.c_str());
return UNSUPPORTED;
@ -236,6 +264,8 @@ Status ReAssignVarAddr(uint64_t session_id,
uint8_t *var_addr = VarManager::Instance(session_id)->GetVarMemoryAddr(var_logic, RT_MEMORY_HBM);
if (var_addr == nullptr) {
REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, var_name:%s, session_id:%lu, when %s",
RT_MEMORY_HBM, var_name.c_str(), session_id, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to convert var %s logic addr to real addr", var_name.c_str());
return INTERNAL_ERROR;
}
@ -263,6 +293,8 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t
// Sync var data from device
std::unique_ptr<uint8_t[]> var_data;
if (trans_road.empty()) {
REPORT_INNER_ERROR("E19999", "Param trans_road is empty, session_id:%lu, check invalid when %s",
session_id, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to get trans_road, trans_road is empty.");
return INTERNAL_ERROR;
}
@ -314,6 +346,10 @@ Status TransTensor(uint8_t *var_data, const NodePtr &var_src, const NodePtr &var
auto ret = formats::TransDataType(
{var_data, static_cast<size_t>(src_data_shape_size), src_data_datatype, dst_data_datatype}, result);
if (ret != SUCCESS) {
REPORT_CALL_ERROR("E19999", "Trans data type from %s to %s failed, data size %ld, ret:%u, "
"when %s", TypeUtils::DataTypeToSerialString(src_data_datatype).c_str(),
TypeUtils::DataTypeToSerialString(dst_data_datatype).c_str(),
src_data_shape_size, ret, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "trans var data on host failed");
return ret;
});
@ -329,7 +365,10 @@ Status CopyTensorFromSrcVarNode(const NodePtr &var_src,
/// unlink edges between var_fp32 and "dst_node" (need fp16) of var_fp32, add edge between var_fp16 and dst_node.
/// need copy value from var_fp32 to var_fp16.
/// [opdesc of var_src and var_dst are checked before passed in, no need to check if they are nullptr]
GE_IF_BOOL_EXEC(var_src == nullptr || var_dst == nullptr, GELOGE(FAILED, "node var is nullptr"); return FAILED);
GE_IF_BOOL_EXEC(var_src == nullptr || var_dst == nullptr,
REPORT_INNER_ERROR("E19999", "Param var_src or var_dst is empty, session_id:%lu, device_id:%u, "
"check invalid when %s", session_id, device_id, __FUNCTION__);
GELOGE(FAILED, "node var is nullptr"); return FAILED);
// src_node output_desc (fp32)
GeTensorDesc output_desc = var_src->GetOpDesc()->GetOutputDesc(0);
auto src_data_type = output_desc.GetDataType();
@ -447,15 +486,21 @@ Status TransVarDataUtils::TransAllVarData(const vector<NodePtr> &variable_nodes,
}
std::future<Status> f = executor.commit(
[](const ge::NodePtr &node, uint64_t session_id, rtContext_t ctx, uint32_t graph_id) -> Status {
[](const ge::NodePtr &node, uint64_t session_id, rtContext_t ctx, uint32_t graph_id,
const struct ErrorMessage::Context &error_context) -> Status {
ErrorManager::GetInstance().SetErrorContext(error_context);
rtError_t rt_ret = rtCtxSetCurrent(ctx);
if (rt_ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, session_id:%lu, graph_id:%u, ret:0x%X, when %s",
session_id, graph_id, rt_ret, __FUNCTION__);
GELOGE(RT_FAILED, "Failed to set context, error_code is: 0x%X.", rt_ret);
return RT_ERROR_TO_GE_STATUS(rt_ret);
}
uint32_t allocated_graph_id = 0;
Status ret = VarManager::Instance(session_id)->GetAllocatedGraphId(node->GetName(), allocated_graph_id);
if (ret != SUCCESS) {
REPORT_CALL_ERROR("E19999", "Get allocated GraphId failed, session_id:%lu, graph_id:%u, ret:0x%X, when %s",
session_id, graph_id, ret, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "var has not been allocated, node:%s, graph_id:%u.", node->GetName().c_str(),
graph_id);
return INTERNAL_ERROR;
@ -480,7 +525,7 @@ Status TransVarDataUtils::TransAllVarData(const vector<NodePtr> &variable_nodes,
}
return SUCCESS;
},
node, session_id, context, graph_id);
node, session_id, context, graph_id, ErrorManager::GetInstance().GetErrorContext());
if (!f.valid()) {
GELOGE(FAILED, "Future is invalid");
return FAILED;
@ -503,6 +548,8 @@ Status TransVarDataUtils::TransAllVarData(const vector<NodePtr> &variable_nodes,
Status TransVarDataUtils::CopyVarData(const ComputeGraphPtr &compute_graph, uint64_t session_id, uint32_t device_id) {
GELOGD("CopyVarData start: session_id:%lu.", session_id);
if (compute_graph == nullptr) {
REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, session_id:%lu, device_id:%u, check invalid when %s",
session_id, device_id, __FUNCTION__);
GELOGE(FAILED, "compute_graph is nullptr");
return FAILED;
}

129
ge/graph/preprocess/graph_preprocess.cc
View File

File diff suppressed because it is too large Load Diff

46
ge/graph/preprocess/insert_op/ge_aipp_op.cc
Unescape View File

@ -110,6 +110,12 @@ Status GetDataDimN(const ge::NodePtr &data_node, ge::Format format, int64_t &bat
batch = shape[NHWC_DIM_N];
return SUCCESS;
default:
REPORT_INPUT_ERROR("E10001", std::vector<std::string>({"parameter", "value", "reason"}),
std::vector<std::string>({
data_node->GetName() + " format",
TypeUtils::FormatToSerialString(format),
"only format " + TypeUtils::FormatToSerialString(FORMAT_NCHW) + " and "
+ TypeUtils::FormatToSerialString(FORMAT_NHWC) + " supported"}));
GELOGE(PARAM_INVALID, "Not support data format: %s", TypeUtils::FormatToSerialString(format).c_str());
return PARAM_INVALID;
}
@ -156,6 +162,7 @@ Format GetAndCheckFormat() {
Status AippOp::Init(domi::AippOpParams *aipp_params) {
aipp_params_ = new (std::nothrow) domi::AippOpParams();
if (aipp_params_ == nullptr) {
REPORT_CALL_ERROR("E19999", "New AippOpParams failed when AippOp %s", __FUNCTION__);
return FAILED;
}
aipp_params_->CopyFrom(*aipp_params);
@ -190,6 +197,13 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig
auto ret = GraphUtils::InsertNodeBetweenDataAnchors(out_in_anchors.first, out_in_anchors.second, aipp);
if (ret != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Insert aipp:%s(%s) node between op:%s(%s) and op:%s:%s failed when AippOp %s",
aipp->GetName().c_str(), aipp->GetType().c_str(),
out_in_anchors.first->GetOwnerNode()->GetName().c_str(),
out_in_anchors.first->GetOwnerNode()->GetType().c_str(),
out_in_anchors.second->GetOwnerNode()->GetName().c_str(),
out_in_anchors.second->GetOwnerNode()->GetType().c_str(),
__FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to link edges for aipp node %s", aipp->GetName().c_str());
return INTERNAL_ERROR;
}
@ -209,6 +223,11 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig
auto &aipp = iter->second;
auto ret = out_in_anchors.second->LinkFrom(aipp->GetOutDataAnchor(0));
if (ret != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "link aipp:%s(%s) to peer op:%s(%s) failed when AippOp %s",
aipp->GetName().c_str(), aipp->GetType().c_str(),
out_in_anchors.second->GetOwnerNode()->GetName().c_str(),
out_in_anchors.second->GetOwnerNode()->GetType().c_str(),
__FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to link aipp %s to the peer node %s", aipp->GetName().c_str(),
out_in_anchors.second->GetOwnerNode()->GetName().c_str());
return INTERNAL_ERROR;
@ -224,6 +243,7 @@ NodePtr AippOp::CreateAipp(const OutDataAnchorPtr &out_anchor,
std::string current_name = node->GetName() + "_" + std::to_string(out_anchor->GetIdx()) + "_huawei_aipp";
auto aipp_opdesc_ptr = MakeShared<OpDesc>(current_name, AIPP);
if (aipp_opdesc_ptr == nullptr) {
REPORT_CALL_ERROR("E19999", "New OpDesc failed when AippOp %s", __FUNCTION__);
GELOGE(OUT_OF_MEMORY, "Failed to alloc aipp desc, name %s", current_name.c_str());
return nullptr;
}
@ -250,6 +270,9 @@ NodePtr AippOp::CreateAipp(const OutDataAnchorPtr &out_anchor,
// but the InferFormat process before InferShape can not infer the format
// if the tensor on the Aipp has an unknown shape
if (aipp_opdesc_ptr->UpdateInputDesc(kAippImageInputIndex, opdesc_src_data) != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Update the output desc from node:%s(%s) to aipp:%s(%s) failed when AippOp %s",
node_desc->GetName().c_str(), node_desc->GetType().c_str(),
aipp_opdesc_ptr->GetName().c_str(), aipp_opdesc_ptr->GetType().c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to update the output desc from node %s to aipp %s", node_desc->GetName().c_str(),
aipp_opdesc_ptr->GetName().c_str());
return nullptr;
@ -341,6 +364,8 @@ Status AippOp::GetAndCheckTarget(const ComputeGraphPtr &graph, int rank, NodePtr
GeAttrValue::NAMED_ATTRS aipp_attr;
ConvertParamToAttr(aipp_attr);
if (!AttrUtils::SetNamedAttrs(data_opdesc, ATTR_NAME_AIPP, aipp_attr)) {
REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed when AippOp %s", ATTR_NAME_AIPP.c_str(),
data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Set name attrs for Data node failed. id: %d", rank);
return INTERNAL_ERROR;
}
@ -371,12 +396,18 @@ Status AippOp::GetStaticTargetNode(const ComputeGraphPtr &graph, NodePtr &data_n
std::string related_node_name;
if (AttrUtils::GetStr(data_node->GetOpDesc(), kMbatchSwitchnName, related_node_name)) {
if (related_node_name.empty()) {
REPORT_INNER_ERROR("E19999", "The data node %s has switchn node flag, but the value of attr:%s is empty, "
"check invalid when AippOp %s", data_node->GetName().c_str(),
kMbatchSwitchnName, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "The data node %s has switchn node flag, but the value is empty",
data_node->GetName().c_str());
return INTERNAL_ERROR;
}
auto switchn = graph->FindNode(related_node_name);
if (switchn == nullptr) {
REPORT_INNER_ERROR("E19999", "The data node %s has switchn node %s, but can not find it on the graph, "
"check invalid when AippOp %s", data_node->GetName().c_str(), related_node_name.c_str(),
__FUNCTION__);
GELOGE(INTERNAL_ERROR, "The data node %s has switchn node %s, but can not find it on the graph",
data_node->GetName().c_str(), related_node_name.c_str());
return INTERNAL_ERROR;
@ -466,6 +497,9 @@ Status AippOp::GetTargetPosition(ComputeGraphPtr graph, NodePtr &target_input,
for (const auto &name : func_desc->GetSubgraphInstanceNames()) {
const auto &subgraph = graph->GetSubgraph(name);
if (subgraph == nullptr) {
REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid when AippOp %s",
name.c_str(), func_desc->GetName().c_str(), func_desc->GetType().c_str(),
graph->GetName().c_str(), __FUNCTION__);
GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str());
return GE_GRAPH_EMPTY_SUBGRAPH;
}
@ -666,11 +700,15 @@ Status AippOp::GenerateOpDesc(OpDescPtr op_desc) {
// Add two InputDesc, add the second after the first one is added successfully.
if ((op_desc->AddInputDesc(GeTensorDesc()) != GRAPH_SUCCESS) ||
(op_desc->AddInputDesc(GeTensorDesc()) != GRAPH_SUCCESS)) {
REPORT_CALL_ERROR("E19999", "Add input desc into op:%s(%s) failed when AippOp %s",
op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "failed to add input desc");
return FAILED;
}
if (op_desc->AddOutputDesc(GeTensorDesc()) != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Add output desc into op:%s(%s) failed when AippOp %s",
op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "add output desc failed.");
return FAILED;
}
@ -678,6 +716,8 @@ Status AippOp::GenerateOpDesc(OpDescPtr op_desc) {
ConvertParamToAttr(aipp_attrs);
GE_IF_BOOL_EXEC(!AttrUtils::SetNamedAttrs(op_desc, ATTR_NAME_AIPP, aipp_attrs),
REPORT_INNER_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when AippOp %s", ATTR_NAME_AIPP.c_str(),
op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "failed to set ATTR_NAME_AIPP");
return FAILED);
@ -858,12 +898,18 @@ Status AippOp::AddNodeToGraph(const NodePtr &aipp_node, int64_t max_dynamic_aipp
// add node desc for aipp node
auto stat3 = aipp_node->GetOpDesc()->UpdateInputDesc(kAippParamsInputIndex, output_tensor);
if (stat1 != GRAPH_SUCCESS || stat2 != GRAPH_SUCCESS || stat3 != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Add and Update InputDesc to op:%s(%s) failed, index:%d, when AippOp %s",
aipp_node->GetName().c_str(), aipp_node->GetType().c_str(), kAippParamsInputIndex, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "node process desc failed!");
return INTERNAL_ERROR;
}
// aipp_node should have two input data but now tbe only one input
if (GraphUtils::AddEdge(aipp_data_node_ptr->GetOutDataAnchor(kAippDataOutputIndex),
aipp_node->GetInDataAnchor(kAippParamsInputIndex)) != GRAPH_SUCCESS) {
REPORT_INNER_ERROR("E19999", "Add edge between op:%s(%s)(out_index:%u) and op:%s(%s)(in_index:%u) failed "
"when AippOp %s", aipp_data_node_ptr->GetName().c_str(), aipp_data_node_ptr->GetType().c_str(),
kAippDataOutputIndex, aipp_node->GetName().c_str(), aipp_node->GetType().c_str(),
kAippParamsInputIndex, __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Add Anchor anchor between aipp data node and aipp failed!");
return INTERNAL_ERROR;
}

27
ge/graph/preprocess/insert_op/util_insert_aipp_op.cc
Unescape View File

@ -306,6 +306,9 @@ Status InsertNewOpUtil::FindMaxSizeNode(const ComputeGraphPtr &graph, const Node
for (const auto &name : func_desc->GetSubgraphInstanceNames()) {
const auto &subgraph = graph->GetSubgraph(name);
if (subgraph == nullptr) {
REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid "
"when InsertNewOpUtil %s", name.c_str(), func_desc->GetName().c_str(),
func_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__);
GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str());
return GE_GRAPH_EMPTY_SUBGRAPH;
}
@ -325,6 +328,9 @@ Status InsertNewOpUtil::FindMaxSizeNode(const ComputeGraphPtr &graph, const Node
uint32_t parent_index = 0;
if (!AttrUtils::GetInt(src_op, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed when InsertNewOpUtil %s",
ATTR_NAME_PARENT_NODE_INDEX.c_str(),
src_op->GetName().c_str(), src_op->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "Parent index not found, name: %s", src_op->GetName().c_str());
return FAILED;
}
@ -376,12 +382,16 @@ Status InsertNewOpUtil::UpdateCaseNode(const ComputeGraphPtr &graph, const NodeP
auto ret = data_opdesc->UpdateOutputDesc(0, *input_desc);
if (ret != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Update OutputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s",
data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to update data %s output using case %s", data->GetName().c_str(),
case_node->GetName().c_str());
return INTERNAL_ERROR;
}
ret = data_opdesc->UpdateInputDesc(0, *input_desc);
if (ret != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Update InputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s",
data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to update data %s input using case %s", data->GetName().c_str(),
case_node->GetName().c_str());
return INTERNAL_ERROR;
@ -404,11 +414,15 @@ Status InsertNewOpUtil::UpdatePrevNodeByAipp(NodePtr &node, std::set<NodePtr> &s
int64_t size = 0;
graphStatus graph_ret = ge::TensorUtils::GetSize(*aipp_input, size);
if (graph_ret != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Get input size of op:%s(%s), index:0, failed, when InsertNewOpUtil %s",
aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "UpdateOutputDesc fail, graph_ret:%d", graph_ret);
return FAILED;
}
GELOGI("Get input size [%ld] from aipp [%s].", size, aipp_op_desc->GetName().c_str());
if (size == 0) {
REPORT_CALL_ERROR("E19999", "Tensor size of op:%s(%s) is 0, input_index:0, check invalid when InsertNewOpUtil %s",
aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "Can not get size from aipp [%s]", aipp_op_desc->GetName().c_str());
return FAILED;
}
@ -495,12 +509,16 @@ Status InsertNewOpUtil::UpdateDataBySwitchN(const NodePtr &switchn, const NodePt
auto ret = data_opdesc->UpdateOutputDesc(0, *input_desc);
if (ret != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Update OutputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s",
data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to update data %s output using switchn %s", data->GetName().c_str(),
switchn->GetName().c_str());
return INTERNAL_ERROR;
}
ret = data_opdesc->UpdateInputDesc(0, *input_desc);
if (ret != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Update InputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s",
data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Failed to update data %s input using switchn %s", data->GetName().c_str(),
switchn->GetName().c_str());
return INTERNAL_ERROR;
@ -600,6 +618,9 @@ Status InsertNewOpUtil::GetAllAipps(const NodePtr &data_node, const NodePtr &nod
for (const auto &name : op->GetSubgraphInstanceNames()) {
const auto &subgraph = graph->GetSubgraph(name);
if (subgraph == nullptr) {
REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid "
"when InsertNewOpUtil %s", name.c_str(), op->GetName().c_str(),
op->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__);
GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str());
return GE_GRAPH_EMPTY_SUBGRAPH;
}
@ -611,6 +632,9 @@ Status InsertNewOpUtil::GetAllAipps(const NodePtr &data_node, const NodePtr &nod
GE_CHECK_NOTNULL(src_op);
uint32_t parent_index = 0;
if (!AttrUtils::GetInt(src_op, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed when InsertNewOpUtil %s",
ATTR_NAME_PARENT_NODE_INDEX.c_str(),
src_op->GetName().c_str(), src_op->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "Parent index not found, name: %s", src_op->GetName().c_str());
return FAILED;
}
@ -750,6 +774,9 @@ Status InsertNewOpUtil::SetModelInputDims(NodePtr &data_node, NodePtr &aipp_node
}
GELOGD("After set N or H/W to -1, the model input dims: %s.", formats::JoinToString(model_input_dims).c_str());
if (!AttrUtils::SetListInt(data_opdesc, ATTR_NAME_INPUT_DIMS, model_input_dims)) {
REPORT_INNER_ERROR("E19999", "Set Attr:%s of op:%s(%s) failed when InsertNewOpUtil %s",
ATTR_NAME_INPUT_DIMS.c_str(),
data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetListInt of %s failed.", ATTR_NAME_INPUT_DIMS.c_str());
return FAILED;
}

Write Preview
Loading…
Cancel
Save