m53297601
/
graphengine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add errlog and modify geloge --2

Browse Source
pull/1389/head
liudingyan 4 years ago
parent 99b60f4f92
commit 1bcc0df932
7 changed files with 46 additions and 18 deletions
Show Stats Download Patch File Download Diff File

2
ge/hybrid/executor/hybrid_model_executor.cc
Unescape View File

@ -72,7 +72,7 @@ Status HybridModelExecutor::Execute(HybridModelExecutor::ExecuteArgs &args) {
if (ret == END_OF_SEQUENCE) {
args.is_eos = true;
} else {
GE_CHK_STATUS_RET(ret, "Failed to execute model");
GE_CHK_STATUS_RET(ret, "[Invoke][ExecuteGraphInternal]Failed when HybridModelExecutor %s.", __FUNCTION__);
}
return SUCCESS;
}

30
ge/hybrid/executor/hybrid_model_pipeline_executor.cc
Unescape View File

@ -59,7 +59,10 @@ Status StageExecutor::Start(const std::vector<TensorValue> &inputs, const std::v
task_queue_.Pop(task_info);
GELOGD("[Executor: %d] Got task, stage = %d, iteration = %ld", id_, task_info.stage, task_info.iteration);
if (task_info.iteration >= pipe_config_->iteration_end) {
GELOGE(INTERNAL_ERROR, "[Executor: %d] Unexpected iteration: %d", id_, task_info.iteration);
GELOGE(INTERNAL_ERROR, "[Check][Range][Executor: %d] Unexpected iteration: %d when StageExecutor %s.",
id_, task_info.iteration, __FUNCTION__);
REPORT_INNER_ERROR("E19999", "[Executor: %d] Unexpected iteration: %d when StageExecutor %s.",
id_, task_info.iteration, __FUNCTION__);
return INTERNAL_ERROR;
}
@ -75,7 +78,8 @@ Status StageExecutor::Start(const std::vector<TensorValue> &inputs, const std::v
if (task_info.stage == 0) {
GELOGD("[Executor: %d] To ResetExecutionContext", id_);
GE_CHK_STATUS_RET(ResetExecutionContext(context_), "[Executor: %d] Failed to reset context", id_);
GE_CHK_STATUS_RET(ResetExecutionContext(context_),
"[Invoke][ResetExecutionContext][Executor: %d] Failed to reset context", id_);
context_.iteration = task_info.iteration;
GE_CHK_STATUS_RET_NOLOG(SetInputs(inputs, input_desc));
}
@ -92,8 +96,11 @@ Status StageExecutor::Start(const std::vector<TensorValue> &inputs, const std::v
auto sync_result = Synchronize();
if (sync_result != SUCCESS) {
GELOGE(sync_result, "[Executor: %d] Failed to sync result. iteration = %d", id_, task_info.iteration);
GELOGE(sync_result,
"[Invoke][Synchronize][Executor: %d] Failed to sync result when StageExecutor %s. iteration = %d",
id_, __FUNCTION__, task_info.iteration);
REPORT_CALL_ERROR("E19999", "[Executor: %d] Failed to sync result when StageExecutor %s. iteration = %d",
id_, __FUNCTION__, task_info.iteration);
context_.profiler->Dump(std::cout);
context_.callback_manager->Destroy();
RuntimeInferenceContext::DestroyContext(std::to_string(context_.context_id));
@ -242,7 +249,10 @@ Status HybridModelPipelineExecutor::Execute(HybridModelExecutor::ExecuteArgs &ar
GELOGD("Start to sync result of executor[%zu]", i);
auto ret = futures[i].get();
if (ret != SUCCESS) {
GELOGE(ret, "[Executor: %zu] Failed to schedule tasks.", i);
GELOGE(ret, "[Check][Result][Executor: %zu] Failed to schedule tasks when HybridModelPipelineExecutor %s.",
i, __FUNCTION__);
REPORT_INNER_ERROR("E19999", "[Executor: %zu] Failed to schedule tasks when HybridModelPipelineExecutor %s.",
i, __FUNCTION__);
has_error = true;
continue;
}
@ -250,7 +260,10 @@ Status HybridModelPipelineExecutor::Execute(HybridModelExecutor::ExecuteArgs &ar
ret = stage_executors_[i]->Synchronize();
if (ret != SUCCESS) {
GELOGE(ret, "[Executor: %zu] Failed to synchronize result.", i);
GELOGE(ret, "[Invoke][Synchronize] failed for [Executor: %zu] when HybridModelPipelineExecutor %s.",
i, __FUNCTION__);
REPORT_CALL_ERROR("E19999", "[Executor: %zu] failed to Synchronize result when HybridModelPipelineExecutor %s.",
i, __FUNCTION__);
has_error = true;
continue;
}
@ -266,13 +279,14 @@ Status HybridModelPipelineExecutor::Execute(HybridModelExecutor::ExecuteArgs &ar
iteration_ = config_.iteration_end;
if (has_error) {
GELOGE(FAILED, "Error occurred while execution");
GELOGE(FAILED, "[Check][Error]Error occurred while execution when HybridModelPipelineExecutor %s.", __FUNCTION__);
REPORT_INNER_ERROR("E19999", "Error occurred while execution when HybridModelPipelineExecutor %s.", __FUNCTION__);
return FAILED;
}
auto last_iter_executor_idx = loop_count % stage_executors_.size();
GE_CHK_STATUS_RET(stage_executors_[last_iter_executor_idx]->GetOutputs(args.outputs, args.output_desc),
"Failed to get output from executor[%zu]", last_iter_executor_idx);
"[Get][Outputs]Failed from executor[%zu]", last_iter_executor_idx);
return SUCCESS;
}

8
ge/hybrid/executor/hybrid_profiler.cc
Unescape View File

@ -40,7 +40,8 @@ void HybridProfiler::RecordEvent(EventType event_type, const char *fmt, ...) {
char buf[kEventDescMax];
if (vsnprintf_s(buf, kEventDescMax, kEventDescMax - 1, fmt, args) == -1) {
GELOGE(FAILED, "Format %s failed.", fmt);
GELOGE(FAILED, "[Parse][Param:fmt]Format %s failed when HybridProfiler %s.", fmt, __FUNCTION__);
REPORT_INNER_ERROR("E19999", "Parse Format %s failed when HybridProfiler %s.", fmt, __FUNCTION__);
va_end(args);
return;
}
@ -48,7 +49,10 @@ void HybridProfiler::RecordEvent(EventType event_type, const char *fmt, ...) {
va_end(args);
auto index = counter_++;
if (index >= static_cast<int>(events_.size())) {
GELOGE(INTERNAL_ERROR, "index out of range. index = %d, max event size = %zu", index, events_.size());
GELOGE(INTERNAL_ERROR,
"[Check][Range]index out of range. index = %d, max event size = %zu", index, events_.size());
REPORT_INNER_ERROR("E19999", "index out of range when HybridProfiler %s. index = %d, max event size = %zu",
__FUNCTION__, index, events_.size());
return;
}
auto &evt = events_[index];

3
ge/hybrid/executor/node_done_manager.cc
Unescape View File

@ -28,7 +28,8 @@ bool NodeDoneManager::Cond::Await() {
if (!cv_.wait_for(lk,
std::chrono::seconds(kDefaultWaitTimeoutInSec),
[&]() { return is_released_ || is_cancelled_; })) {
GELOGE(INTERNAL_ERROR, "Wait timed out.");
GELOGE(INTERNAL_ERROR, "[Invoke][wait_for]Wait timed out when %s.", __FUNCTION__);
REPORT_INNER_ERROR("E19999", "wait timed out when %s.", __FUNCTION__);
return false;
}

17
ge/hybrid/executor/node_state.cc
Unescape View File

@ -67,7 +67,10 @@ Status ShapeInferenceState::UpdateInputShape(int idx, const GeTensorDesc &target
Format format = input_desc.GetFormat();
DataType data_type = input_desc.GetDataType();
if (TensorUtils::CalcTensorMemSize(shape, format, data_type, tensor_size) != GRAPH_SUCCESS) {
GELOGE(FAILED, "[%s] Calculate tensor memory size failed.", node_item.NodeName().c_str());
GELOGE(FAILED, "[Invoke][CalcTensorMemSize] failed for [%s] when ShapeInferenceState %s.",
node_item.NodeName().c_str(), __FUNCTION__);
REPORT_CALL_ERROR("E19999", "CalcTensorMemSize failed for [%s] when ShapeInferenceState %s.",
node_item.NodeName().c_str(), __FUNCTION__);
return FAILED;
}
}
@ -121,13 +124,19 @@ Status ShapeInferenceState::AwaitShapesReady(const GraphExecutionContext &contex
}
if (context.GetStatus() != SUCCESS) {
GELOGE(FAILED, "[%s] Await pending shape cancelled", node_item.NodeName().c_str());
GELOGE(FAILED, "[Check][Status][%s] Await pending shape cancelled when %s.",
node_item.NodeName().c_str(), __FUNCTION__);
REPORT_CALL_ERROR("E19999", "[%s] Await pending shape cancelled when %s.",
node_item.NodeName().c_str(), __FUNCTION__);
break;
}
}
if (!wait_success) {
GELOGE(FAILED, "[%s] Wait for shape timeout.", node_item.NodeName().c_str());
GELOGE(FAILED, "[Check][Status][%s] Wait for shape timeout when %s.",
node_item.NodeName().c_str(), __FUNCTION__);
REPORT_CALL_ERROR("E19999", "[%s] Wait for shape timeout when %s.",
node_item.NodeName().c_str(), __FUNCTION__);
return FAILED;
}
}
@ -232,7 +241,7 @@ Status NodeState::WaitForPrepareDone() {
if (prepare_future_.valid()) {
GELOGD("[%s] Start to wait for prepare future.", GetName().c_str());
GE_CHK_STATUS_RET(prepare_future_.get(),
"[%s] PreRun failed.", GetName().c_str());
"[Check][Status][%s] PreRun failed when NodeState %s.", GetName().c_str(), __FUNCTION__);
}
return SUCCESS;

2
metadef

@ -1 +1 @@
Subproject commit 620e9b9ac3210db3e4cf47babfb23d248bb9f17e
Subproject commit 4ff5e3987f2e5d2980019defacaf0891861c84fc

2
parser

@ -1 +1 @@
Subproject commit d744541c6ca7f6966c1befacc9f83f53b0829e0a
Subproject commit 51fb6c4850906e8342598d47eccfca0b87ffea59
Write Preview
Loading…
Cancel
Save