diff --git a/mindspore/ccsrc/backend/session/infer_session.cc b/mindspore/ccsrc/backend/session/infer_session.cc index e955f70034..b2a4b36656 100644 --- a/mindspore/ccsrc/backend/session/infer_session.cc +++ b/mindspore/ccsrc/backend/session/infer_session.cc @@ -21,9 +21,11 @@ #include "utils/load_onnx/anf_converter.h" #include "backend/session/session_basic.h" #include "backend/session/session_factory.h" +#include "backend/session/executor_manager.h" #include "base/base_ref_utils.h" #include "backend/kernel_compiler/oplib/oplib.h" #include "utils/context/context_extends.h" +#include "runtime/device/kernel_runtime_manager.h" #ifdef ENABLE_D #include "utils/ms_context.h" @@ -236,6 +238,8 @@ Status MSInferSession::ExecuteModel(uint32_t model_id, const RequestBase &reques } Status MSInferSession::FinalizeEnv() { + session::ExecutorManager::Instance().Clear(); + device::KernelRuntimeManager::Instance().ClearRuntimeResource(); auto ms_context = MsContext::GetInstance(); if (ms_context == nullptr) { MS_LOG(ERROR) << "Get Context failed!"; diff --git a/serving/core/http_process.cc b/serving/core/http_process.cc index 290359e7a9..e0b302574a 100644 --- a/serving/core/http_process.cc +++ b/serving/core/http_process.cc @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include "serving/ms_service.pb.h" #include "util/status.h" @@ -51,7 +53,7 @@ Status GetPostMessage(struct evhttp_request *req, std::string *buf) { return status; } else { buf->resize(post_size); - memcpy(buf->data(), evbuffer_pullup(req->input_buffer, -1), post_size); + memcpy_s(buf->data(), post_size, evbuffer_pullup(req->input_buffer, -1), post_size); return status; } } diff --git a/serving/core/server.cc b/serving/core/server.cc index 57842c8ccb..ae667ade55 100644 --- a/serving/core/server.cc +++ b/serving/core/server.cc @@ -138,14 +138,9 @@ static std::pair NewHttpServer() { return std::make_pair(http_server, eb); } -Status Server::BuildAndStart() { - // handle exit signal - signal(SIGINT, HandleSignal); - signal(SIGTERM, HandleSignal); +Status BuildAndStartModelInner() { Status res; - auto option_args = Options::Instance().GetArgs(); - std::string server_address = "0.0.0.0:" + std::to_string(option_args->grpc_port); std::string model_path = option_args->model_path; std::string model_name = option_args->model_name; std::string device_type = option_args->device_type; @@ -156,7 +151,6 @@ Status Server::BuildAndStart() { << device_id; std::cout << "Serving Error: create inference session failed, device type " << device_type << " device id " << device_id << std::endl; - ClearEnv(); return res; } VersionController version_controller(option_args->poll_model_wait_seconds, model_path, model_name); @@ -166,9 +160,43 @@ Status Server::BuildAndStart() { << option_args->model_name; std::cout << "Serving Error: load model failed, model directory " << option_args->model_path << " model name " << option_args->model_name << std::endl; + return res; + } + return SUCCESS; +} + +Status BuildAndStartModel() { + try { + auto status = BuildAndStartModelInner(); + return status; + } catch (const std::bad_alloc &ex) { + MSI_LOG(ERROR) << "Serving Error: malloc memory failed"; + std::cout << "Serving Error: malloc memory failed" << std::endl; + } catch (const std::runtime_error &ex) { + MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what(); + std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl; + } catch (const std::exception &ex) { + MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what(); + std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl; + } catch (...) { + MSI_LOG(ERROR) << "Serving Error: exception occurred"; + std::cout << "Serving Error: exception occurred"; + } + return FAILED; +} + +Status Server::BuildAndStart() { + // handle exit signal + signal(SIGINT, HandleSignal); + signal(SIGTERM, HandleSignal); + Status res = BuildAndStartModel(); + if (res != SUCCESS) { ClearEnv(); return res; } + auto option_args = Options::Instance().GetArgs(); + std::string server_address = "0.0.0.0:" + std::to_string(option_args->grpc_port); + auto http_server_new_ret = NewHttpServer(); struct evhttp *http_server = http_server_new_ret.first; struct event_base *eb = http_server_new_ret.second; diff --git a/serving/core/session.cc b/serving/core/session.cc index 9c0ba32f85..a5204b6038 100644 --- a/serving/core/session.cc +++ b/serving/core/session.cc @@ -52,6 +52,26 @@ Session &Session::Instance() { } Status Session::Predict(const PredictRequest &request, PredictReply &reply) { + try { + auto status = PredictInner(request, reply); + return status; + } catch (const std::bad_alloc &ex) { + MSI_LOG(ERROR) << "Serving Error: malloc memory failed"; + std::cout << "Serving Error: malloc memory failed" << std::endl; + } catch (const std::runtime_error &ex) { + MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what(); + std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl; + } catch (const std::exception &ex) { + MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what(); + std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl; + } catch (...) { + MSI_LOG(ERROR) << "Serving Error: exception occurred"; + std::cout << "Serving Error: exception occurred"; + } + return FAILED; +} + +Status Session::PredictInner(const PredictRequest &request, PredictReply &reply) { if (!model_loaded_) { MSI_LOG(ERROR) << "the model has not loaded"; return FAILED; diff --git a/serving/core/session.h b/serving/core/session.h index ae50f6f78f..08168f30d0 100644 --- a/serving/core/session.h +++ b/serving/core/session.h @@ -40,7 +40,6 @@ class Session { public: static Session &Instance(); Status CreatDeviceSession(const std::string &device, uint32_t device_id); - // Status Predict(const inference::MultiTensor &inputs, inference::MultiTensor &output); Status Predict(const PredictRequest &request, PredictReply &reply); Status Warmup(const MindSporeModelPtr model); Status Clear(); @@ -55,6 +54,8 @@ class Session { uint32_t graph_id_{0}; std::mutex mutex_; std::string device_type_; + + Status PredictInner(const PredictRequest &request, PredictReply &reply); }; } // namespace serving diff --git a/serving/core/util/status.h b/serving/core/util/status.h index e1f1df6874..7ed33a57df 100644 --- a/serving/core/util/status.h +++ b/serving/core/util/status.h @@ -19,10 +19,10 @@ namespace mindspore { namespace serving { -using inference::Status; -using inference::SUCCESS; using inference::FAILED; using inference::INVALID_INPUTS; +using inference::Status; +using inference::SUCCESS; } // namespace serving } // namespace mindspore diff --git a/serving/core/version_control/version_controller.cc b/serving/core/version_control/version_controller.cc index 41035f68e0..59e98eb8af 100644 --- a/serving/core/version_control/version_controller.cc +++ b/serving/core/version_control/version_controller.cc @@ -30,8 +30,8 @@ volatile bool stop_poll = false; std::string GetVersionFromPath(const std::string &path) { std::string new_path = path; - if (path.back() == '/') { - new_path = path.substr(0, path.size() - 1); + while (!new_path.empty() && new_path.back() == '/') { + new_path = new_path.substr(0, new_path.size() - 1); } std::string::size_type index = new_path.find_last_of("/"); @@ -90,8 +90,6 @@ Status VersionController::Run() { if (ret != SUCCESS) { return ret; } - // disable periodic check - // StartPollModelPeriodic(); return SUCCESS; } diff --git a/tests/st/serving/client_example.py b/tests/st/serving/client_example.py index 560dd72c07..79e046c9b0 100644 --- a/tests/st/serving/client_example.py +++ b/tests/st/serving/client_example.py @@ -15,9 +15,9 @@ import random import json -import requests import grpc import numpy as np +import requests import ms_service_pb2 import ms_service_pb2_grpc import mindspore.dataset as de