serving: handle exception

mindspore/ccsrc/backend/session/infer_session.cc

@@ -21,9 +21,11 @@
 #include "utils/load_onnx/anf_converter.h"
 #include "backend/session/session_basic.h"
 #include "backend/session/session_factory.h"
+#include "backend/session/executor_manager.h"
 #include "base/base_ref_utils.h"
 #include "backend/kernel_compiler/oplib/oplib.h"
 #include "utils/context/context_extends.h"
+#include "runtime/device/kernel_runtime_manager.h"
 
 #ifdef ENABLE_D
 #include "utils/ms_context.h"
@@ -236,6 +238,8 @@ Status MSInferSession::ExecuteModel(uint32_t model_id, const RequestBase &reques
 }
 
 Status MSInferSession::FinalizeEnv() {
+  session::ExecutorManager::Instance().Clear();
+  device::KernelRuntimeManager::Instance().ClearRuntimeResource();
   auto ms_context = MsContext::GetInstance();
   if (ms_context == nullptr) {
     MS_LOG(ERROR) << "Get Context failed!";

serving/core/http_process.cc

@@ -16,6 +16,8 @@
 #include <map>
 #include <vector>
 #include <string>
+#include <functional>
+#include <utility>
 #include <nlohmann/json.hpp>
 #include "serving/ms_service.pb.h"
 #include "util/status.h"
@@ -51,7 +53,7 @@ Status GetPostMessage(struct evhttp_request *req, std::string *buf) {
     return status;
   } else {
     buf->resize(post_size);
-    memcpy(buf->data(), evbuffer_pullup(req->input_buffer, -1), post_size);
+    memcpy_s(buf->data(), post_size, evbuffer_pullup(req->input_buffer, -1), post_size);
     return status;
   }
 }

serving/core/server.cc

@@ -138,14 +138,9 @@ static std::pair<struct evhttp *, struct event_base *> NewHttpServer() {
   return std::make_pair(http_server, eb);
 }
 
-Status Server::BuildAndStart() {
+Status BuildAndStartModelInner() {
-  // handle exit signal
-  signal(SIGINT, HandleSignal);
-  signal(SIGTERM, HandleSignal);
   Status res;
-
   auto option_args = Options::Instance().GetArgs();
-  std::string server_address = "0.0.0.0:" + std::to_string(option_args->grpc_port);
   std::string model_path = option_args->model_path;
   std::string model_name = option_args->model_name;
   std::string device_type = option_args->device_type;
@@ -156,7 +151,6 @@ Status Server::BuildAndStart() {
                    << device_id;
     std::cout << "Serving Error: create inference session failed, device type  " << device_type << " device id "
               << device_id << std::endl;
-    ClearEnv();
     return res;
   }
   VersionController version_controller(option_args->poll_model_wait_seconds, model_path, model_name);
@@ -166,9 +160,43 @@ Status Server::BuildAndStart() {
                    << option_args->model_name;
     std::cout << "Serving Error: load model failed, model directory " << option_args->model_path << " model name "
               << option_args->model_name << std::endl;
+    return res;
+  }
+  return SUCCESS;
+}
+
+Status BuildAndStartModel() {
+  try {
+    auto status = BuildAndStartModelInner();
+    return status;
+  } catch (const std::bad_alloc &ex) {
+    MSI_LOG(ERROR) << "Serving Error: malloc memory failed";
+    std::cout << "Serving Error: malloc memory failed" << std::endl;
+  } catch (const std::runtime_error &ex) {
+    MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what();
+    std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl;
+  } catch (const std::exception &ex) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what();
+    std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl;
+  } catch (...) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred";
+    std::cout << "Serving Error: exception occurred";
+  }
+  return FAILED;
+}
+
+Status Server::BuildAndStart() {
+  // handle exit signal
+  signal(SIGINT, HandleSignal);
+  signal(SIGTERM, HandleSignal);
+  Status res = BuildAndStartModel();
+  if (res != SUCCESS) {
     ClearEnv();
     return res;
   }
+  auto option_args = Options::Instance().GetArgs();
+  std::string server_address = "0.0.0.0:" + std::to_string(option_args->grpc_port);
+
   auto http_server_new_ret = NewHttpServer();
   struct evhttp *http_server = http_server_new_ret.first;
   struct event_base *eb = http_server_new_ret.second;

serving/core/session.cc

@@ -52,6 +52,26 @@ Session &Session::Instance() {
 }
 
 Status Session::Predict(const PredictRequest &request, PredictReply &reply) {
+  try {
+    auto status = PredictInner(request, reply);
+    return status;
+  } catch (const std::bad_alloc &ex) {
+    MSI_LOG(ERROR) << "Serving Error: malloc memory failed";
+    std::cout << "Serving Error: malloc memory failed" << std::endl;
+  } catch (const std::runtime_error &ex) {
+    MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what();
+    std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl;
+  } catch (const std::exception &ex) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what();
+    std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl;
+  } catch (...) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred";
+    std::cout << "Serving Error: exception occurred";
+  }
+  return FAILED;
+}
+
+Status Session::PredictInner(const PredictRequest &request, PredictReply &reply) {
   if (!model_loaded_) {
     MSI_LOG(ERROR) << "the model has not loaded";
     return FAILED;

serving/core/session.h

@@ -40,7 +40,6 @@ class Session {
  public:
   static Session &Instance();
   Status CreatDeviceSession(const std::string &device, uint32_t device_id);
-  // Status Predict(const inference::MultiTensor &inputs, inference::MultiTensor &output);
   Status Predict(const PredictRequest &request, PredictReply &reply);
   Status Warmup(const MindSporeModelPtr model);
   Status Clear();
@@ -55,6 +54,8 @@ class Session {
   uint32_t graph_id_{0};
   std::mutex mutex_;
   std::string device_type_;
+
+  Status PredictInner(const PredictRequest &request, PredictReply &reply);
 };
 
 }  // namespace serving

serving/core/util/status.h

@@ -19,10 +19,10 @@
 
 namespace mindspore {
 namespace serving {
-using inference::Status;
-using inference::SUCCESS;
 using inference::FAILED;
 using inference::INVALID_INPUTS;
+using inference::Status;
+using inference::SUCCESS;
 }  // namespace serving
 }  // namespace mindspore
 

serving/core/version_control/version_controller.cc

@@ -30,8 +30,8 @@ volatile bool stop_poll = false;
 
 std::string GetVersionFromPath(const std::string &path) {
   std::string new_path = path;
-  if (path.back() == '/') {
+  while (!new_path.empty() && new_path.back() == '/') {
-    new_path = path.substr(0, path.size() - 1);
+    new_path = new_path.substr(0, new_path.size() - 1);
   }
 
   std::string::size_type index = new_path.find_last_of("/");
@@ -90,8 +90,6 @@ Status VersionController::Run() {
   if (ret != SUCCESS) {
     return ret;
   }
-  // disable periodic check
-  // StartPollModelPeriodic();
   return SUCCESS;
 }
 

tests/st/serving/client_example.py

@@ -15,9 +15,9 @@
 
 import random
 import json
-import requests
 import grpc
 import numpy as np
+import requests
 import ms_service_pb2
 import ms_service_pb2_grpc
 import mindspore.dataset as de