init serving

CMakeLists.txt

@@ -96,4 +96,8 @@ if (ENABLE_TESTCASES)
     add_subdirectory(tests)
 endif()
 
+if (ENABLE_SERVING)
+    add_subdirectory(serving)
+endif()
+
 include(cmake/package.cmake)

build.sh

@@ -53,6 +53,7 @@ usage()
   echo "    -V Specify the minimum required cuda version, default CUDA 9.2"
   echo "    -I Compile predict, default off"
   echo "    -K Compile with AKG, default off"
+  echo "    -s Enable serving module, default off"
 }
 
 # check value of input is 'on' or 'off'
@@ -92,9 +93,9 @@ checkopts()
   USE_GLOG="on"
   PREDICT_PLATFORM=""
   ENABLE_AKG="off"
-
+  ENABLE_SERVING="off"
   # Process the options
-  while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K' opt
+  while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:s' opt
   do
     OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
     case "${opt}" in
@@ -235,6 +236,10 @@ checkopts()
         ENABLE_AKG="on"
         echo "enable compile with akg"
         ;;
+      s)
+        ENABLE_SERVING="on"
+        echo "enable serving"
+        ;;
       *)
         echo "Unknown option ${opt}!"
         usage
@@ -314,6 +319,10 @@ build_mindspore()
     if [[ "X$ENABLE_AKG" = "Xon" ]] && [[ "X$ENABLE_D" = "Xon" ]]; then
         CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_AKG=ON"
     fi
+    if [[ "X$ENABLE_SERVING" = "Xon" ]]; then
+        CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_SERVING=ON"
+    fi
+
     echo "${CMAKE_ARGS}"
     if [[ "X$INC_BUILD" = "Xoff" ]]; then
       cmake ${CMAKE_ARGS} ../..

include/inference.h

@@ -37,6 +37,8 @@ class MS_API MSSession {
 };
 
 std::shared_ptr<FuncGraph> MS_API LoadModel(const char *model_buf, size_t size, const std::string &device);
+
+void MS_API ExitInference();
 }  // namespace inference
 }  // namespace mindspore
 #endif  // MINDSPORE_INCLUDE_MS_SESSION_H

mindspore/ccsrc/CMakeLists.txt

@@ -247,7 +247,7 @@ add_library(inference SHARED
         ${CMAKE_CURRENT_SOURCE_DIR}/session/session.cc
         ${LOAD_ONNX_SRC}
         )
-target_link_libraries(inference PRIVATE ${PYTHON_LIB} ${SECUREC_LIBRARY}
+target_link_libraries(inference PRIVATE ${PYTHON_LIBRARY} ${SECUREC_LIBRARY}
         -Wl,--whole-archive mindspore -Wl,--no-whole-archive mindspore_gvar mindspore::protobuf)
 
 if (ENABLE_CPU)

mindspore/ccsrc/session/session.cc

@@ -38,6 +38,18 @@ std::shared_ptr<FuncGraph> LoadModel(const char *model_buf, size_t size, const s
   return anf_graph;
 }
 
+void ExitInference() {
+  auto ms_context = MsContext::GetInstance();
+  if (ms_context == nullptr) {
+    MS_LOG(ERROR) << "Get Context failed!";
+    return;
+  }
+  if (!ms_context->CloseTsd()) {
+    MS_LOG(ERROR) << "Inference CloseTsd failed!";
+    return;
+  }
+}
+
 std::shared_ptr<MSSession> MSSession::CreateSession(const std::string &device, uint32_t device_id) {
   auto session = std::make_shared<inference::Session>();
   auto ret = session->Init(device, device_id);
@@ -101,11 +113,14 @@ void Session::RegAllOp() {
 
 uint32_t Session::CompileGraph(std::shared_ptr<FuncGraph> funcGraphPtr) {
   MS_ASSERT(session_impl_ != nullptr);
-  return session_impl_->CompileGraph(NOT_NULL(funcGraphPtr));
+  auto graph_id = session_impl_->CompileGraph(NOT_NULL(funcGraphPtr));
+  py::gil_scoped_release gil_release;
+  return graph_id;
 }
 
 MultiTensor Session::RunGraph(uint32_t graph_id, const std::vector<std::shared_ptr<inference::MSTensor>> &inputs) {
   std::vector<tensor::TensorPtr> inTensors;
+  inTensors.resize(inputs.size());
   bool has_error = false;
   std::transform(inputs.begin(), inputs.end(), inTensors.begin(),
                  [&has_error](const std::shared_ptr<inference::MSTensor> &tensor_ptr) -> tensor::TensorPtr {
@@ -144,6 +159,14 @@ int Session::Init(const std::string &device, uint32_t device_id) {
     return -1;
   }
   session_impl_->Init(device_id);
+  if (ms_context == nullptr) {
+    MS_LOG(ERROR) << "Get Context failed!";
+    return -1;
+  }
+  if (!ms_context->OpenTsd()) {
+    MS_LOG(ERROR) << "Session init OpenTsd failed!";
+    return -1;
+  }
   return 0;
 }
 

serving/CMakeLists.txt

@@ -0,0 +1,69 @@
+find_package(Threads REQUIRED)
+
+# This branch assumes that gRPC and all its dependencies are already installed
+# on this system, so they can be located by find_package().
+
+# Find Protobuf installation
+# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation.
+
+#set(protobuf_MODULE_COMPATIBLE TRUE)
+#find_package(Protobuf CONFIG REQUIRED)
+#message(STATUS "Using protobuf ${protobuf_VERSION}")
+add_library(protobuf::libprotobuf ALIAS protobuf::protobuf)
+add_executable(protobuf::libprotoc ALIAS protobuf::protoc)
+
+set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
+set(_REFLECTION gRPC::grpc++_reflection)
+if(CMAKE_CROSSCOMPILING)
+    find_program(_PROTOBUF_PROTOC protoc)
+else()
+    set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+endif()
+
+# Find gRPC installation
+# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
+find_package(gRPC CONFIG REQUIRED)
+message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+set(_GRPC_GRPCPP gRPC::grpc++)
+if(CMAKE_CROSSCOMPILING)
+    find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+else()
+    set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+endif()
+
+# Proto file
+get_filename_component(hw_proto "ms_service.proto" ABSOLUTE)
+get_filename_component(hw_proto_path "${hw_proto}" PATH)
+
+# Generated sources
+set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.cc")
+set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.h")
+set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.cc")
+set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.h")
+add_custom_command(
+        OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}"
+        COMMAND ${_PROTOBUF_PROTOC}
+        ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${hw_proto_path}"
+        --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+        "${hw_proto}"
+        DEPENDS "${hw_proto}")
+
+# Include generated *.pb.h files
+include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/core"
+        "${PROJECT_SOURCE_DIR}/mindspore/ccsrc")
+file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+        "core/*.cc" "core/util/*.cc" "core/version_control/*.cc")
+
+list(APPEND SERVING_SRC "main.cc" ${hw_proto_srcs} ${hw_grpc_srcs} ${CORE_SRC_LIST})
+
+include_directories(${CMAKE_BINARY_DIR})
+add_executable(ms_serving ${SERVING_SRC})
+target_link_libraries(ms_serving inference mindspore_gvar)
+target_link_libraries(ms_serving ${_REFLECTION} ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} pthread)
+if (ENABLE_D)
+    add_compile_definitions(ENABLE_D)
+    target_link_libraries(ms_serving ${RUNTIME_LIB})
+endif()

serving/README.en.md

@@ -0,0 +1,36 @@
+# serving
+
+#### Description
+A flexible, high-performance serving system for deep learning models
+
+#### Software Architecture
+Software architecture description
+
+#### Installation
+
+1.  xxxx
+2.  xxxx
+3.  xxxx
+
+#### Instructions
+
+1.  xxxx
+2.  xxxx
+3.  xxxx
+
+#### Contribution
+
+1.  Fork the repository
+2.  Create Feat_xxx branch
+3.  Commit your code
+4.  Create Pull Request
+
+
+#### Gitee Feature
+
+1.  You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md
+2.  Gitee blog [blog.gitee.com](https://blog.gitee.com)
+3.  Explore open source project [https://gitee.com/explore](https://gitee.com/explore)
+4.  The most valuable open source project [GVP](https://gitee.com/gvp)
+5.  The manual of Gitee [https://gitee.com/help](https://gitee.com/help)
+6.  The most popular members  [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)

serving/README.md

@@ -0,0 +1,37 @@
+# serving
+
+#### 介绍
+A flexible, high-performance serving system for deep learning models
+
+#### 软件架构
+软件架构说明
+
+
+#### 安装教程
+
+1.  xxxx
+2.  xxxx
+3.  xxxx
+
+#### 使用说明
+
+1.  xxxx
+2.  xxxx
+3.  xxxx
+
+#### 参与贡献
+
+1.  Fork 本仓库
+2.  新建 Feat_xxx 分支
+3.  提交代码
+4.  新建 Pull Request
+
+
+#### 码云特技
+
+1.  使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md
+2.  码云官方博客 [blog.gitee.com](https://blog.gitee.com)
+3.  你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解码云上的优秀开源项目
+4.  [GVP](https://gitee.com/gvp) 全称是码云最有价值开源项目，是码云综合评定出的优秀开源项目
+5.  码云官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help)
+6.  码云封面人物是一档用来展示码云会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)

serving/core/server.h

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_SERVER_H
+#define MINDSPORE_SERVER_H
+
+#include <string>
+#include <mutex>
+#include <vector>
+#include <memory>
+#include "util/status.h"
+#include "version_control/model.h"
+#include "include/inference.h"
+#include "mindspore/ccsrc/debug/info.h"
+namespace mindspore {
+namespace serving {
+class Session {
+ public:
+  static Session &Instance();
+  Status CreatDeviceSession(const std::string &device, uint32_t device_id);
+  Status Predict(const std::vector<std::shared_ptr<inference::MSTensor>> &inputs, inference::MultiTensor *output);
+  Status Warmup(const MindSporeModelPtr model);
+  Status Clear();
+
+ private:
+  Session() = default;
+  ~Session() = default;
+  int sesseion_id_{0};
+  std::shared_ptr<inference::MSSession> session_{nullptr};
+  FuncGraphPtr last_graph_{nullptr};
+  uint32_t graph_id_{0};
+  std::mutex mutex_;
+  std::string device_type_;
+};
+
+class Server {
+ public:
+  Server() = default;
+  ~Server() = default;
+  Status BuildAndStart();
+};
+}  // namespace serving
+}  // namespace mindspore
+#endif  // MINDSPORE_SERVER_H

serving/core/util/file_system_operation.cc

@@ -0,0 +1,102 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/util/file_system_operation.h"
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string>
+#include <vector>
+#include <iostream>
+#include <algorithm>
+#include <ctime>
+#include <fstream>
+#include <memory>
+#include "mindspore/ccsrc/utils/log_adapter.h"
+
+namespace mindspore {
+namespace serving {
+
+char *ReadFile(const char *file, size_t *size) {
+  if (file == nullptr) {
+    MS_LOG(ERROR) << "file is nullptr";
+    return nullptr;
+  }
+  MS_ASSERT(size != nullptr);
+  std::string realPath = file;
+  std::ifstream ifs(realPath);
+  if (!ifs.good()) {
+    MS_LOG(ERROR) << "file: " << realPath << " is not exist";
+    return nullptr;
+  }
+
+  if (!ifs.is_open()) {
+    MS_LOG(ERROR) << "file: " << realPath << "open failed";
+    return nullptr;
+  }
+
+  ifs.seekg(0, std::ios::end);
+  *size = ifs.tellg();
+  std::unique_ptr<char> buf(new (std::nothrow) char[*size]);
+  if (buf == nullptr) {
+    MS_LOG(ERROR) << "malloc buf failed, file: " << realPath;
+    ifs.close();
+    return nullptr;
+  }
+
+  ifs.seekg(0, std::ios::beg);
+  ifs.read(buf.get(), *size);
+  ifs.close();
+
+  return buf.release();
+}
+
+bool DirOrFileExist(const std::string &file_path) {
+  int ret = access(file_path.c_str(), 0);
+  return (ret == -1) ? false : true;
+}
+
+std::vector<std::string> GetAllSubDirs(const std::string &dir_path) {
+  DIR *dir;
+  struct dirent *ptr;
+  std::vector<std::string> SubDirs;
+
+  if ((dir = opendir(dir_path.c_str())) == NULL) {
+    MS_LOG(ERROR) << "Open " << dir_path << " error!";
+    return std::vector<std::string>();
+  }
+
+  while ((ptr = readdir(dir)) != NULL) {
+    std::string name = ptr->d_name;
+    if (name == "." || name == "..") {
+      continue;
+    }
+    if (ptr->d_type == DT_DIR) {
+      SubDirs.push_back(dir_path + "/" + name);
+    }
+  }
+  closedir(dir);
+  std::sort(SubDirs.begin(), SubDirs.end());
+  return SubDirs;
+}
+
+time_t GetModifyTime(const std::string &file_path) {
+  struct stat info;
+  (void)stat(file_path.c_str(), &info);
+  return info.st_mtime;
+}
+}  // namespace serving
+}  // namespace mindspore

serving/core/util/file_system_operation.h

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_
+#define MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_
+
+#include <string>
+#include <vector>
+#include <ctime>
+
+namespace mindspore {
+namespace serving {
+char *ReadFile(const char *file, size_t *size);
+bool DirOrFileExist(const std::string &file_path);
+std::vector<std::string> GetAllSubDirs(const std::string &dir_path);
+time_t GetModifyTime(const std::string &file_path);
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // !MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_

serving/core/util/option_parser.cc

@@ -0,0 +1,243 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/util/option_parser.h"
+#include <vector>
+#include <string>
+#include <cstring>
+#include <iostream>
+#include <iomanip>
+#include "mindspore/ccsrc/utils/log_adapter.h"
+
+namespace mindspore {
+namespace serving {
+bool StartWith(const std::string &str, const std::string &expected) {
+  return expected.empty() ||
+         (str.size() >= expected.size() && memcmp(str.data(), expected.data(), expected.size()) == 0);
+}
+
+bool RemovePrefix(std::string *str, const std::string &prefix) {
+  if (!StartWith(*str, prefix)) return false;
+  str->replace(str->begin(), str->begin() + prefix.size(), "");
+  return true;
+}
+
+bool Option::ParseInt32(std::string *arg) {
+  if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) {
+    char extra;
+    int32_t parsed_value;
+    if (sscanf(arg->data(), "%d%c", &parsed_value, &extra) != 1) {
+      std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl;
+      return false;
+    } else {
+      *int32_default_ = parsed_value;
+    }
+    return true;
+  }
+
+  return false;
+}
+
+bool Option::ParseBool(std::string *arg) {
+  if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) {
+    if (*arg == "true") {
+      *bool_default_ = true;
+    } else if (*arg == "false") {
+      *bool_default_ = false;
+    } else {
+      std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl;
+      return false;
+    }
+    return true;
+  }
+
+  return false;
+}
+
+bool Option::ParseString(std::string *arg) {
+  if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) {
+    *string_default_ = *arg;
+    return true;
+  }
+  return false;
+}
+
+bool Option::ParseFloat(std::string *arg) {
+  if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) {
+    char extra;
+    float parsed_value;
+    if (sscanf(arg->data(), "%f%c", &parsed_value, &extra) != 1) {
+      std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl;
+      return false;
+    } else {
+      *float_default_ = parsed_value;
+    }
+    return true;
+  }
+
+  return false;
+}
+
+Option::Option(const std::string &name, int32_t *default_point, const std::string &usage)
+    : name_(name),
+      type_(MS_TYPE_INT32),
+      int32_default_(default_point),
+      bool_default_(nullptr),
+      string_default_(nullptr),
+      float_default_(nullptr),
+      usage_(usage) {}
+
+Option::Option(const std::string &name, bool *default_point, const std::string &usage)
+    : name_(name),
+      type_(MS_TYPE_BOOL),
+      int32_default_(nullptr),
+      bool_default_(default_point),
+      string_default_(nullptr),
+      float_default_(nullptr),
+      usage_(usage) {}
+
+Option::Option(const std::string &name, std::string *default_point, const std::string &usage)
+    : name_(name),
+      type_(MS_TYPE_STRING),
+      int32_default_(nullptr),
+      bool_default_(nullptr),
+      string_default_(default_point),
+      float_default_(nullptr),
+      usage_(usage) {}
+
+Option::Option(const std::string &name, float *default_point, const std::string &usage)
+    : name_(name),
+      type_(MS_TYPE_FLOAT),
+      int32_default_(nullptr),
+      bool_default_(nullptr),
+      string_default_(nullptr),
+      float_default_(default_point),
+      usage_(usage) {}
+
+bool Option::Parse(std::string *arg) {
+  bool result = false;
+  switch (type_) {
+    case MS_TYPE_BOOL:
+      result = ParseBool(arg);
+      break;
+    case MS_TYPE_FLOAT:
+      result = ParseFloat(arg);
+      break;
+    case MS_TYPE_INT32:
+      result = ParseInt32(arg);
+      break;
+    case MS_TYPE_STRING:
+      result = ParseString(arg);
+      break;
+    default:
+      break;
+  }
+  return result;
+}
+
+std::shared_ptr<Options> Options::inst_ = nullptr;
+
+Options &Options::Instance() {
+  static Options instance;
+  return instance;
+}
+
+Options::Options() : args_(nullptr) { CreateOptions(); }
+
+void Options::CreateOptions() {
+  args_ = std::make_shared<Arguments>();
+  std::vector<Option> options = {
+    Option("port", &args_->grpc_port, "Port to listen on for gRPC API, default is 5500"),
+    Option("model_name", &args_->model_name, "model name "),
+    Option("model_path", &args_->model_path, "the path of the model files"),
+    Option("device_id", &args_->device_id, "the device id, default is 0"),
+  };
+  options_ = options;
+}
+
+bool Options::CheckOptions() {
+  if (args_->model_name == "" || args_->model_path == "") {
+    std::cout << "model_path and model_name should not be null" << std::endl;
+    return false;
+  }
+  if (args_->device_type != "Ascend") {
+    std::cout << "device_type only support Ascend right now" << std::endl;
+    return false;
+  }
+  return true;
+}
+
+bool Options::ParseCommandLine(int argc, char **argv) {
+  if (argc < 2 || (strcmp(argv[1], "--help") == 0)) {
+    Usage();
+    return false;
+  }
+  std::vector<std::string> unkown_options;
+  for (int i = 1; i < argc; ++i) {
+    bool found = false;
+    for (auto &option : options_) {
+      std::string arg = argv[i];
+      if (option.Parse(&arg)) {
+        found = true;
+        break;
+      }
+    }
+
+    if (found == false) {
+      unkown_options.push_back(argv[i]);
+    }
+  }
+
+  if (!unkown_options.empty()) {
+    std::cout << "unkown options:" << std::endl;
+    for (const auto &option : unkown_options) {
+      std::cout << option << std::endl;
+    }
+  }
+  bool valid = (unkown_options.empty() && CheckOptions());
+  if (!valid) {
+    Usage();
+  }
+  return valid;
+}
+
+void Options::Usage() {
+  std::cout << "USAGE: mindspore-serving [options]" << std::endl;
+
+  for (const auto &option : options_) {
+    std::string type;
+    switch (option.type_) {
+      case Option::MS_TYPE_BOOL:
+        type = "bool";
+        break;
+      case Option::MS_TYPE_FLOAT:
+        type = "float";
+        break;
+      case Option::MS_TYPE_INT32:
+        type = "int32";
+        break;
+      case Option::MS_TYPE_STRING:
+        type = "string";
+        break;
+      default:
+        break;
+    }
+    std::cout << "--" << std::setw(30) << std::left << option.name_ << std::setw(10) << std::left << type
+              << option.usage_ << std::endl;
+  }
+}
+
+}  // namespace serving
+}  // namespace mindspore

serving/core/util/option_parser.h

@@ -0,0 +1,84 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_SERVING_OPTION_PARSER_H_
+#define MINDSPORE_SERVING_OPTION_PARSER_H_
+
+#include <string>
+#include <vector>
+#include <memory>
+
+namespace mindspore {
+namespace serving {
+
+struct Arguments {
+  int32_t grpc_port = 5500;
+  std::string grpc_socket_path;
+  std::string ssl_config_file;
+  int32_t poll_model_wait_seconds = 1;
+  std::string model_name;
+  std::string model_path;
+  std::string device_type = "Ascend";
+  int32_t device_id = 0;
+};
+
+class Option {
+ public:
+  Option(const std::string &name, int32_t *default_point, const std::string &usage);
+  Option(const std::string &name, bool *default_point, const std::string &usage);
+  Option(const std::string &name, std::string *default_point, const std::string &usage);
+  Option(const std::string &name, float *default_point, const std::string &usage);
+
+ private:
+  friend class Options;
+
+  bool ParseInt32(std::string *arg);
+  bool ParseBool(std::string *arg);
+  bool ParseString(std::string *arg);
+  bool ParseFloat(std::string *arg);
+  bool Parse(std::string *arg);
+  std::string name_;
+  enum { MS_TYPE_INT32, MS_TYPE_BOOL, MS_TYPE_STRING, MS_TYPE_FLOAT } type_;
+  int32_t *int32_default_;
+  bool *bool_default_;
+  std::string *string_default_;
+  float *float_default_;
+  std::string usage_;
+};
+
+class Options {
+ public:
+  ~Options() = default;
+  Options(const Options &) = delete;
+  Options &operator=(const Options &) = delete;
+  static Options &Instance();
+  bool ParseCommandLine(int argc, char **argv);
+  void Usage();
+  std::shared_ptr<Arguments> GetArgs() { return args_; }
+
+ private:
+  Options();
+  void CreateOptions();
+  bool CheckOptions();
+  static std::shared_ptr<Options> inst_;
+  std::string usage_;
+  std::vector<Option> options_;
+  std::shared_ptr<Arguments> args_;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif

serving/core/util/status.h

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_STATUS_H
+#define MINDSPORE_STATUS_H
+namespace mindspore {
+namespace serving {
+using Status = uint32_t;
+enum ServingStatus { SUCCESS = 0, FAILED };
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_STATUS_H

serving/core/version_control/model.cc

@@ -0,0 +1,33 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/version_control/model.h"
+#include <string>
+#include "mindspore/ccsrc/utils/log_adapter.h"
+
+namespace mindspore {
+namespace serving {
+
+MindSporeModel::MindSporeModel(const std::string &model_name, const std::string &model_path,
+                               const std::string &model_version, const time_t &last_update_time)
+    : model_name_(model_name),
+      model_path_(model_path),
+      model_version_(model_version),
+      last_update_time_(last_update_time) {
+  MS_LOG(INFO) << "init mindspore model, model_name = " << model_name_ << ", model_path = " << model_path_
+               << ", model_version = " << model_version_ << ", last_update_time = " << last_update_time_;
+}
+}  // namespace serving
+}  // namespace mindspore

serving/core/version_control/model.h

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_SERVING_MODEL_H_
+#define MINDSPORE_SERVING_MODEL_H_
+
+#include <string>
+#include <ctime>
+#include <memory>
+
+namespace mindspore {
+namespace serving {
+class MindSporeModel {
+ public:
+  MindSporeModel(const std::string &model_name, const std::string &model_path, const std::string &model_version,
+                 const time_t &last_update_time);
+  ~MindSporeModel() = default;
+  std::string GetModelName() { return model_name_; }
+  std::string GetModelPath() { return model_path_; }
+  std::string GetModelVersion() { return model_version_; }
+  time_t GetLastUpdateTime() { return last_update_time_; }
+  void SetLastUpdateTime(const time_t &last_update_time) { last_update_time_ = last_update_time; }
+
+ private:
+  std::string model_name_;
+  std::string model_path_;
+  std::string model_version_;
+  time_t last_update_time_;
+};
+
+using MindSporeModelPtr = std::shared_ptr<MindSporeModel>;
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // !MINDSPORE_SERVING_MODEL_H_

serving/core/version_control/version_controller.cc

@@ -0,0 +1,134 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/version_control/version_controller.h"
+
+#include <string>
+#include <iostream>
+#include <ctime>
+#include <memory>
+#include "util/file_system_operation.h"
+#include "mindspore/ccsrc/utils/log_adapter.h"
+#include "core/server.h"
+
+namespace mindspore {
+namespace serving {
+
+volatile bool stop_poll = false;
+
+std::string GetVersionFromPath(const std::string &path) {
+  std::string new_path = path;
+  if (path.back() == '/') {
+    new_path = path.substr(0, path.size() - 1);
+  }
+
+  std::string::size_type index = new_path.find_last_of("/");
+  std::string version = new_path.substr(index + 1);
+  return version;
+}
+
+void PeriodicFunction::operator()() {
+  while (true) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(poll_model_wait_seconds_ * 1000));
+    std::vector<std::string> SubDirs = GetAllSubDirs(models_path_);
+
+    if (version_control_strategy_ == VersionController::VersionControllerStrategy::kLastest) {
+      auto path = SubDirs.empty() ? models_path_ : SubDirs.back();
+      std::string model_version = GetVersionFromPath(path);
+      time_t last_update_time = GetModifyTime(path);
+      if (model_version != valid_models_.back()->GetModelVersion()) {
+        MindSporeModelPtr model_ptr = std::make_shared<MindSporeModel>(valid_models_.front()->GetModelName(), path,
+                                                                       model_version, last_update_time);
+        valid_models_.back() = model_ptr;
+        Session::Instance().Warmup(valid_models_.back());
+      } else {
+        if (difftime(valid_models_.back()->GetLastUpdateTime(), last_update_time) < 0) {
+          valid_models_.back()->SetLastUpdateTime(last_update_time);
+        }
+      }
+    } else {
+      // not support
+    }
+
+    if (stop_poll == true) {
+      break;
+    }
+  }
+}
+
+VersionController::VersionController(int32_t poll_model_wait_seconds, const std::string &models_path,
+                                     const std::string &model_name)
+    : version_control_strategy_(kLastest),
+      poll_model_wait_seconds_(poll_model_wait_seconds),
+      models_path_(models_path),
+      model_name_(model_name) {}
+
+void StopPollModelPeriodic() { stop_poll = true; }
+
+VersionController::~VersionController() {
+  StopPollModelPeriodic();
+  if (poll_model_thread_.joinable()) {
+    poll_model_thread_.join();
+  }
+}
+
+Status VersionController::Run() {
+  Status ret;
+  ret = CreateInitModels();
+  if (ret != SUCCESS) {
+    return ret;
+  }
+  // disable periodic check
+  // StartPollModelPeriodic();
+  return SUCCESS;
+}
+
+Status VersionController::CreateInitModels() {
+  if (!DirOrFileExist(models_path_)) {
+    MS_LOG(ERROR) << "Model Path Not Exist!" << std::endl;
+    return FAILED;
+  }
+  std::vector<std::string> SubDirs = GetAllSubDirs(models_path_);
+  if (version_control_strategy_ == kLastest) {
+    auto path = SubDirs.empty() ? models_path_ : SubDirs.back();
+    std::string model_version = GetVersionFromPath(path);
+    time_t last_update_time = GetModifyTime(path);
+    MindSporeModelPtr model_ptr = std::make_shared<MindSporeModel>(model_name_, path, model_version, last_update_time);
+    valid_models_.emplace_back(model_ptr);
+  } else {
+    for (auto &dir : SubDirs) {
+      std::string model_version = GetVersionFromPath(dir);
+      time_t last_update_time = GetModifyTime(dir);
+      MindSporeModelPtr model_ptr = std::make_shared<MindSporeModel>(model_name_, dir, model_version, last_update_time);
+      valid_models_.emplace_back(model_ptr);
+    }
+  }
+  if (valid_models_.empty()) {
+    MS_LOG(ERROR) << "There is no valid model for serving";
+    return FAILED;
+  }
+  Session::Instance().Warmup(valid_models_.back());
+  return SUCCESS;
+}
+
+void VersionController::StartPollModelPeriodic() {
+  poll_model_thread_ = std::thread(
+    PeriodicFunction(poll_model_wait_seconds_, models_path_, version_control_strategy_, std::ref(valid_models_)));
+}
+
+void VersionController::StopPollModelPeriodic() {}
+
+}  // namespace serving
+}  // namespace mindspore

serving/core/version_control/version_controller.h

@@ -0,0 +1,71 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_SERVING_VERSOIN_CONTROLLER_H_
+#define MINDSPORE_SERVING_VERSOIN_CONTROLLER_H_
+
+#include <string>
+#include <vector>
+#include <thread>
+#include "./model.h"
+#include "util/status.h"
+
+namespace mindspore {
+namespace serving {
+class VersionController {
+ public:
+  enum VersionControllerStrategy { kLastest = 0, kMulti = 1 };
+
+  VersionController(int32_t poll_model_wait_seconds, const std::string &models_path, const std::string &model_name);
+  ~VersionController();
+  Status Run();
+  void StartPollModelPeriodic();
+  void StopPollModelPeriodic();
+
+ private:
+  Status CreateInitModels();
+
+ private:
+  VersionControllerStrategy version_control_strategy_;
+  std::vector<MindSporeModelPtr> valid_models_;
+  int32_t poll_model_wait_seconds_;
+  std::thread poll_model_thread_;
+  std::string models_path_;
+  std::string model_name_;
+};
+
+class PeriodicFunction {
+ public:
+  PeriodicFunction(int32_t poll_model_wait_seconds, const std::string &models_path,
+                   VersionController::VersionControllerStrategy version_control_strategy,
+                   const std::vector<MindSporeModelPtr> &valid_models)
+      : poll_model_wait_seconds_(poll_model_wait_seconds),
+        models_path_(models_path),
+        version_control_strategy_(version_control_strategy),
+        valid_models_(valid_models) {}
+  ~PeriodicFunction() = default;
+  void operator()();
+
+ private:
+  int32_t poll_model_wait_seconds_;
+  std::string models_path_;
+  VersionController::VersionControllerStrategy version_control_strategy_;
+  std::vector<MindSporeModelPtr> valid_models_;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // !MINDSPORE_SERVING_VERSOIN_CONTROLLER_H_

serving/cpp_example/CMakeLists.txt

@@ -0,0 +1,72 @@
+cmake_minimum_required(VERSION 3.5.1)
+
+project(HelloWorld C CXX)
+
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+
+
+find_package(Threads REQUIRED)
+
+# This branch assumes that gRPC and all its dependencies are already installed
+  # on this system, so they can be located by find_package().
+
+  # Find Protobuf installation
+  # Looks for protobuf-config.cmake file installed by Protobuf's cmake installation.
+  set(protobuf_MODULE_COMPATIBLE TRUE)
+  find_package(Protobuf CONFIG REQUIRED)
+  message(STATUS "Using protobuf ${protobuf_VERSION}")
+
+  set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
+  set(_REFLECTION gRPC::grpc++_reflection)
+  if(CMAKE_CROSSCOMPILING)
+      find_program(_PROTOBUF_PROTOC protoc)
+  else()
+      set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+  endif()
+
+  # Find gRPC installation
+  # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
+  find_package(gRPC CONFIG REQUIRED)
+  message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+  set(_GRPC_GRPCPP gRPC::grpc++)
+  if(CMAKE_CROSSCOMPILING)
+      find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+  else()
+      set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+  endif()
+
+# Proto file
+get_filename_component(hw_proto "../ms_service.proto" ABSOLUTE)
+get_filename_component(hw_proto_path "${hw_proto}" PATH)
+
+# Generated sources
+set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.cc")
+set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.h")
+set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.cc")
+set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.h")
+add_custom_command(
+        OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}"
+        COMMAND ${_PROTOBUF_PROTOC}
+        ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${hw_proto_path}"
+        --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+        "${hw_proto}"
+        DEPENDS "${hw_proto}")
+
+# Include generated *.pb.h files
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+
+# Targets greeter_[async_](client|server)
+foreach(_target
+        ms_client ms_server)
+    add_executable(${_target} "${_target}.cc"
+            ${hw_proto_srcs}
+            ${hw_grpc_srcs})
+    target_link_libraries(${_target}
+            ${_REFLECTION}
+            ${_GRPC_GRPCPP}
+            ${_PROTOBUF_LIBPROTOBUF})
+endforeach()

serving/cpp_example/ms_client.cc

@@ -0,0 +1,105 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <grpcpp/grpcpp.h>
+#include <iostream>
+
+#include "serving/ms_service.grpc.pb.h"
+
+using grpc::Channel;
+using grpc::ClientContext;
+using grpc::Status;
+using ms_serving::MSService;
+using ms_serving::PredictReply;
+using ms_serving::PredictRequest;
+using ms_serving::Tensor;
+using ms_serving::TensorShape;
+
+class MSClient {
+ public:
+  explicit MSClient(std::shared_ptr<Channel> channel) : stub_(MSService::NewStub(channel)) {}
+
+  std::string Predict(const std::string &user) {
+    // Data we are sending to the server.
+    PredictRequest request;
+    Tensor data;
+    TensorShape shape;
+    shape.add_dims(1);
+    shape.add_dims(1);
+    shape.add_dims(2);
+    shape.add_dims(2);
+    *data.mutable_tensor_shape() = shape;
+    data.set_tensor_type(ms_serving::MS_FLOAT32);
+    vector<float> input_data{1.1, 2.1, 3.1, 4.1};
+    data.set_data(input_data.data(), input_data.size());
+    *request.add_data() = data;
+    *request.add_data() = data;
+
+    // Container for the data we expect from the server.
+    PredictReply reply;
+
+    // Context for the client. It could be used to convey extra information to
+    // the server and/or tweak certain RPC behaviors.
+    ClientContext context;
+
+    // The actual RPC.
+    Status status = stub_->Predict(&context, request, &reply);
+
+    // Act upon its status.
+    if (status.ok()) {
+      return "RPC OK";
+    } else {
+      std::cout << status.error_code() << ": " << status.error_message() << std::endl;
+      return "RPC failed";
+    }
+  }
+
+ private:
+  std::unique_ptr<MSService::Stub> stub_;
+};
+
+int main(int argc, char **argv) {
+  // Instantiate the client. It requires a channel, out of which the actual RPCs
+  // are created. This channel models a connection to an endpoint specified by
+  // the argument "--target=" which is the only expected argument.
+  // We indicate that the channel isn't authenticated (use of
+  // InsecureChannelCredentials()).
+  std::string target_str;
+  std::string arg_str("--target");
+  if (argc > 1) {
+    std::string arg_val = argv[1];
+    size_t start_pos = arg_val.find(arg_str);
+    if (start_pos != std::string::npos) {
+      start_pos += arg_str.size();
+      if (arg_val[start_pos] == '=') {
+        target_str = arg_val.substr(start_pos + 1);
+      } else {
+        std::cout << "The only correct argument syntax is --target=" << std::endl;
+        return 0;
+      }
+    } else {
+      std::cout << "The only acceptable argument is --target=" << std::endl;
+      return 0;
+    }
+  } else {
+    target_str = "localhost:85010";
+  }
+  MSClient client(grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials()));
+  string request;
+  string reply = client.Predict(request);
+  std::cout << "client received: " << reply << std::endl;
+
+  return 0;
+}

serving/cpp_example/ms_server.cc

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/health_check_service_interface.h>
+#include <grpcpp/ext/proto_server_reflection_plugin.h>
+#include <iostream>
+
+#include "serving/ms_service.grpc.pb.h"
+
+using grpc::Server;
+using grpc::ServerBuilder;
+using grpc::ServerContext;
+using grpc::Status;
+using ms_serving::MSService;
+using ms_serving::PredictReply;
+using ms_serving::PredictRequest;
+
+// Logic and data behind the server's behavior.
+class MSServiceImpl final : public MSService::Service {
+  Status Predict(ServerContext *context, const PredictRequest *request, PredictReply *reply) override {
+    cout << "server eval" << endl;
+    return Status::OK;
+  }
+};
+
+void RunServer() {
+  std::string server_address("0.0.0.0:50051");
+  MSServiceImpl service;
+
+  grpc::EnableDefaultHealthCheckService(true);
+  grpc::reflection::InitProtoReflectionServerBuilderPlugin();
+  auto option = grpc::MakeChannelArgumentOption(GRPC_ARG_ALLOW_REUSEPORT, 0);
+
+  ServerBuilder builder;
+  builder.SetOption(std::move(option));
+  // Listen on the given address without any authentication mechanism.
+  builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
+  // Register "service" as the instance through which we'll communicate with
+  // clients. In this case it corresponds to an *synchronous* service.
+  builder.RegisterService(&service);
+  // Finally assemble the server.
+  std::unique_ptr<Server> server(builder.BuildAndStart());
+  std::cout << "Server listening on " << server_address << std::endl;
+
+  // Wait for the server to shutdown. Note that some other thread must be
+  // responsible for shutting down the server for this call to ever return.
+  server->Wait();
+}
+
+int main(int argc, char **argv) {
+  RunServer();
+
+  return 0;
+}

serving/main.cc

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/server.h"
+#include "core/util/option_parser.h"
+
+using mindspore::serving::Options;
+
+int main(int argc, char **argv) {
+  auto flag = Options::Instance().ParseCommandLine(argc, argv);
+  if (!flag) {
+    return 0;
+  }
+  mindspore::serving::Server server;
+  server.BuildAndStart();
+  return 0;
+}

serving/ms_service.proto

@@ -0,0 +1,48 @@
+// ms_service.proto
+syntax = "proto3";
+
+package ms_serving;
+
+service MSService {
+    rpc Predict(PredictRequest) returns (PredictReply) {}
+    rpc Test(PredictRequest) returns (PredictReply) {}
+}
+
+message PredictRequest {
+    repeated Tensor data = 1;
+}
+
+message PredictReply {
+    repeated Tensor result = 1;
+}
+enum DataType {
+  MS_UNKNOWN = 0;
+  MS_BOOL = 1;
+  MS_INT8 = 2;
+  MS_UINT8 = 3;
+  MS_INT16 = 4;
+  MS_UINT16 = 5;
+  MS_INT32 = 6;
+  MS_UINT32 = 7;
+  MS_INT64 = 8;
+  MS_UINT64 = 9;
+  MS_FLOAT16 = 10;
+  MS_FLOAT32 = 11;
+  MS_FLOAT64 = 12;
+}
+
+message TensorShape {
+  repeated int64 dims = 1;
+};
+
+message Tensor {
+  // tensor shape info
+  TensorShape tensor_shape = 1;
+
+  // tensor content data type
+  DataType tensor_type = 2;
+
+  // tensor data
+  bytes data = 3;
+}
+

serving/python_example/ms_client.py

@@ -0,0 +1,57 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+import grpc
+import numpy as np
+import ms_service_pb2
+import ms_service_pb2_grpc
+
+
+def run():
+    channel = grpc.insecure_channel('localhost:50051')
+    stub = ms_service_pb2_grpc.MSServiceStub(channel)
+    # request = ms_service_pb2.PredictRequest()
+    # request.name = 'haha'
+    # response = stub.Eval(request)
+    # print("ms client received: " + response.message)
+
+    request = ms_service_pb2.PredictRequest()
+    request.data.tensor_shape.dims.extend([32, 1, 32, 32])
+    request.data.tensor_type = ms_service_pb2.MS_FLOAT32
+    request.data.data = (np.ones([32, 1, 32, 32]).astype(np.float32) * 0.01).tobytes()
+
+    request.label.tensor_shape.dims.extend([32])
+    request.label.tensor_type = ms_service_pb2.MS_INT32
+    request.label.data = np.ones([32]).astype(np.int32).tobytes()
+
+    result = stub.Predict(request)
+    #result_np = np.frombuffer(result.result.data, dtype=np.float32).reshape(result.result.tensor_shape.dims)
+    print("ms client received: ")
+    #print(result_np)
+
+    # future_list = []
+    # times = 1000
+    # for i in range(times):
+    #     async_future = stub.Eval.future(request)
+    #     future_list.append(async_future)
+    #     print("async call, future list add item " + str(i));
+    #
+    # for i in range(len(future_list)):
+    #     async_result = future_list[i].result()
+    #     print("ms client async get result of item " + str(i))
+
+
+
+if __name__ == '__main__':
+    run()