!9765 remove serving in mindspore repo
From: @xu-yfei Reviewed-by: @zhoufeng54,@kisnwang Signed-off-by: @kisnwangpull/9765/MERGE
mindspore-ci-bot
4 years ago
committed by
Gitee
commit
058fbd2d1f
File diff suppressed because it is too large
Load Diff
@ -1,65 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_SESSION_SESSION_H
|
||||
#define MINDSPORE_CCSRC_SESSION_SESSION_H
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <memory>
|
||||
#include <map>
|
||||
|
||||
#include "backend/session/session_basic.h"
|
||||
#include "ir/anf.h"
|
||||
#include "include/inference.h"
|
||||
|
||||
#ifdef ENABLE_D
|
||||
#include "runtime/context.h"
|
||||
#endif
|
||||
|
||||
namespace mindspore {
|
||||
namespace inference {
|
||||
class MSInferSession : public InferSession {
|
||||
public:
|
||||
MSInferSession();
|
||||
~MSInferSession();
|
||||
|
||||
Status InitEnv(const std::string &device_type, uint32_t device_id) override;
|
||||
Status FinalizeEnv() override;
|
||||
Status LoadModelFromFile(const std::string &file_name, uint32_t &model_id) override;
|
||||
Status UnloadModel(uint32_t model_id) override;
|
||||
Status ExecuteModel(uint32_t model_id, const RequestBase &inputs, ReplyBase &outputs) override;
|
||||
Status GetModelInputsInfo(uint32_t graph_id, std::vector<inference::InferTensor> *tensor_list) const override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<session::SessionBasic> session_impl_ = nullptr;
|
||||
std::vector<uint32_t> graph_id_;
|
||||
std::string device_type_;
|
||||
int32_t device_id_ = 0;
|
||||
#ifdef ENABLE_D
|
||||
rtContext_t context_ = nullptr;
|
||||
#endif
|
||||
|
||||
static void RegAllOp();
|
||||
string AjustTargetName(const std::string &device);
|
||||
Status CompileGraph(std::shared_ptr<FuncGraph> funcGraphPtr, uint32_t &model_id);
|
||||
Status CheckModelInputs(uint32_t graph_id, const std::vector<tensor::TensorPtr> &inputs) const;
|
||||
std::vector<tensor::TensorPtr> RunGraph(uint32_t graph_id, const std::vector<tensor::TensorPtr> &inputs);
|
||||
};
|
||||
} // namespace inference
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_SESSION_SESSION_BASIC_H
|
||||
@ -1,125 +0,0 @@
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
# This branch assumes that gRPC and all its dependencies are already installed
|
||||
# on this system, so they can be located by find_package().
|
||||
|
||||
# Find Protobuf installation
|
||||
# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation.
|
||||
|
||||
#set(protobuf_MODULE_COMPATIBLE TRUE)
|
||||
#find_package(Protobuf CONFIG REQUIRED)
|
||||
#message(STATUS "Using protobuf ${protobuf_VERSION}")
|
||||
add_library(protobuf::libprotobuf ALIAS protobuf::protobuf)
|
||||
add_executable(protobuf::libprotoc ALIAS protobuf::protoc)
|
||||
|
||||
set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
|
||||
if (CMAKE_CROSSCOMPILING)
|
||||
find_program(_PROTOBUF_PROTOC protoc)
|
||||
else ()
|
||||
set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
|
||||
endif ()
|
||||
|
||||
# Find gRPC installation
|
||||
# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
|
||||
if (EXISTS ${grpc_ROOT}/lib64)
|
||||
set(gRPC_DIR "${grpc_ROOT}/lib64/cmake/grpc")
|
||||
else ()
|
||||
set(gRPC_DIR "${grpc_ROOT}/lib/cmake/grpc")
|
||||
endif ()
|
||||
message("serving using grpc_DIR : " ${gPRC_DIR})
|
||||
|
||||
find_package(gRPC CONFIG REQUIRED)
|
||||
message(STATUS "Using gRPC ${gRPC_VERSION}")
|
||||
|
||||
set(_GRPC_GRPCPP gRPC::grpc++)
|
||||
set(_REFLECTION gRPC::grpc++_reflection)
|
||||
|
||||
if (CMAKE_CROSSCOMPILING)
|
||||
find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
|
||||
find_program(_GRPC_PYTHON_PLUGIN_EXECUTABLE grpc_python_plugin)
|
||||
else ()
|
||||
set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
|
||||
set(_GRPC_PYTHON_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_python_plugin>)
|
||||
endif ()
|
||||
|
||||
# Proto file
|
||||
get_filename_component(hw_proto "ms_service.proto" ABSOLUTE)
|
||||
get_filename_component(hw_proto_path "${hw_proto}" PATH)
|
||||
# Generated sources
|
||||
set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.cc")
|
||||
set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.h")
|
||||
set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.cc")
|
||||
set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.h")
|
||||
set(hw_py_pb2 "${CMAKE_CURRENT_BINARY_DIR}/ms_service_pb2.py")
|
||||
set(hw_py_pb2_grpc "${CMAKE_CURRENT_BINARY_DIR}/ms_service_pb2_grpc.py")
|
||||
add_custom_command(
|
||||
OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}" "${hw_py_pb2}" "${hw_py_pb2_grpc}"
|
||||
COMMAND ${_PROTOBUF_PROTOC}
|
||||
ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
--cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
-I "${hw_proto_path}"
|
||||
--plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
|
||||
"${hw_proto}"
|
||||
COMMAND ${_PROTOBUF_PROTOC}
|
||||
ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
--python_out "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
-I "${hw_proto_path}"
|
||||
--plugin=protoc-gen-grpc="${_GRPC_PYTHON_PLUGIN_EXECUTABLE}"
|
||||
"${hw_proto}"
|
||||
DEPENDS "${hw_proto}")
|
||||
|
||||
# Include generated *.pb.h files
|
||||
include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/core"
|
||||
"${PROJECT_SOURCE_DIR}/mindspore/ccsrc" "${PROJECT_SOURCE_DIR}/mindspore/core")
|
||||
file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
"core/*.cc" "core/util/*.cc" "core/version_control/*.cc")
|
||||
|
||||
list(APPEND SERVING_SRC "main.cc" ${hw_proto_srcs} ${hw_grpc_srcs} ${CORE_SRC_LIST})
|
||||
|
||||
if (ENABLE_ACL)
|
||||
if (DEFINED ENV{ASCEND_CUSTOM_PATH})
|
||||
set(ASCEND_PATH $ENV{ASCEND_CUSTOM_PATH})
|
||||
else ()
|
||||
set(ASCEND_PATH /usr/local/Ascend)
|
||||
endif ()
|
||||
set(ACL_LIB_DIR ${ASCEND_PATH}/acllib/)
|
||||
set(ATLAS_ACL_LIB_DIR ${ASCEND_PATH}/ascend-toolkit/latest/acllib)
|
||||
MESSAGE("hisi acl lib dir " ${ACL_LIB_DIR} " ,atlas acl lib dir " ${ATLAS_ACL_LIB_DIR})
|
||||
|
||||
include_directories(${ACL_LIB_DIR}/include/)
|
||||
include_directories(${ATLAS_ACL_LIB_DIR}/include/)
|
||||
file(GLOB_RECURSE ACL_SESSION_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "acl/*.cc")
|
||||
list(APPEND SERVING_SRC ${ACL_SESSION_SRC_LIST})
|
||||
endif ()
|
||||
|
||||
include_directories(${CMAKE_BINARY_DIR})
|
||||
|
||||
add_executable(ms_serving ${SERVING_SRC})
|
||||
#libevent
|
||||
target_link_libraries(ms_serving mindspore::event mindspore::event_pthreads)
|
||||
|
||||
target_link_libraries(ms_serving ${_REFLECTION} ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} pthread)
|
||||
|
||||
include(CheckPIESupported)
|
||||
check_pie_supported()
|
||||
set_property(TARGET ms_serving PROPERTY POSITION_INDEPENDENT_CODE TRUE)
|
||||
|
||||
if (ENABLE_D)
|
||||
add_compile_definitions(ENABLE_D)
|
||||
target_link_libraries(ms_serving ${RUNTIME_LIB})
|
||||
endif ()
|
||||
|
||||
if (ENABLE_ACL)
|
||||
add_compile_definitions(ENABLE_ACL)
|
||||
add_compile_definitions(ENABLE_DVPP_INTERFACE)
|
||||
find_library(acl libascendcl.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64)
|
||||
find_library(acl_retr libacl_retr.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64)
|
||||
find_library(acl_cblas libacl_cblas.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64)
|
||||
find_library(acl_dvpp libacl_dvpp.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64)
|
||||
find_library(acl_runtime libruntime.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64)
|
||||
|
||||
target_link_libraries(ms_serving ${acl} ${acl_retr} ${acl_cblas} ${acl_dvpp} ${acl_runtime})
|
||||
target_link_libraries(ms_serving jpeg_turbo::jpeg securec)
|
||||
else ()
|
||||
target_link_libraries(ms_serving inference mindspore_gvar)
|
||||
endif ()
|
||||
@ -1,243 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include "serving/acl/acl_session.h"
|
||||
#include "include/infer_log.h"
|
||||
|
||||
namespace mindspore::inference {
|
||||
|
||||
std::shared_ptr<InferSession> InferSession::CreateSession(const std::string &device, uint32_t device_id) {
|
||||
try {
|
||||
auto session = std::make_shared<AclSession>();
|
||||
auto ret = session->InitEnv(device, device_id);
|
||||
if (ret != SUCCESS) {
|
||||
return nullptr;
|
||||
}
|
||||
return session;
|
||||
} catch (std::exception &e) {
|
||||
MSI_LOG_ERROR << "Inference CreatSession failed";
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
Status AclSession::LoadModelFromFile(const std::string &file_name, uint32_t &model_id) {
|
||||
Status ret = model_process_.LoadModelFromFile(file_name, model_id);
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG_ERROR << "Load model from file failed, model file " << file_name;
|
||||
return FAILED;
|
||||
}
|
||||
std::string dvpp_config_file;
|
||||
auto index = file_name.rfind(".");
|
||||
if (index == std::string::npos) {
|
||||
dvpp_config_file = file_name;
|
||||
} else {
|
||||
dvpp_config_file = file_name.substr(0, index);
|
||||
}
|
||||
dvpp_config_file += "_dvpp_config.json";
|
||||
std::ifstream fp(dvpp_config_file);
|
||||
if (!fp.is_open()) {
|
||||
MSI_LOG_INFO << "Dvpp config file not exist, model will execute with tensors as inputs, dvpp config file "
|
||||
<< dvpp_config_file;
|
||||
return SUCCESS;
|
||||
}
|
||||
fp.close();
|
||||
if (dvpp_process_.InitWithJsonConfig(dvpp_config_file) != SUCCESS) {
|
||||
MSI_LOG_ERROR << "Dvpp config file parse error, dvpp config file " << dvpp_config_file;
|
||||
return FAILED;
|
||||
}
|
||||
execute_with_dvpp_ = true;
|
||||
MSI_LOG_INFO << "Dvpp config success";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status AclSession::UnloadModel(uint32_t /*model_id*/) {
|
||||
model_process_.UnLoad();
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status AclSession::ExecuteModel(uint32_t /*model_id*/, const RequestBase &request,
|
||||
ReplyBase &reply) { // set d context
|
||||
aclError rt_ret = aclrtSetCurrentContext(context_);
|
||||
if (rt_ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "set the ascend device context failed";
|
||||
return FAILED;
|
||||
}
|
||||
return model_process_.Execute(request, reply);
|
||||
}
|
||||
|
||||
Status AclSession::PreProcess(uint32_t /*model_id*/, const InferImagesBase *images_input,
|
||||
ImagesDvppOutput &dvpp_output) {
|
||||
if (images_input == nullptr) {
|
||||
MSI_LOG_ERROR << "images input is nullptr";
|
||||
return FAILED;
|
||||
}
|
||||
auto batch_size = images_input->batch_size();
|
||||
if (batch_size <= 0) {
|
||||
MSI_LOG_ERROR << "invalid batch size " << images_input->batch_size();
|
||||
return FAILED;
|
||||
}
|
||||
std::vector<const void *> pic_buffer_list;
|
||||
std::vector<size_t> pic_size_list;
|
||||
for (size_t i = 0; i < batch_size; i++) {
|
||||
const void *pic_buffer = nullptr;
|
||||
uint32_t pic_size = 0;
|
||||
if (!images_input->get(i, pic_buffer, pic_size) || pic_buffer == nullptr || pic_size == 0) {
|
||||
MSI_LOG_ERROR << "Get request " << 0 << "th buffer failed";
|
||||
return FAILED;
|
||||
}
|
||||
pic_buffer_list.push_back(pic_buffer);
|
||||
pic_size_list.push_back(pic_size);
|
||||
}
|
||||
auto ret = dvpp_process_.Process(pic_buffer_list, pic_size_list, dvpp_output.buffer_device, dvpp_output.buffer_size);
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG_ERROR << "dvpp process failed";
|
||||
return ret;
|
||||
}
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status AclSession::ExecuteModel(uint32_t model_id, const ImagesRequestBase &images_inputs, // images for preprocess
|
||||
const RequestBase &request, ReplyBase &reply) {
|
||||
if (!execute_with_dvpp_) {
|
||||
MSI_LOG_ERROR << "Unexpected images as inputs, DVPP not config";
|
||||
return INFER_STATUS(INVALID_INPUTS) << "Unexpected images as inputs, DVPP not config";
|
||||
}
|
||||
aclError rt_ret = aclrtSetCurrentContext(context_);
|
||||
if (rt_ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "set the ascend device context failed";
|
||||
return FAILED;
|
||||
}
|
||||
if (images_inputs.size() != 1) {
|
||||
MSI_LOG_ERROR << "Only support one input to do DVPP preprocess";
|
||||
return INFER_STATUS(INVALID_INPUTS) << "Only support one input to do DVPP preprocess";
|
||||
}
|
||||
if (images_inputs[0] == nullptr) {
|
||||
MSI_LOG_ERROR << "Get first images input failed";
|
||||
return FAILED;
|
||||
}
|
||||
if (images_inputs[0]->batch_size() != model_process_.GetBatchSize()) {
|
||||
MSI_LOG_ERROR << "Input batch size " << images_inputs[0]->batch_size() << " not match Model batch size "
|
||||
<< model_process_.GetBatchSize();
|
||||
return INFER_STATUS(INVALID_INPUTS) << "Input batch size " << images_inputs[0]->batch_size()
|
||||
<< " not match Model batch size " << model_process_.GetBatchSize();
|
||||
}
|
||||
if (request.size() != 0) {
|
||||
MSI_LOG_ERROR << "only support one input, images input size is 1, tensor inputs is not 0 " << request.size();
|
||||
return INFER_STATUS(INVALID_INPUTS) << "only support one input, images input size is 1, tensor inputs is not 0 "
|
||||
<< request.size();
|
||||
}
|
||||
ImagesDvppOutput dvpp_output;
|
||||
Status ret = PreProcess(model_id, images_inputs[0], dvpp_output);
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG_ERROR << "DVPP preprocess failed";
|
||||
return ret;
|
||||
}
|
||||
ret = model_process_.Execute(dvpp_output.buffer_device, dvpp_output.buffer_size, reply);
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG_ERROR << "Execute model failed";
|
||||
return ret;
|
||||
}
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status AclSession::InitEnv(const std::string &device_type, uint32_t device_id) {
|
||||
device_type_ = device_type;
|
||||
device_id_ = device_id;
|
||||
auto ret = aclInit(nullptr);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "Execute aclInit Failed";
|
||||
return FAILED;
|
||||
}
|
||||
MSI_LOG_INFO << "acl init success";
|
||||
|
||||
ret = aclrtSetDevice(device_id_);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "acl open device " << device_id_ << " failed";
|
||||
return FAILED;
|
||||
}
|
||||
MSI_LOG_INFO << "open device " << device_id_ << " success";
|
||||
|
||||
ret = aclrtCreateContext(&context_, device_id_);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "acl create context failed";
|
||||
return FAILED;
|
||||
}
|
||||
MSI_LOG_INFO << "create context success";
|
||||
|
||||
ret = aclrtCreateStream(&stream_);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "acl create stream failed";
|
||||
return FAILED;
|
||||
}
|
||||
MSI_LOG_INFO << "create stream success";
|
||||
|
||||
aclrtRunMode run_mode;
|
||||
ret = aclrtGetRunMode(&run_mode);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "acl get run mode failed";
|
||||
return FAILED;
|
||||
}
|
||||
bool is_device = (run_mode == ACL_DEVICE);
|
||||
model_process_.SetIsDevice(is_device);
|
||||
MSI_LOG_INFO << "get run mode success is device input/output " << is_device;
|
||||
|
||||
if (dvpp_process_.InitResource(stream_) != SUCCESS) {
|
||||
MSI_LOG_ERROR << "dvpp init resource failed";
|
||||
return FAILED;
|
||||
}
|
||||
MSI_LOG_INFO << "Init acl success, device id " << device_id_;
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status AclSession::FinalizeEnv() {
|
||||
dvpp_process_.Finalize();
|
||||
aclError ret;
|
||||
if (stream_ != nullptr) {
|
||||
ret = aclrtDestroyStream(stream_);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "destroy stream failed";
|
||||
}
|
||||
stream_ = nullptr;
|
||||
}
|
||||
MSI_LOG_INFO << "end to destroy stream";
|
||||
if (context_ != nullptr) {
|
||||
ret = aclrtDestroyContext(context_);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "destroy context failed";
|
||||
}
|
||||
context_ = nullptr;
|
||||
}
|
||||
MSI_LOG_INFO << "end to destroy context";
|
||||
|
||||
ret = aclrtResetDevice(device_id_);
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "reset devie " << device_id_ << " failed";
|
||||
}
|
||||
MSI_LOG_INFO << "end to reset device " << device_id_;
|
||||
|
||||
ret = aclFinalize();
|
||||
if (ret != ACL_ERROR_NONE) {
|
||||
MSI_LOG_ERROR << "finalize acl failed";
|
||||
}
|
||||
MSI_LOG_INFO << "end to finalize acl";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
AclSession::AclSession() = default;
|
||||
} // namespace mindspore::inference
|
||||
@ -1,57 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_SERVING_ACL_SESSION_H
|
||||
#define MINDSPORE_SERVING_ACL_SESSION_H
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <memory>
|
||||
#include <map>
|
||||
|
||||
#include "include/inference.h"
|
||||
#include "serving/acl/model_process.h"
|
||||
#include "serving/acl/dvpp_process.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace inference {
|
||||
class AclSession : public InferSession {
|
||||
public:
|
||||
AclSession();
|
||||
|
||||
Status InitEnv(const std::string &device_type, uint32_t device_id) override;
|
||||
Status FinalizeEnv() override;
|
||||
Status LoadModelFromFile(const std::string &file_name, uint32_t &model_id) override;
|
||||
Status UnloadModel(uint32_t model_id) override;
|
||||
Status ExecuteModel(uint32_t model_id, const RequestBase &request, ReplyBase &reply) override;
|
||||
Status ExecuteModel(uint32_t model_id, const ImagesRequestBase &images_inputs, // images for preprocess
|
||||
const RequestBase &request, ReplyBase &reply) override;
|
||||
|
||||
private:
|
||||
std::string device_type_;
|
||||
int32_t device_id_;
|
||||
aclrtStream stream_ = nullptr;
|
||||
aclrtContext context_ = nullptr;
|
||||
ModelProcess model_process_;
|
||||
bool execute_with_dvpp_ = false;
|
||||
DvppProcess dvpp_process_;
|
||||
|
||||
Status PreProcess(uint32_t model_id, const InferImagesBase *images_input, ImagesDvppOutput &dvpp_output);
|
||||
};
|
||||
} // namespace inference
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_SERVING_ACL_SESSION_H
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,159 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef INC_DVPP_PROCESS_ACL
|
||||
#define INC_DVPP_PROCESS_ACL
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include "acl/acl.h"
|
||||
#include "acl/acl_mdl.h"
|
||||
#include "acl/acl_rt.h"
|
||||
#include "acl/ops/acl_dvpp.h"
|
||||
#include "include/inference.h"
|
||||
|
||||
namespace mindspore::inference {
|
||||
|
||||
struct DvppDecodePara {
|
||||
acldvppPixelFormat pixel_format = PIXEL_FORMAT_YUV_SEMIPLANAR_420;
|
||||
};
|
||||
|
||||
struct DvppResizePara {
|
||||
uint32_t output_width = 0;
|
||||
uint32_t output_height = 0;
|
||||
};
|
||||
|
||||
enum DvppCropType {
|
||||
// crop left,top,right,bottom is given in config
|
||||
kDvppCropTypeOffset = 0,
|
||||
// crop left,top,right,bottom is calculated by image width/height and output crop width/height
|
||||
kDvppCropTypeCentre = 1,
|
||||
};
|
||||
|
||||
struct DvppRoiArea {
|
||||
uint32_t left = 0;
|
||||
uint32_t top = 0;
|
||||
uint32_t right = 0;
|
||||
uint32_t bottom = 0;
|
||||
};
|
||||
|
||||
struct DvppCropInfo {
|
||||
DvppCropType crop_type = kDvppCropTypeOffset;
|
||||
DvppRoiArea crop_area; // when kDvppCropTypeOffset
|
||||
uint32_t crop_width = 0; // when kDvppCropTypeCentre
|
||||
uint32_t crop_height = 0; // when kDvppCropTypeCentre
|
||||
};
|
||||
|
||||
struct DvppCropPara {
|
||||
DvppCropInfo crop_info;
|
||||
uint32_t output_width = 0;
|
||||
uint32_t output_height = 0;
|
||||
};
|
||||
|
||||
struct DvppCropAndPastePara {
|
||||
DvppCropInfo crop_info;
|
||||
DvppRoiArea paste_area;
|
||||
uint32_t output_width = 0;
|
||||
uint32_t output_height = 0;
|
||||
};
|
||||
|
||||
class DvppProcess {
|
||||
public:
|
||||
DvppProcess();
|
||||
~DvppProcess();
|
||||
|
||||
Status InitResource(aclrtStream stream);
|
||||
void Finalize();
|
||||
Status InitJpegDecodePara(const DvppDecodePara &decode_para); // jpeg decode + (resize | crop)
|
||||
Status InitResizePara(const DvppResizePara &resize_para); // jpeg decode + resize
|
||||
Status InitCropPara(const DvppCropPara &crop_para); // jpeg decode + crop
|
||||
Status InitCropAndPastePara(const DvppCropAndPastePara &crop_and_paste_para); // jpeg decode + crop&paste
|
||||
|
||||
Status InitWithJsonConfig(const std::string &json_config);
|
||||
|
||||
// output device buffer will be destroy by DvppProcess itself.
|
||||
Status Process(const void *pic_buffer, size_t pic_buffer_size, void *&output_device_buffer, size_t &output_size);
|
||||
Status Process(const std::vector<const void *> &pic_buffer_list, const std::vector<size_t> &pic_buffer_size_list,
|
||||
void *&output_device_buffer, size_t &output_size);
|
||||
|
||||
private:
|
||||
uint32_t pic_width_ = 0;
|
||||
uint32_t pic_height_ = 0;
|
||||
|
||||
DvppDecodePara decode_para_;
|
||||
DvppResizePara resize_para_;
|
||||
DvppCropPara crop_para_;
|
||||
DvppCropAndPastePara crop_and_paste_para_;
|
||||
// only one of the resize or crop flag can be true
|
||||
bool to_resize_flag_ = false;
|
||||
bool to_crop_flag_ = false;
|
||||
bool to_crop_and_paste_flag_ = false;
|
||||
|
||||
void *input_pic_dev_buffer_ = nullptr;
|
||||
uint32_t input_pic_buffer_size_ = 0;
|
||||
|
||||
uint32_t decode_output_buffer_size_ = 0;
|
||||
void *decode_output_buffer_dev_ = nullptr;
|
||||
acldvppPicDesc *decode_output_desc_ = nullptr;
|
||||
|
||||
acldvppResizeConfig *resize_config_ = nullptr;
|
||||
acldvppRoiConfig *crop_area_ = nullptr;
|
||||
acldvppRoiConfig *paste_area_ = nullptr;
|
||||
|
||||
acldvppPicDesc *vpc_output_desc_ = nullptr;
|
||||
void *vpc_output_buffer_dev_ = nullptr; // vpc_output_buffer_size_ length
|
||||
uint32_t vpc_output_buffer_size_ = 0;
|
||||
|
||||
void *batch_vpc_output_buffer_dev_ = nullptr; // batch_size_ * vpc_output_buffer_size_ length
|
||||
uint32_t batch_size_ = 0;
|
||||
|
||||
aclrtStream stream_ = nullptr;
|
||||
acldvppChannelDesc *dvpp_channel_desc_ = nullptr;
|
||||
|
||||
uint32_t AlignmentHelper(uint32_t org_size, uint32_t alignment) const;
|
||||
uint32_t GetImageBufferSize(uint32_t stride_width, uint32_t stride_height, acldvppPixelFormat pixel_format) const;
|
||||
Status GetPicDescStride(uint32_t width, uint32_t height, uint32_t &stride_width, uint32_t &stride_height);
|
||||
Status GetPicDescStrideDecode(uint32_t width, uint32_t height, uint32_t &stride_width, uint32_t &stride_height);
|
||||
Status InputInputBuffer(const void *pic_buffer, size_t pic_buffer_size);
|
||||
Status InitDecodeOutputDesc(uint32_t image_width,
|
||||
uint32_t image_height); // decode_output_desc_, decode_output_buffer_dev_
|
||||
Status CheckRoiAreaWidthHeight(uint32_t width, uint32_t height);
|
||||
Status CheckAndAdjustRoiArea(DvppRoiArea &area);
|
||||
Status UpdateCropArea(uint32_t image_width, uint32_t image_height);
|
||||
Status CheckResizeImageInfo(uint32_t image_width, uint32_t image_height) const;
|
||||
void DestroyDecodeDesc();
|
||||
|
||||
Status InitVpcOutputDesc(uint32_t output_width, uint32_t output_height,
|
||||
acldvppPixelFormat pixel_format); // vpc_output_desc_, vpc_output_buffer_dev_batch_
|
||||
Status InitRoiAreaConfig(acldvppRoiConfig *&roi_area, const DvppRoiArea &init_para);
|
||||
Status InitCommonCropPara(DvppCropInfo &crop_info, uint32_t out_width, uint32_t out_height);
|
||||
Status InitResizeOutputDesc(); // vpc_output_desc_, vpc_output_buffer_dev_, resize_config
|
||||
Status InitCropOutputDesc(); // vpc_output_desc_, vpc_output_buffer_dev_, crop_area_
|
||||
Status InitCropAndPasteOutputDesc(); // vpc_output_desc_, vpc_output_buffer_dev_, crop_area_, paste_area_
|
||||
void DestroyVpcOutputDesc();
|
||||
|
||||
Status ProcessDecode();
|
||||
Status ProcessResize();
|
||||
Status ProcessCrop();
|
||||
Status ProcessCropAndPaste();
|
||||
void DestroyResource();
|
||||
|
||||
Status GetJpegWidthHeight(const void *pic_buffer, size_t pic_buffer_size, uint32_t &image_width,
|
||||
uint32_t &image_height);
|
||||
};
|
||||
|
||||
} // namespace mindspore::inference
|
||||
|
||||
#endif // INC_DVPP_PROCESS_ACL
|
||||
@ -1,68 +0,0 @@
|
||||
{
|
||||
"preprocess": [
|
||||
{
|
||||
"input": {
|
||||
"index": 0
|
||||
},
|
||||
"decode_para": {
|
||||
"out_pixel_format": "YUV420SP"
|
||||
},
|
||||
"dvpp_process": {
|
||||
"op_name": "resize",
|
||||
"out_width": 224,
|
||||
"out_height": 224
|
||||
},
|
||||
"sample of dvpp_process content": [
|
||||
{
|
||||
"op_name": "resize",
|
||||
"out_width": 224,
|
||||
"out_height": 224
|
||||
},
|
||||
{
|
||||
"op_name": "crop",
|
||||
"crop_type": "offset",
|
||||
"crop_left": 10,
|
||||
"crop_top": 10,
|
||||
"crop_right": 100,
|
||||
"crop_bottom": 200,
|
||||
"out_width": 224,
|
||||
"out_height": 224
|
||||
},
|
||||
{
|
||||
"op_name": "crop",
|
||||
"crop_type": "centre",
|
||||
"crop_width": 100,
|
||||
"crop_height": 100,
|
||||
"out_width": 224,
|
||||
"out_height": 224
|
||||
},
|
||||
{
|
||||
"op_name": "crop_and_paste",
|
||||
"crop_type": "offset",
|
||||
"crop_left": 10,
|
||||
"crop_top": 10,
|
||||
"crop_right": 100,
|
||||
"crop_bottom": 200,
|
||||
"paste_left": 10,
|
||||
"paste_top": 10,
|
||||
"paste_right": 100,
|
||||
"paste_bottom": 200,
|
||||
"out_width": 224,
|
||||
"out_height": 224
|
||||
},
|
||||
{
|
||||
"op_name": "crop_and_paste",
|
||||
"crop_type": "centre",
|
||||
"crop_width": 100,
|
||||
"crop_height": 100,
|
||||
"paste_left": 10,
|
||||
"paste_top": 10,
|
||||
"paste_right": 100,
|
||||
"paste_bottom": 200,
|
||||
"out_width": 224,
|
||||
"out_height": 224
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,83 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef INC_MODEL_PROCESS_ACL
|
||||
#define INC_MODEL_PROCESS_ACL
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include "acl/acl.h"
|
||||
#include "acl/acl_mdl.h"
|
||||
#include "acl/acl_rt.h"
|
||||
#include "include/inference.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace inference {
|
||||
struct AclTensorInfo {
|
||||
void *device_data;
|
||||
size_t buffer_size;
|
||||
aclDataType data_type;
|
||||
std::vector<int64_t> dims;
|
||||
};
|
||||
|
||||
struct ImagesDvppOutput {
|
||||
void *buffer_device = nullptr;
|
||||
size_t buffer_size = 0;
|
||||
size_t input_index = 0;
|
||||
};
|
||||
|
||||
class ModelProcess {
|
||||
public:
|
||||
ModelProcess() {}
|
||||
~ModelProcess() {}
|
||||
|
||||
Status LoadModelFromFile(const std::string &file_name, uint32_t &model_id);
|
||||
void UnLoad();
|
||||
|
||||
// override this method to avoid request/reply data copy
|
||||
Status Execute(const RequestBase &request, ReplyBase &reply);
|
||||
Status Execute(const void *dvpp_outputs_buffer_dev, size_t dvpp_outputs_buffer_size, ReplyBase &reply);
|
||||
void SetIsDevice(bool is_device) { is_run_on_device_ = is_device; }
|
||||
|
||||
size_t GetBatchSize() const;
|
||||
|
||||
private:
|
||||
uint32_t model_id_ = 0xffffffff;
|
||||
// if run one device(AICPU), there is no need to alloc device memory and copy inputs to(/outputs from) device
|
||||
bool is_run_on_device_ = false;
|
||||
aclmdlDesc *model_desc_ = nullptr;
|
||||
aclmdlDataset *inputs_ = nullptr;
|
||||
aclmdlDataset *outputs_ = nullptr;
|
||||
std::vector<AclTensorInfo> input_infos_;
|
||||
std::vector<AclTensorInfo> output_infos_;
|
||||
|
||||
Status PreInitModelResource();
|
||||
Status CreateDataBuffer(void *&data_mem_buffer, size_t buffer_size, aclmdlDataset *dataset);
|
||||
Status CheckAndInitInput(const RequestBase &request);
|
||||
Status CheckAndInitDvppInput(const void *dvpp_outputs_buffer_dev, size_t dvpp_outputs_buffer_size,
|
||||
size_t input_index);
|
||||
Status BuildOutputs(ReplyBase &reply);
|
||||
|
||||
Status InitInputsBuffer();
|
||||
Status InitOutputsBuffer();
|
||||
void DestroyInputsDataset();
|
||||
void DestroyInputsDataMem();
|
||||
void DestroyInputsBuffer();
|
||||
void DestroyOutputsBuffer();
|
||||
};
|
||||
} // namespace inference
|
||||
} // namespace mindspore
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,29 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_SERVING_HTTP_PROCESS_H
|
||||
#define MINDSPORE_SERVING_HTTP_PROCESS_H
|
||||
|
||||
#include <evhttp.h>
|
||||
#include <event.h>
|
||||
#include <event2/http.h>
|
||||
#include <event2/http_struct.h>
|
||||
|
||||
namespace mindspore {
|
||||
namespace serving {
|
||||
void http_handler_msg(struct evhttp_request *req, void *arg);
|
||||
} // namespace serving
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_SERVER_H
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,30 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_SERVER_H
|
||||
#define MINDSPORE_SERVER_H
|
||||
|
||||
#include "util/status.h"
|
||||
namespace mindspore {
|
||||
namespace serving {
|
||||
class Server {
|
||||
public:
|
||||
Server() = default;
|
||||
~Server() = default;
|
||||
Status BuildAndStart();
|
||||
};
|
||||
} // namespace serving
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_SERVER_H
|
||||
@ -1,194 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "core/serving_tensor.h"
|
||||
#include <vector>
|
||||
#include <unordered_map>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
#include "include/infer_log.h"
|
||||
|
||||
using std::string;
|
||||
using std::unordered_map;
|
||||
using std::vector;
|
||||
|
||||
namespace mindspore {
|
||||
namespace serving {
|
||||
using inference::DataType;
|
||||
using inference::InferTensorBase;
|
||||
|
||||
const size_t kMaxShapeElementCount = INT32_MAX;
|
||||
const size_t kMaxDataBufferSize = UINT32_MAX;
|
||||
|
||||
ServingTensor::ServingTensor(ms_serving::Tensor &other) : tensor_(other) {}
|
||||
|
||||
ServingTensor::~ServingTensor() {}
|
||||
|
||||
DataType ServingTensor::data_type() const {
|
||||
const std::unordered_map<ms_serving::DataType, inference::DataType> type2id_map{
|
||||
{ms_serving::MS_UNKNOWN, inference::kMSI_Unknown}, {ms_serving::MS_BOOL, inference::kMSI_Bool},
|
||||
{ms_serving::MS_INT8, inference::kMSI_Int8}, {ms_serving::MS_UINT8, inference::kMSI_Uint8},
|
||||
{ms_serving::MS_INT16, inference::kMSI_Int16}, {ms_serving::MS_UINT16, inference::kMSI_Uint16},
|
||||
{ms_serving::MS_INT32, inference::kMSI_Int32}, {ms_serving::MS_UINT32, inference::kMSI_Uint32},
|
||||
{ms_serving::MS_INT64, inference::kMSI_Int64}, {ms_serving::MS_UINT64, inference::kMSI_Uint64},
|
||||
{ms_serving::MS_FLOAT16, inference::kMSI_Float16}, {ms_serving::MS_FLOAT32, inference::kMSI_Float32},
|
||||
{ms_serving::MS_FLOAT64, inference::kMSI_Float64},
|
||||
};
|
||||
auto it = type2id_map.find(tensor_.tensor_type());
|
||||
if (it == type2id_map.end()) {
|
||||
MSI_LOG_WARNING << "failed to get data type, undefined data type " << tensor_.tensor_type();
|
||||
return inference::kMSI_Unknown;
|
||||
} else {
|
||||
return it->second;
|
||||
}
|
||||
}
|
||||
|
||||
void ServingTensor::set_data_type(DataType data_type) {
|
||||
const std::unordered_map<inference::DataType, ms_serving::DataType> id2type_map{
|
||||
{inference::kMSI_Unknown, ms_serving::MS_UNKNOWN}, {inference::kMSI_Bool, ms_serving::MS_BOOL},
|
||||
{inference::kMSI_Float64, ms_serving::MS_FLOAT64}, {inference::kMSI_Int8, ms_serving::MS_INT8},
|
||||
{inference::kMSI_Uint8, ms_serving::MS_UINT8}, {inference::kMSI_Int16, ms_serving::MS_INT16},
|
||||
{inference::kMSI_Uint16, ms_serving::MS_UINT16}, {inference::kMSI_Int32, ms_serving::MS_INT32},
|
||||
{inference::kMSI_Uint32, ms_serving::MS_UINT32}, {inference::kMSI_Int64, ms_serving::MS_INT64},
|
||||
{inference::kMSI_Uint64, ms_serving::MS_UINT64}, {inference::kMSI_Float16, ms_serving::MS_FLOAT16},
|
||||
{inference::kMSI_Float32, ms_serving::MS_FLOAT32},
|
||||
};
|
||||
auto it = id2type_map.find(data_type);
|
||||
if (it == id2type_map.end()) {
|
||||
MSI_LOG_WARNING << "failed to set data type, undefined data type " << data_type;
|
||||
tensor_.set_tensor_type(ms_serving::MS_UNKNOWN);
|
||||
} else {
|
||||
tensor_.set_tensor_type(it->second);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<int64_t> ServingTensor::shape() const {
|
||||
std::vector<int64_t> result;
|
||||
auto dims = tensor_.tensor_shape().dims();
|
||||
std::transform(dims.begin(), dims.end(), std::back_inserter(result), [](const int64_t dim) { return dim; });
|
||||
return result;
|
||||
}
|
||||
|
||||
void ServingTensor::set_shape(const std::vector<int64_t> &shape) {
|
||||
auto tensor_shape = tensor_.mutable_tensor_shape();
|
||||
tensor_shape->Clear();
|
||||
size_t element_count = 1;
|
||||
for (auto dim : shape) {
|
||||
if (dim <= 0 || element_count > kMaxShapeElementCount / dim) {
|
||||
MSI_LOG_ERROR << "failed to set shape, invalid dim num " << dim;
|
||||
tensor_shape->Clear();
|
||||
return;
|
||||
}
|
||||
element_count *= dim;
|
||||
tensor_shape->add_dims(dim);
|
||||
}
|
||||
}
|
||||
|
||||
bool ServingTensor::resize_data(size_t data_len) {
|
||||
string *buffer = tensor_.mutable_data();
|
||||
if (buffer == nullptr) {
|
||||
MSI_LOG_ERROR << "invalid buffer data";
|
||||
return false;
|
||||
}
|
||||
buffer->resize(data_len);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t ServingTensor::data_size() const { return tensor_.data().size(); }
|
||||
|
||||
void *ServingTensor::mutable_data() { return const_cast<char *>(tensor_.mutable_data()->data()); }
|
||||
|
||||
const void *ServingTensor::data() const { return tensor_.data().data(); }
|
||||
|
||||
ServingRequest::ServingRequest(const ms_serving::PredictRequest &request) : request_(request) {
|
||||
auto &data = request_.data();
|
||||
std::transform(data.begin(), data.end(), std::back_inserter(cache_),
|
||||
[](const ms_serving::Tensor &item) { return ServingTensor(const_cast<ms_serving::Tensor &>(item)); });
|
||||
}
|
||||
|
||||
size_t ServingRequest::size() const { return cache_.size(); }
|
||||
|
||||
const InferTensorBase *ServingRequest::operator[](size_t index) const {
|
||||
if (index >= cache_.size()) {
|
||||
MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size();
|
||||
return nullptr;
|
||||
}
|
||||
return &(cache_[index]);
|
||||
}
|
||||
|
||||
ServingImages::ServingImages(const ms_serving::Images &images) : images_(images) {}
|
||||
|
||||
size_t ServingImages::batch_size() const { return images_.images_size(); }
|
||||
|
||||
bool ServingImages::get(size_t index, const void *&pic_buffer, uint32_t &pic_size) const {
|
||||
if (index >= static_cast<size_t>(images_.images_size())) {
|
||||
MSI_LOG_ERROR << "visit invalid index " << index << " total size " << images_.images_size();
|
||||
return false;
|
||||
}
|
||||
pic_buffer = images_.images(index).data();
|
||||
pic_size = images_.images(index).size();
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t ServingImages::input_index() const { return static_cast<size_t>(images_.input_index()); }
|
||||
|
||||
size_t ServingReply::size() const { return cache_.size(); }
|
||||
|
||||
InferTensorBase *ServingReply::operator[](size_t index) {
|
||||
if (index >= cache_.size()) {
|
||||
MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size();
|
||||
return nullptr;
|
||||
}
|
||||
return &(cache_[index]);
|
||||
}
|
||||
|
||||
const InferTensorBase *ServingReply::operator[](size_t index) const {
|
||||
if (index >= cache_.size()) {
|
||||
MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size();
|
||||
return nullptr;
|
||||
}
|
||||
return &(cache_[index]);
|
||||
}
|
||||
|
||||
InferTensorBase *ServingReply::add() {
|
||||
auto new_item = reply_.add_result();
|
||||
if (new_item == nullptr) {
|
||||
MSI_LOG_ERROR << "add new item failed, current total size " << cache_.size();
|
||||
return nullptr;
|
||||
}
|
||||
cache_.push_back(ServingTensor(*new_item));
|
||||
return &(cache_.back());
|
||||
}
|
||||
|
||||
void ServingReply::clear() { reply_.mutable_result()->Clear(); }
|
||||
|
||||
ServingImagesRequest::ServingImagesRequest(const ms_serving::PredictRequest &request) : request_(request) {
|
||||
auto &images_inputs = request_.images();
|
||||
std::transform(images_inputs.begin(), images_inputs.end(), std::back_inserter(cache_),
|
||||
[](const ms_serving::Images &item) { return ServingImages(const_cast<ms_serving::Images &>(item)); });
|
||||
}
|
||||
|
||||
size_t ServingImagesRequest::size() const { return cache_.size(); }
|
||||
|
||||
const inference::InferImagesBase *ServingImagesRequest::operator[](size_t index) const {
|
||||
if (index >= cache_.size()) {
|
||||
MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size();
|
||||
return nullptr;
|
||||
}
|
||||
return &(cache_[index]);
|
||||
}
|
||||
} // namespace serving
|
||||
} // namespace mindspore
|
||||
@ -1,105 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_SERVING_TENSOR_H_
|
||||
#define MINDSPORE_SERVING_TENSOR_H_
|
||||
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include "include/infer_tensor.h"
|
||||
#include "serving/ms_service.pb.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace serving {
|
||||
class MS_API ServingTensor : public inference::InferTensorBase {
|
||||
public:
|
||||
// the other's lifetime must longer than this object
|
||||
explicit ServingTensor(ms_serving::Tensor &other);
|
||||
~ServingTensor();
|
||||
|
||||
inference::DataType data_type() const override;
|
||||
void set_data_type(inference::DataType type) override;
|
||||
std::vector<int64_t> shape() const override;
|
||||
void set_shape(const std::vector<int64_t> &shape) override;
|
||||
const void *data() const override;
|
||||
size_t data_size() const override;
|
||||
bool resize_data(size_t data_len) override;
|
||||
void *mutable_data() override;
|
||||
|
||||
private:
|
||||
// if tensor_ is reference from other ms_serving::Tensor, the other's lifetime must
|
||||
// longer than this object
|
||||
ms_serving::Tensor &tensor_;
|
||||
};
|
||||
|
||||
class ServingImages : public inference::InferImagesBase {
|
||||
public:
|
||||
explicit ServingImages(const ms_serving::Images &images);
|
||||
~ServingImages() = default;
|
||||
|
||||
size_t batch_size() const override;
|
||||
bool get(size_t index, const void *&pic_buffer, uint32_t &pic_size) const override;
|
||||
size_t input_index() const override;
|
||||
|
||||
private:
|
||||
const ms_serving::Images &images_;
|
||||
};
|
||||
|
||||
class ServingRequest : public inference::RequestBase {
|
||||
public:
|
||||
explicit ServingRequest(const ms_serving::PredictRequest &request);
|
||||
~ServingRequest() = default;
|
||||
|
||||
size_t size() const override;
|
||||
const inference::InferTensorBase *operator[](size_t index) const override;
|
||||
|
||||
private:
|
||||
const ms_serving::PredictRequest &request_;
|
||||
std::vector<ServingTensor> cache_;
|
||||
};
|
||||
|
||||
class ServingReply : public inference::ReplyBase {
|
||||
public:
|
||||
explicit ServingReply(ms_serving::PredictReply &reply) : reply_(reply) {}
|
||||
~ServingReply() = default;
|
||||
|
||||
size_t size() const override;
|
||||
inference::InferTensorBase *operator[](size_t index) override;
|
||||
const inference::InferTensorBase *operator[](size_t index) const override;
|
||||
inference::InferTensorBase *add() override;
|
||||
void clear() override;
|
||||
|
||||
private:
|
||||
ms_serving::PredictReply &reply_;
|
||||
std::vector<ServingTensor> cache_;
|
||||
};
|
||||
|
||||
class ServingImagesRequest : public inference::ImagesRequestBase {
|
||||
public:
|
||||
explicit ServingImagesRequest(const ms_serving::PredictRequest &request);
|
||||
~ServingImagesRequest() = default;
|
||||
|
||||
size_t size() const override;
|
||||
const inference::InferImagesBase *operator[](size_t index) const override;
|
||||
|
||||
private:
|
||||
const ms_serving::PredictRequest &request_;
|
||||
std::vector<ServingImages> cache_;
|
||||
};
|
||||
} // namespace serving
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_SERVING_TENSOR_H_
|
||||
@ -1,154 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "core/session.h"
|
||||
#include <grpcpp/grpcpp.h>
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
#include <utility>
|
||||
#include <memory>
|
||||
#include <chrono>
|
||||
|
||||
#include "include/infer_log.h"
|
||||
#include "serving/ms_service.grpc.pb.h"
|
||||
#include "core/util/option_parser.h"
|
||||
#include "core/version_control/version_controller.h"
|
||||
#include "core/util/file_system_operation.h"
|
||||
#include "core/serving_tensor.h"
|
||||
|
||||
using ms_serving::MSService;
|
||||
using ms_serving::PredictReply;
|
||||
using ms_serving::PredictRequest;
|
||||
|
||||
namespace mindspore {
|
||||
namespace serving {
|
||||
Status Session::CreatDeviceSession(const std::string &device, uint32_t device_id) {
|
||||
session_ = inference::InferSession::CreateSession(device, device_id);
|
||||
if (session_ == nullptr) {
|
||||
MSI_LOG(ERROR) << "Creat Session Failed";
|
||||
return FAILED;
|
||||
}
|
||||
device_type_ = device;
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Session &Session::Instance() {
|
||||
static Session instance;
|
||||
return instance;
|
||||
}
|
||||
|
||||
Status Session::Predict(const PredictRequest &request, PredictReply &reply) {
|
||||
try {
|
||||
auto status = PredictInner(request, reply);
|
||||
return status;
|
||||
} catch (const std::bad_alloc &ex) {
|
||||
MSI_LOG(ERROR) << "Serving Error: malloc memory failed";
|
||||
std::cout << "Serving Error: malloc memory failed" << std::endl;
|
||||
} catch (const std::runtime_error &ex) {
|
||||
MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what();
|
||||
std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl;
|
||||
} catch (const std::exception &ex) {
|
||||
MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what();
|
||||
std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl;
|
||||
} catch (...) {
|
||||
MSI_LOG(ERROR) << "Serving Error: exception occurred";
|
||||
std::cout << "Serving Error: exception occurred";
|
||||
}
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
Status Session::PredictInner(const PredictRequest &request, PredictReply &reply) {
|
||||
if (!model_loaded_) {
|
||||
MSI_LOG(ERROR) << "the model has not loaded";
|
||||
return FAILED;
|
||||
}
|
||||
if (session_ == nullptr) {
|
||||
MSI_LOG(ERROR) << "the inference session has not be initialized";
|
||||
return FAILED;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
MSI_LOG(INFO) << "run Predict";
|
||||
|
||||
if (request.images_size() > 0) {
|
||||
ServingImagesRequest serving_images(request);
|
||||
ServingRequest serving_request(request);
|
||||
ServingReply serving_reply(reply);
|
||||
Status ret = session_->ExecuteModel(graph_id_, serving_images, serving_request, serving_reply);
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG(ERROR) << "execute model with images return failed";
|
||||
return ret;
|
||||
}
|
||||
} else if (request.data_size() > 0) {
|
||||
ServingRequest serving_request(request);
|
||||
ServingReply serving_reply(reply);
|
||||
Status ret = session_->ExecuteModel(graph_id_, serving_request, serving_reply);
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG(ERROR) << "execute model with datas return failed";
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
MSI_LOG(INFO) << "run Predict finished";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status Session::Warmup(const MindSporeModelPtr model) {
|
||||
if (session_ == nullptr) {
|
||||
MSI_LOG(ERROR) << "The CreatDeviceSession should be called, before warmup";
|
||||
return FAILED;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
std::string file_name = model->GetModelPath() + '/' + model->GetModelName();
|
||||
model_loaded_ = false;
|
||||
MSI_TIME_STAMP_START(LoadModelFromFile)
|
||||
auto ret = session_->LoadModelFromFile(file_name, graph_id_);
|
||||
MSI_TIME_STAMP_END(LoadModelFromFile)
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str();
|
||||
return ret;
|
||||
}
|
||||
model_loaded_ = true;
|
||||
MSI_LOG(INFO) << "Session Warmup finished";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status Session::Clear() {
|
||||
if (session_ != nullptr) {
|
||||
session_->UnloadModel(graph_id_);
|
||||
session_->FinalizeEnv();
|
||||
session_ = nullptr;
|
||||
}
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status Session::GetModelInputsInfo(std::vector<inference::InferTensor> &tensor_list) {
|
||||
if (!model_loaded_) {
|
||||
MSI_LOG(ERROR) << "the model has not loaded";
|
||||
return FAILED;
|
||||
}
|
||||
if (session_ == nullptr) {
|
||||
MSI_LOG(ERROR) << "the inference session has not be initialized";
|
||||
return FAILED;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
Status ret = session_->GetModelInputsInfo(graph_id_, &tensor_list);
|
||||
if (ret != SUCCESS) {
|
||||
MSI_LOG(ERROR) << "get model inputs info failed";
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
} // namespace serving
|
||||
} // namespace mindspore
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue