Luo Tao
7 years ago
commit
326221acec
@ -0,0 +1,10 @@
|
||||
==================================
|
||||
Data Reader Interface and DataSets
|
||||
==================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
data/data_reader.rst
|
||||
data/image.rst
|
||||
data/dataset.rst
|
||||
@ -0,0 +1,72 @@
|
||||
=====================
|
||||
Data Reader Interface
|
||||
=====================
|
||||
|
||||
|
||||
DataTypes
|
||||
=========
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.dense_array
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.integer_value
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.integer_value_sequence
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.integer_value_sub_sequence
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_binary_vector
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_binary_vector_sequence
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_binary_vector_sub_sequence
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_float_vector
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_float_vector_sequence
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_float_vector_sub_sequence
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_non_value_slot
|
||||
:noindex:
|
||||
|
||||
.. autofunction:: paddle.v2.data_type.sparse_value_slot
|
||||
:noindex:
|
||||
|
||||
.. autoclass:: paddle.v2.data_type.InputType
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
DataFeeder
|
||||
==========
|
||||
|
||||
.. automodule:: paddle.v2.data_feeder
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
Reader
|
||||
======
|
||||
|
||||
.. automodule:: paddle.v2.reader
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
.. automodule:: paddle.v2.reader.creator
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
minibatch
|
||||
=========
|
||||
|
||||
.. automodule:: paddle.v2.minibatch
|
||||
:members:
|
||||
:noindex:
|
||||
@ -0,0 +1,82 @@
|
||||
Dataset
|
||||
=======
|
||||
|
||||
.. automodule:: paddle.dataset
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
mnist
|
||||
+++++
|
||||
|
||||
.. automodule:: paddle.dataset.mnist
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
cifar
|
||||
+++++
|
||||
|
||||
.. automodule:: paddle.dataset.cifar
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
conll05
|
||||
+++++++
|
||||
|
||||
.. automodule:: paddle.dataset.conll05
|
||||
:members: get_dict,get_embedding,test
|
||||
:noindex:
|
||||
|
||||
imdb
|
||||
++++
|
||||
|
||||
.. automodule:: paddle.dataset.imdb
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
imikolov
|
||||
++++++++
|
||||
|
||||
.. automodule:: paddle.dataset.imikolov
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
movielens
|
||||
+++++++++
|
||||
|
||||
.. automodule:: paddle.dataset.movielens
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
.. autoclass:: paddle.dataset.movielens.MovieInfo
|
||||
:noindex:
|
||||
|
||||
.. autoclass:: paddle.dataset.movielens.UserInfo
|
||||
:noindex:
|
||||
|
||||
sentiment
|
||||
+++++++++
|
||||
|
||||
.. automodule:: paddle.dataset.sentiment
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
uci_housing
|
||||
+++++++++++
|
||||
|
||||
.. automodule:: paddle.dataset.uci_housing
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
wmt14
|
||||
+++++
|
||||
|
||||
.. automodule:: paddle.dataset.wmt14
|
||||
:members:
|
||||
:noindex:
|
||||
|
||||
wmt16
|
||||
+++++
|
||||
|
||||
.. automodule:: paddle.dataset.wmt16
|
||||
:members:
|
||||
:noindex:
|
||||
@ -0,0 +1,5 @@
|
||||
Image Interface
|
||||
===============
|
||||
|
||||
.. automodule:: paddle.v2.image
|
||||
:members:
|
||||
|
After Width: | Height: | Size: 29 KiB |
@ -0,0 +1,131 @@
|
||||
# Background
|
||||
|
||||
[ONNX (Open Neural Network Exchange)](https://github.com/onnx/onnx) bridges different deep learning frameworks by providing an open source graph format for models. The models trained in other frameworks can be converted into the ONNX format to execute inference by utilizing the built-in operators in ONNX - this is called a **frontend**. With the inverse conversion (called a **backend**), different frameworks can share any models supported by ONNX in principle. Now most mainstream frameworks have joined the ONNX community, e.g. Caffe2, PyTorch, and MXNet etc. And there is a momentum driving more and more vendors to begin supporting ONNX or even choose ONNX as the only machine learning runtime in their devices.
|
||||
|
||||
Therefore, it is necessary to enable the conversion between PaddlePaddle and ONNX. This design doc is aimed at implementing a convertor, mainly for converting between **Fluid** models and ONNX (it is very likely that we may support older v2 models in the future). A complete convertor should be bidirectional - with a frontend AND a backend, but considering the importance, the we will start with the frontend i.e. Fluid models to ONNX models.
|
||||
|
||||
|
||||
# How it works
|
||||
|
||||
ONNX has a [working list of operators](https://github.com/onnx/onnx/blob/master/docs/Operators.md) which is versioned.
|
||||
|
||||
When prioritizing implementation of a frontend over a backend, choice of coverage of Fluid -> ONNX operators comes down to choices of models to be supported (see section `Supported models`). Eventually, this will allow us to reach a really-wide coverage of all operators.
|
||||
|
||||
Here are a few major considerations when it comes to converting models:
|
||||
|
||||
- **Op-level conversion**: How to map the inputs, attributes, and outputs of each Paddle operator to those of the ONNX operator. In several cases, these require transformations. For each direction (frontend vs. backend), a different conversion mapping is needed.
|
||||
- **Parameters (weights) initialization**: Setting initial parameters on different nodes.
|
||||
- **Tensor data type mapping** (Note: Some ONNX data types are not supported in Fluid)
|
||||
- **Network representation adaption**: Fluid `ProgramDesc` include nested blocks. Since ONNX is free of nesting, the `ProgramDesc` ops need to be traversed to only include ops from the global scope in the root block. The variables used as inputs and outputs should also be in this scope.
|
||||
- **Model validation**: There are two kinds of validations that are necessary:
|
||||
1. We need to ensure that the inference outputs of the ops in run inside a model are the same as those when running the ONNX converted ops through an alternative ONNX backend.
|
||||
2. Checking to see if the generated nodes on the graph are validated by the internal ONNX checkers.
|
||||
- **Versioning**: ONNX versions its op listing over versions. In fact, it has versioning on 3 different levels: ops, graphs, and ONNX models. This requires that we are conscious about versioning the convertor and updating tests and op convertor logic for each release. It also implies that we release pre-trained ONNX models upon each version release.
|
||||
|
||||
One thing that makes this conversion more feasible in Fluid's case is the use of a static IR - the `ProgramDesc` - as opposed to a dynamic graph, as created in the cases of frameworks like PyTorch.
|
||||
|
||||
|
||||
# Project structure
|
||||
|
||||
<p align="center">
|
||||
<img src="./images/project_structure.png"/>
|
||||
</p>
|
||||
|
||||
The project contains four important parts:
|
||||
|
||||
* **fluid**: The directory that contains wrappers for fluid related APIs. Fluid has provided some low-level APIs to parse or generate the inference model. However, directly using these low-level APIs makes the code tediously long. This module wraps low-level APIs to provide simplified interfaces.
|
||||
|
||||
* **onnx**: This is a Python package provided by ONNX containing helpers for creating nodes, graphs, and eventually binary protobuf models with initializer parameters.
|
||||
|
||||
* **onnx_fluid**: Contains two-way mapping (Fluid -> ONNX ops and ONNX -> Fluid ops). Called from `convert.py`, the program uses this mapping along with modifier functions to construct ONNX nodes with the help of ONNX's `make_node` helper. It also contains mapping between datatypes and tensor deprecation / amplification logic.
|
||||
|
||||
* **convert.py**: The interface exposed to users. This will traverse the global program blocks/variables and construct the write-able model.
|
||||
|
||||
|
||||
# Usage
|
||||
The converter should be designed to very easy-to-use. Bidirectional conversion between a Fluid inference model and an ONNX binary model will be supported. Model validation will also provided to verify the correctness of converted model.
|
||||
|
||||
* Convert Fluid inference model to ONNX binary model
|
||||
|
||||
```
|
||||
python convert.py --fluid_model <fluid inference model> --onnx_model <ONNX model> validate True
|
||||
```
|
||||
|
||||
* Validate the converted model
|
||||
|
||||
```
|
||||
python validate.py --fluid_model <fluid inference model> --onnx_model <ONNX model>
|
||||
```
|
||||
|
||||
The conversion and model validation will be completed consecutively, finally output a readable model structure description. And for the converse conversion, users only need to exchange the input and output.
|
||||
|
||||
|
||||
# Challenges and mitigation
|
||||
|
||||
## Cycles
|
||||
|
||||
Cycles are unsupported in ONNX. In Paddle, the `while` op is the most prominent example of a cycle.
|
||||
|
||||
*Resolution*: We won't support models with `while`s which can't be substituted until ONNX adds support for such ops.
|
||||
|
||||
## Sequences
|
||||
|
||||
Sequence processing operators like `sequence_expand`, `sequence_reshape`, `sequence_concat`, and `sequence_pool` are not supported by ONNX as well, because they do not support non-padded datatypes like LoDTensors.
|
||||
|
||||
*Resolution*: Since the runtimes using our ONNX exported graphs won't be using LoDTensors in the first place, such sequence operators should be mapped to ONNX ops that will do the necessary transposing ops with the knowledge of the padding and shape of the Tensors.
|
||||
|
||||
## Ops that can't easily be mapped
|
||||
|
||||
There are ops that just aren't possible to map today:
|
||||
|
||||
**Control flow operators**
|
||||
|
||||
Paddle supports control flow ops like `If/Else` and `Switch` (if we ignore the CSP operations like `select` for now). ONNX has `If` support in the experimental phase.
|
||||
|
||||
*Resolution*: Map Paddle's `If/Else` to ONNX's `If`, but ignore other control flow operators until ONNX brings support for them.
|
||||
|
||||
|
||||
**Non-existent in Fluid**
|
||||
|
||||
There are several ONNX operators that are not available in Fluid today, e.g. `InstanceNormalization`, `RandomUniform`, `Unsqueeze`, etc.
|
||||
|
||||
*Resolution*: For the initial phase, we can choose to not support ops that our models don't care for and are subsequently not available in Fluid. However, for ops that we think might be necessary for Fluid users also, we must implement them on our side and support the ONNX conversion to them. This list is TBD.
|
||||
|
||||
|
||||
**Concurrency**
|
||||
|
||||
ONNX does not have any considerations for concurrency right now.
|
||||
|
||||
*Resolution*: There are two ways to approach this:
|
||||
|
||||
a. We choose to not support concurrent models.
|
||||
b. We only support `go_op`s (basically threads) shallowly. This could mean that we enqueue `go_op` ops prior to gradient calculations OR even prior to the entire graph, and that's it - since `go_op`s do not have support for backprop anyways. One of the core target use cases of `go_op`: batch reading - can be handled through this approach.
|
||||
|
||||
|
||||
**Overloaded in Fluid**
|
||||
|
||||
There are ops in ONNX whose job can't be accomplished by a single corresponding Paddle operator (e.g. ), but a collection of operators.
|
||||
|
||||
*Resolution*: Chain multiple Paddle operators.
|
||||
|
||||
|
||||
## Lack of LoDTensors
|
||||
|
||||
As stated above, ONNX only supports simple Tensor values.
|
||||
|
||||
*Resolution*: Deprecate to plain old numpy-able tensors.
|
||||
|
||||
|
||||
## Reconstruction from deprecated ONNX ops
|
||||
|
||||
For higher-level Fluid ops, such as a few offered by the `nn` layer that do not have direct corresponding mappings but can be converted to ONNX by chaining a series of ops without cycles, it would be useful to map them back to the higher-level Fluid ops once converted back from the deprecated ONNX graphs.
|
||||
|
||||
*Resolution*: Graphs that have the deprecation from Paddle -> ONNX. When converting back from ONNX, if we encounter the identical graphs by doing a forward search, we can replace the subgraphs with the matching ONNX op.
|
||||
|
||||
|
||||
# Supported models
|
||||
|
||||
As mentioned above, potential risks may come from the conversion of sequence-related models, including the LodTensor, ```if/else``` and ```while``` operator. So a good choice is to focus on some important feedforward models first, then implement some simple recurrent models.
|
||||
|
||||
- Feedforward models: common models selected in PaddleBook, e.g. VGG, ResNet and some other models proposed by application teams.
|
||||
- Recurrent models: language model, stacked LSTMs etc.
|
||||
@ -1,19 +1,35 @@
|
||||
Use different clusters
|
||||
======================
|
||||
|
||||
PaddlePaddle supports running jobs on several platforms including:
|
||||
- `Kubernetes <http://kubernetes.io>`_ open-source system for automating deployment, scaling, and management of containerized applications from Google.
|
||||
- `OpenMPI <https://www.open-mpi.org>`_ Mature high performance parallel computing framework.
|
||||
- `Fabric <http://www.fabfile.org>`_ A cluster management tool. Write scripts to submit jobs or manage the cluster.
|
||||
The user's cluster environment is not the same. To facilitate everyone's deployment, we provide a variety of cluster deployment methods to facilitate the submission of cluster training tasks, which will be introduced as follows:
|
||||
|
||||
We'll introduce cluster job management on these platforms. The examples can be found under `cluster_train_v2 <https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2>`_ .
|
||||
`Kubernetes <http://kubernetes.io>`_ is a scheduling framework of Google open source container cluster, supporting a complete cluster solution for large-scale cluster production environment. The following guidelines show PaddlePaddle's support for Kubernetes:
|
||||
|
||||
These cluster platforms provide API or environment variables for training processes, when the job is dispatched to different nodes. Like node ID, IP or total number of nodes etc.
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
k8s_cn.md
|
||||
k8s_distributed_cn.md
|
||||
|
||||
`OpenMPI <https://www.open-mpi.org>`_ is a mature high-performance parallel computing framework, which is widely used in the field of HPC. The following guide describes how to use OpenMPI to build PaddlePaddle's cluster training task:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
fabric_en.md
|
||||
openmpi_en.md
|
||||
k8s_en.md
|
||||
k8s_aws_en.md
|
||||
openmpi_cn.md
|
||||
|
||||
`Fabric <http://www.fabfile.org>`_ is a convenient tool for program deployment and management. We provide a way to deploy and manage with Fabric. If you want to know more about it, please read the following guidelines:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
fabric_cn.md
|
||||
|
||||
We also support the deployment of PaddlePaddle on AWS. Learn more about:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
k8s_aws_cn.md
|
||||
|
||||
The examples can be found under `cluster_train_v2 <https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2>`_ .
|
||||
@ -0,0 +1,74 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable> // NOLINT
|
||||
#include <deque>
|
||||
#include <mutex> // NOLINT
|
||||
#include <utility>
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
|
||||
template <typename T>
|
||||
class BlockingQueue {
|
||||
public:
|
||||
void Push(const T &item) {
|
||||
{
|
||||
std::lock_guard<std::mutex> g(mutex_);
|
||||
q_.emplace_back(item);
|
||||
}
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
void Extend(const U &items) {
|
||||
{
|
||||
std::lock_guard<std::mutex> g(mutex_);
|
||||
for (auto &item : items) {
|
||||
q_.emplace_back(item);
|
||||
}
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
std::deque<T> PopAll(size_t ms, bool *timeout) {
|
||||
auto time =
|
||||
std::chrono::system_clock::now() + std::chrono::milliseconds(ms);
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
*timeout = !cv_.wait_until(lock, time, [this] { return !q_.empty(); });
|
||||
std::deque<T> ret;
|
||||
if (!*timeout) {
|
||||
std::swap(ret, q_);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
T Pop() {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
cv_.wait(lock, [=] { return !q_.empty(); });
|
||||
T rc(std::move(q_.front()));
|
||||
q_.pop_front();
|
||||
return rc;
|
||||
}
|
||||
|
||||
private:
|
||||
std::mutex mutex_;
|
||||
std::condition_variable cv_;
|
||||
std::deque<T> q_;
|
||||
};
|
||||
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,53 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/fluid/framework/framework.pb.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace inference {
|
||||
|
||||
/*
|
||||
* EngineBase is the base class of all inference engines. An inference engine
|
||||
* takes a paddle program as input, and outputs the result in fluid Tensor
|
||||
* format. It can be used to optimize performance of computation sub-blocks, for
|
||||
* example, break down the original block into sub-blocks and execute each
|
||||
* sub-blocks in different engines.
|
||||
*
|
||||
* For example:
|
||||
* When inference, the resnet50 model can put most of the model into subgraph
|
||||
* and run it on a TensorRT engine.
|
||||
*
|
||||
* There are several engines such as TensorRT and other frameworks, so an
|
||||
* EngineBase is put forward to give an unified interface for all the
|
||||
* different engine implemention.
|
||||
*/
|
||||
class EngineBase {
|
||||
public:
|
||||
using DescType = ::paddle::framework::proto::BlockDesc;
|
||||
|
||||
// Build the model and do some preparation, for example, in TensorRT, run
|
||||
// createInferBuilder, buildCudaEngine.
|
||||
virtual void Build(const DescType& paddle_model) = 0;
|
||||
|
||||
// Execute the engine, that will run the inference network.
|
||||
virtual void Execute(int batch_size) = 0;
|
||||
|
||||
virtual ~EngineBase() {}
|
||||
|
||||
}; // class EngineBase
|
||||
|
||||
} // namespace inference
|
||||
} // namespace paddle
|
||||
@ -1,3 +1,4 @@
|
||||
nv_test(test_tensorrt SRCS test_tensorrt.cc DEPS dynload_cuda device_context dynamic_loader)
|
||||
nv_test(test_tensorrt_engine SRCS test_engine.cc engine.cc DEPS dynload_cuda)
|
||||
cc_library(tensorrt DEPS tensorrt_convert)
|
||||
add_subdirectory(convert)
|
||||
|
||||
@ -0,0 +1,134 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/inference/tensorrt/engine.h"
|
||||
|
||||
#include <NvInfer.h>
|
||||
#include <cuda.h>
|
||||
#include <glog/logging.h>
|
||||
#include "paddle/fluid/inference/tensorrt/helper.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace inference {
|
||||
namespace tensorrt {
|
||||
|
||||
void TensorRTEngine::Build(const DescType& paddle_model) {
|
||||
PADDLE_ENFORCE(false, "not implemented");
|
||||
}
|
||||
|
||||
void TensorRTEngine::Execute(int batch_size) {
|
||||
infer_context_->enqueue(batch_size, buffers_.data(), *stream_, nullptr);
|
||||
cudaStreamSynchronize(*stream_);
|
||||
}
|
||||
|
||||
TensorRTEngine::~TensorRTEngine() {
|
||||
// clean buffer
|
||||
for (auto& buffer : buffers_) {
|
||||
if (buffer != nullptr) {
|
||||
PADDLE_ENFORCE_EQ(0, cudaFree(buffer));
|
||||
buffer = nullptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TensorRTEngine::FreezeNetwork() {
|
||||
PADDLE_ENFORCE(infer_builder_ != nullptr,
|
||||
"Call InitNetwork first to initialize network.");
|
||||
PADDLE_ENFORCE(infer_network_ != nullptr,
|
||||
"Call InitNetwork first to initialize network.");
|
||||
// build engine.
|
||||
infer_builder_->setMaxBatchSize(max_batch_);
|
||||
infer_builder_->setMaxWorkspaceSize(max_workspace_);
|
||||
|
||||
infer_engine_.reset(infer_builder_->buildCudaEngine(*infer_network_));
|
||||
PADDLE_ENFORCE(infer_engine_ != nullptr, "build cuda engine failed!");
|
||||
|
||||
infer_context_.reset(infer_engine_->createExecutionContext());
|
||||
|
||||
// allocate GPU buffers.
|
||||
buffers_.resize(buffer_sizes_.size(), nullptr);
|
||||
for (auto& item : buffer_sizes_) {
|
||||
if (item.second == 0) {
|
||||
auto slot_offset = infer_engine_->getBindingIndex(item.first.c_str());
|
||||
item.second = kDataTypeSize[static_cast<int>(
|
||||
infer_engine_->getBindingDataType(slot_offset))] *
|
||||
AccumDims(infer_engine_->getBindingDimensions(slot_offset));
|
||||
}
|
||||
PADDLE_ENFORCE_EQ(0, cudaMalloc(&buffer(item.first), item.second));
|
||||
}
|
||||
}
|
||||
|
||||
nvinfer1::ITensor* TensorRTEngine::DeclareInput(const std::string& name,
|
||||
nvinfer1::DataType dtype,
|
||||
const nvinfer1::Dims& dim) {
|
||||
PADDLE_ENFORCE_EQ(0, buffer_sizes_.count(name), "duplicate input name %s",
|
||||
name);
|
||||
|
||||
PADDLE_ENFORCE(infer_network_ != nullptr, "should initnetwork first");
|
||||
auto* input = infer_network_->addInput(name.c_str(), dtype, dim);
|
||||
PADDLE_ENFORCE(input, "infer network add input %s failed", name);
|
||||
|
||||
buffer_sizes_[name] = kDataTypeSize[static_cast<int>(dtype)] * AccumDims(dim);
|
||||
return input;
|
||||
}
|
||||
|
||||
void TensorRTEngine::DeclareOutput(const nvinfer1::ILayer* layer, int offset,
|
||||
const std::string& name) {
|
||||
PADDLE_ENFORCE_EQ(0, buffer_sizes_.count(name), "duplicate output name %s",
|
||||
name);
|
||||
|
||||
auto* output = layer->getOutput(offset);
|
||||
PADDLE_ENFORCE(output != nullptr);
|
||||
output->setName(name.c_str());
|
||||
infer_network_->markOutput(*output);
|
||||
// output buffers' size can only be decided latter, set zero here to mark this
|
||||
// and will reset latter.
|
||||
buffer_sizes_[name] = 0;
|
||||
}
|
||||
|
||||
void* TensorRTEngine::GetOutputInGPU(const std::string& name) {
|
||||
return buffer(name);
|
||||
}
|
||||
|
||||
void TensorRTEngine::GetOutputInCPU(const std::string& name, void* dst,
|
||||
size_t max_size) {
|
||||
// determine data size
|
||||
auto it = buffer_sizes_.find(name);
|
||||
PADDLE_ENFORCE(it != buffer_sizes_.end());
|
||||
PADDLE_ENFORCE_GT(it->second, 0);
|
||||
PADDLE_ENFORCE_GE(max_size, it->second);
|
||||
|
||||
PADDLE_ENFORCE_EQ(0, cudaMemcpyAsync(dst, buffer(name), it->second,
|
||||
cudaMemcpyDeviceToHost, *stream_));
|
||||
}
|
||||
|
||||
void*& TensorRTEngine::buffer(const std::string& name) {
|
||||
PADDLE_ENFORCE(infer_engine_ != nullptr, "call FreezeNetwork first.");
|
||||
auto it = buffer_sizes_.find(name);
|
||||
PADDLE_ENFORCE(it != buffer_sizes_.end());
|
||||
auto slot_offset = infer_engine_->getBindingIndex(name.c_str());
|
||||
return buffers_[slot_offset];
|
||||
}
|
||||
|
||||
void TensorRTEngine::SetInputFromCPU(const std::string& name, void* data,
|
||||
size_t size) {
|
||||
void* buf = buffer(name);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
0, cudaMemcpyAsync(buf, data, size, cudaMemcpyHostToDevice, *stream_));
|
||||
}
|
||||
|
||||
} // namespace tensorrt
|
||||
} // namespace inference
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,144 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <NvInfer.h>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include "paddle/fluid/inference/engine.h"
|
||||
#include "paddle/fluid/inference/tensorrt/helper.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace inference {
|
||||
namespace tensorrt {
|
||||
|
||||
/*
|
||||
* TensorRT Engine.
|
||||
*
|
||||
* There are two alternative ways to use it, one is to build from a paddle
|
||||
* protobuf model, another way is to manully construct the network.
|
||||
*/
|
||||
class TensorRTEngine : public EngineBase {
|
||||
public:
|
||||
// Weight is model parameter.
|
||||
class Weight {
|
||||
public:
|
||||
Weight(nvinfer1::DataType dtype, void* value, int num_elem) {
|
||||
w_.type = dtype;
|
||||
w_.values = value;
|
||||
w_.count = num_elem;
|
||||
}
|
||||
const nvinfer1::Weights& get() { return w_; }
|
||||
|
||||
private:
|
||||
nvinfer1::Weights w_;
|
||||
};
|
||||
|
||||
TensorRTEngine(int max_batch, int max_workspace, cudaStream_t* stream,
|
||||
nvinfer1::ILogger& logger = NaiveLogger::Global())
|
||||
: max_batch_(max_batch),
|
||||
max_workspace_(max_workspace),
|
||||
stream_(stream),
|
||||
logger_(logger) {}
|
||||
|
||||
virtual ~TensorRTEngine();
|
||||
|
||||
// TODO(Superjomn) implement it later when graph segmentation is supported.
|
||||
virtual void Build(const DescType& paddle_model) override;
|
||||
|
||||
virtual void Execute(int batch_size) override;
|
||||
|
||||
// Initialize the inference network, so that TensorRT layers can add to this
|
||||
// network.
|
||||
void InitNetwork() {
|
||||
infer_builder_.reset(createInferBuilder(logger_));
|
||||
infer_network_.reset(infer_builder_->createNetwork());
|
||||
}
|
||||
// After finishing adding ops, freeze this network and creates the executation
|
||||
// environment.
|
||||
void FreezeNetwork();
|
||||
|
||||
// Add an input and set its name, data type and dimention.
|
||||
nvinfer1::ITensor* DeclareInput(const std::string& name,
|
||||
nvinfer1::DataType dtype,
|
||||
const nvinfer1::Dims& dim);
|
||||
// Set the offset-th output from a layer as the network's output, and set its
|
||||
// name.
|
||||
void DeclareOutput(const nvinfer1::ILayer* layer, int offset,
|
||||
const std::string& name);
|
||||
|
||||
// GPU memory address for an ITensor with specific name. One can operate on
|
||||
// these memory directly for acceleration, for example, output the converted
|
||||
// data directly to the buffer to save data copy overhead.
|
||||
// NOTE this should be used after calling `FreezeNetwork`.
|
||||
void*& buffer(const std::string& name);
|
||||
|
||||
// Fill an input from CPU memory with name and size.
|
||||
void SetInputFromCPU(const std::string& name, void* data, size_t size);
|
||||
// TODO(Superjomn) is this method necessary given that buffer(xxx) can be
|
||||
// accessed directly. Fill an input from GPU memory with name and size.
|
||||
void SetInputFromGPU(const std::string& name, void* data, size_t size);
|
||||
// Get an output called name, the output of tensorrt is in GPU, so this method
|
||||
// will just return the output's GPU memory address.
|
||||
void* GetOutputInGPU(const std::string& name);
|
||||
// LOW EFFICENCY! Get output to CPU, this will trigger a memory copy from GPU
|
||||
// to CPU.
|
||||
void GetOutputInCPU(const std::string& name, void* dst, size_t max_size);
|
||||
|
||||
nvinfer1::ICudaEngine* engine() { return infer_engine_.get(); }
|
||||
nvinfer1::INetworkDefinition* network() { return infer_network_.get(); }
|
||||
|
||||
private:
|
||||
// the max batch size
|
||||
int max_batch_;
|
||||
// the max memory size the engine uses
|
||||
int max_workspace_;
|
||||
cudaStream_t* stream_;
|
||||
nvinfer1::ILogger& logger_;
|
||||
|
||||
std::vector<void*> buffers_;
|
||||
// max data size for the buffers.
|
||||
std::unordered_map<std::string /*name*/, size_t /*max size*/> buffer_sizes_;
|
||||
|
||||
// TensorRT related internal members
|
||||
template <typename T>
|
||||
struct Destroyer {
|
||||
void operator()(T* x) { x->destroy(); }
|
||||
};
|
||||
template <typename T>
|
||||
using infer_ptr = std::unique_ptr<T, Destroyer<T>>;
|
||||
infer_ptr<nvinfer1::IBuilder> infer_builder_;
|
||||
infer_ptr<nvinfer1::INetworkDefinition> infer_network_;
|
||||
infer_ptr<nvinfer1::ICudaEngine> infer_engine_;
|
||||
infer_ptr<nvinfer1::IExecutionContext> infer_context_;
|
||||
}; // class TensorRTEngine
|
||||
|
||||
// Add an layer__ into engine__ with args ARGS.
|
||||
// For example:
|
||||
// TRT_ENGINE_ADD_LAYER(xxx, FullyConnected, input, dim, weights, bias)
|
||||
//
|
||||
// Reference
|
||||
// https://docs.nvidia.com/deeplearning/sdk/tensorrt-developer-guide/index.html#charRNN_define_network
|
||||
//
|
||||
// will add a fully connected layer into the engine.
|
||||
// TensorRT has too many layers, so that is not wise to add member functions for
|
||||
// them, and an macro like this is more extensible when underlying TensorRT
|
||||
// library add new layer supports.
|
||||
#define TRT_ENGINE_ADD_LAYER(engine__, layer__, ARGS...) \
|
||||
engine__->network()->add##layer__(ARGS);
|
||||
|
||||
} // namespace tensorrt
|
||||
} // namespace inference
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,88 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <NvInfer.h>
|
||||
#include <cuda.h>
|
||||
#include <glog/logging.h>
|
||||
#include "paddle/fluid/platform/dynload/tensorrt.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace inference {
|
||||
namespace tensorrt {
|
||||
|
||||
namespace dy = paddle::platform::dynload;
|
||||
|
||||
static size_t AccumDims(nvinfer1::Dims dims) {
|
||||
size_t num = dims.nbDims == 0 ? 0 : 1;
|
||||
for (int i = 0; i < dims.nbDims; i++) {
|
||||
PADDLE_ENFORCE_GT(dims.d[i], 0);
|
||||
num *= dims.d[i];
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
// TensorRT data type to size
|
||||
const int kDataTypeSize[] = {
|
||||
4, // kFLOAT
|
||||
2, // kHALF
|
||||
1, // kINT8
|
||||
4 // kINT32
|
||||
};
|
||||
|
||||
// The following two API are implemented in TensorRT's header file, cannot load
|
||||
// from the dynamic library. So create our own implementation and directly
|
||||
// trigger the method from the dynamic library.
|
||||
static nvinfer1::IBuilder* createInferBuilder(nvinfer1::ILogger& logger) {
|
||||
return static_cast<nvinfer1::IBuilder*>(
|
||||
dy::createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
|
||||
}
|
||||
static nvinfer1::IRuntime* createInferRuntime(nvinfer1::ILogger& logger) {
|
||||
return static_cast<nvinfer1::IRuntime*>(
|
||||
dy::createInferRuntime_INTERNAL(&logger, NV_TENSORRT_VERSION));
|
||||
}
|
||||
|
||||
// A logger for create TensorRT infer builder.
|
||||
class NaiveLogger : public nvinfer1::ILogger {
|
||||
public:
|
||||
void log(nvinfer1::ILogger::Severity severity, const char* msg) override {
|
||||
switch (severity) {
|
||||
case Severity::kINFO:
|
||||
LOG(INFO) << msg;
|
||||
break;
|
||||
case Severity::kWARNING:
|
||||
LOG(WARNING) << msg;
|
||||
break;
|
||||
case Severity::kINTERNAL_ERROR:
|
||||
case Severity::kERROR:
|
||||
LOG(ERROR) << msg;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static nvinfer1::ILogger& Global() {
|
||||
static nvinfer1::ILogger* x = new NaiveLogger;
|
||||
return *x;
|
||||
}
|
||||
|
||||
virtual ~NaiveLogger() override {}
|
||||
};
|
||||
|
||||
} // namespace tensorrt
|
||||
} // namespace inference
|
||||
} // namespace paddle
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue