You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
129 lines
3.5 KiB
129 lines
3.5 KiB
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#pragma once
|
|
|
|
#include <NvInfer.h>
|
|
#include <cuda_runtime_api.h>
|
|
#include <atomic>
|
|
#include <memory>
|
|
#include <mutex> // NOLINT
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <utility>
|
|
#include <vector>
|
|
#include "paddle/fluid/framework/tensor.h"
|
|
#include "paddle/fluid/inference/tensorrt/engine.h"
|
|
#include "paddle/fluid/platform/place.h"
|
|
|
|
namespace paddle {
|
|
namespace inference {
|
|
namespace tensorrt {
|
|
|
|
class TensorRTEngine;
|
|
|
|
struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
|
|
public:
|
|
TRTInt8Calibrator(const std::unordered_map<std::string, size_t>& buffers,
|
|
int batch_size, std::string engine_name,
|
|
const platform::Place place);
|
|
|
|
explicit TRTInt8Calibrator(const std::string& calibration_data);
|
|
~TRTInt8Calibrator();
|
|
|
|
int getBatchSize() const override;
|
|
|
|
bool getBatch(void* bindings[], const char* names[],
|
|
int num_bindings) override;
|
|
|
|
bool setBatch(const std::unordered_map<std::string, void*>& data);
|
|
void setDone();
|
|
void waitAndSetDone();
|
|
|
|
const void* readCalibrationCache(std::size_t& length) override;
|
|
void writeCalibrationCache(const void* ptr, std::size_t length) override;
|
|
const std::string& getCalibrationTableAsString() {
|
|
return calibration_table_;
|
|
}
|
|
|
|
private:
|
|
const int batch_size_;
|
|
|
|
bool calib_running_{true};
|
|
bool data_is_set_{false};
|
|
bool done_{false};
|
|
|
|
std::mutex mut_;
|
|
std::condition_variable cond_;
|
|
|
|
std::unordered_map<std::string, std::pair<void*, size_t>> data_buffers_;
|
|
std::vector<framework::Tensor> data_tensors_;
|
|
|
|
std::string engine_name_;
|
|
std::string calibration_table_;
|
|
};
|
|
|
|
class TRTCalibratorEngine {
|
|
public:
|
|
TRTCalibratorEngine() {}
|
|
std::unique_ptr<TRTInt8Calibrator> calib_;
|
|
std::unique_ptr<std::thread> thr_;
|
|
std::unique_ptr<TensorRTEngine> engine_;
|
|
};
|
|
/*
|
|
* Manager to control the TensorRT Int8 calibration creation and deltetion.
|
|
*/
|
|
class TRTCalibratorEngineManager {
|
|
public:
|
|
bool Has() const { return res_.size() > 0; }
|
|
bool Has(const std::string& name) const {
|
|
if (res_.count(name) == 0) return false;
|
|
return res_.at(name).get() != nullptr;
|
|
}
|
|
|
|
// Get Int8Calibrator via name
|
|
TRTCalibratorEngine* Get(const std::string& name) const {
|
|
return res_.at(name).get();
|
|
}
|
|
|
|
// Look up or create a calibrator.
|
|
TRTCalibratorEngine* LookupOrCreate(const std::string& engine_name) {
|
|
if (res_.count(engine_name) == 0) {
|
|
auto* p = new TRTCalibratorEngine;
|
|
res_[engine_name].reset(p);
|
|
}
|
|
return res_.at(engine_name).get();
|
|
}
|
|
|
|
// Create an Int8Calibrator
|
|
TRTCalibratorEngine* Create(const std::string& engine_name) {
|
|
auto* p = new TRTCalibratorEngine;
|
|
res_[engine_name].reset(p);
|
|
return p;
|
|
}
|
|
|
|
void DeleteALL() {
|
|
for (auto& item : res_) {
|
|
item.second.reset(nullptr);
|
|
}
|
|
}
|
|
|
|
private:
|
|
std::unordered_map<std::string, std::unique_ptr<TRTCalibratorEngine>> res_;
|
|
};
|
|
|
|
} // namespace tensorrt
|
|
} // namespace inference
|
|
} // namespace paddle
|