Merge pull request #12149 from reyoung/feature/combine_open_files_and_double_buffer
Change and polish readersguochaorong-patch-1
Yu Yang
7 years ago
committed by
GitHub
commit
b06309381b
@ -0,0 +1,96 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/operators/reader/buffered_reader.h"
|
||||
#include <vector>
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace reader {
|
||||
BufferedReader::~BufferedReader() { reader_->Shutdown(); }
|
||||
BufferedReader::BufferedReader(
|
||||
const std::shared_ptr<framework::ReaderBase> &reader,
|
||||
const platform::Place &place, size_t buffer_size)
|
||||
: framework::DecoratedReader(reader),
|
||||
thread_pool_(1),
|
||||
place_(place),
|
||||
buffer_size_(buffer_size) {
|
||||
cpu_buffer_.resize(buffer_size);
|
||||
gpu_buffer_.resize(buffer_size);
|
||||
ReadTillBufferFullAsync();
|
||||
}
|
||||
void BufferedReader::ReadTillBufferFullAsync() {
|
||||
PADDLE_ENFORCE_EQ(position_.size(), 0U);
|
||||
for (size_t i = 0; i < buffer_size_; ++i) {
|
||||
ReadAsync(i);
|
||||
}
|
||||
}
|
||||
void BufferedReader::ReadAsync(size_t i) {
|
||||
position_.emplace(thread_pool_.enqueue([this, i]() -> size_t {
|
||||
TensorVec &cpu = cpu_buffer_[i];
|
||||
reader_->ReadNext(&cpu);
|
||||
|
||||
if (cpu.empty()) {
|
||||
return -1UL;
|
||||
}
|
||||
|
||||
if (platform::is_gpu_place(place_)) {
|
||||
TensorVec &gpu = gpu_buffer_[i];
|
||||
gpu.resize(cpu.size());
|
||||
for (size_t i = 0; i < cpu.size(); ++i) {
|
||||
framework::TensorCopySync(cpu[i], place_, &gpu[i]);
|
||||
gpu[i].set_lod(cpu[i].lod());
|
||||
}
|
||||
}
|
||||
return i;
|
||||
}));
|
||||
}
|
||||
void BufferedReader::ShutdownImpl() {
|
||||
reader_->Shutdown();
|
||||
while (!position_.empty()) {
|
||||
position_.pop();
|
||||
}
|
||||
prev_pos_ = -1UL;
|
||||
}
|
||||
void BufferedReader::StartImpl() {
|
||||
reader_->Start();
|
||||
ReadTillBufferFullAsync();
|
||||
}
|
||||
void BufferedReader::ReadNextImpl(std::vector<framework::LoDTensor> *out) {
|
||||
if (position_.empty()) {
|
||||
out->clear();
|
||||
return;
|
||||
}
|
||||
size_t i = position_.front().get();
|
||||
position_.pop();
|
||||
|
||||
if (i == -1UL) {
|
||||
ReadNextImpl(out);
|
||||
return;
|
||||
}
|
||||
|
||||
*out = platform::is_gpu_place(place_) ? gpu_buffer_[i] : cpu_buffer_[i];
|
||||
|
||||
// Do not push current position into ReadAsync. Push the previous position
|
||||
// Since all computation in fluid are async, change the data of
|
||||
// current position may cause data error.
|
||||
if (prev_pos_ != -1Ul) {
|
||||
ReadAsync(prev_pos_);
|
||||
}
|
||||
prev_pos_ = i;
|
||||
}
|
||||
|
||||
} // namespace reader
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,66 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <list>
|
||||
#include <queue>
|
||||
#include <vector>
|
||||
#include "ThreadPool.h"
|
||||
#include "paddle/fluid/framework/reader.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace reader {
|
||||
|
||||
class BufferedReader : public framework::DecoratedReader {
|
||||
using TensorVec = std::vector<framework::LoDTensor>;
|
||||
using VecFuture = std::future<TensorVec>;
|
||||
|
||||
public:
|
||||
BufferedReader(const std::shared_ptr<framework::ReaderBase>& reader,
|
||||
const platform::Place& place, size_t buffer_size);
|
||||
|
||||
~BufferedReader() override;
|
||||
|
||||
private:
|
||||
void ReadTillBufferFullAsync();
|
||||
|
||||
void ReadAsync(size_t i);
|
||||
|
||||
protected:
|
||||
void ShutdownImpl() override;
|
||||
void StartImpl() override;
|
||||
void ReadNextImpl(std::vector<framework::LoDTensor>* out) override;
|
||||
|
||||
private:
|
||||
ThreadPool thread_pool_;
|
||||
platform::Place place_;
|
||||
const size_t buffer_size_;
|
||||
|
||||
std::queue<std::future<size_t>> position_;
|
||||
|
||||
// The buffer for reading data.
|
||||
// NOTE: the simplest way to implement buffered reader is do not use any
|
||||
// buffer, just read async and create futures as buffer size. However, to
|
||||
// malloc tensors every time is extremely slow. Here we store all data in
|
||||
// buffers and prevent alloc every time.
|
||||
std::vector<TensorVec> cpu_buffer_;
|
||||
std::vector<TensorVec> gpu_buffer_;
|
||||
size_t prev_pos_{-1UL};
|
||||
};
|
||||
|
||||
} // namespace reader
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,95 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy
|
||||
|
||||
import paddle
|
||||
import paddle.dataset.mnist as mnist
|
||||
import paddle.fluid as fluid
|
||||
import paddle.v2
|
||||
|
||||
|
||||
def network(is_train):
|
||||
reader = fluid.layers.py_reader(
|
||||
capacity=10,
|
||||
shapes=((-1, 784), (-1, 1)),
|
||||
dtypes=('float32', 'int64'),
|
||||
name="train_reader" if is_train else "test_reader")
|
||||
img, label = fluid.layers.read_file(reader)
|
||||
|
||||
hidden = img
|
||||
|
||||
for i in xrange(2):
|
||||
hidden = fluid.layers.fc(input=hidden, size=100, act='tanh')
|
||||
hidden = fluid.layers.dropout(
|
||||
hidden, dropout_prob=0.5, is_test=not is_train)
|
||||
|
||||
prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
|
||||
loss = fluid.layers.cross_entropy(input=prediction, label=label)
|
||||
return fluid.layers.mean(loss), reader
|
||||
|
||||
|
||||
def main():
|
||||
train_prog = fluid.Program()
|
||||
startup_prog = fluid.Program()
|
||||
|
||||
with fluid.program_guard(train_prog, startup_prog):
|
||||
with fluid.unique_name.guard():
|
||||
loss, train_reader = network(True)
|
||||
adam = fluid.optimizer.Adam(learning_rate=0.01)
|
||||
adam.minimize(loss)
|
||||
|
||||
test_prog = fluid.Program()
|
||||
test_startup = fluid.Program()
|
||||
with fluid.program_guard(test_prog, test_startup):
|
||||
with fluid.unique_name.guard():
|
||||
test_loss, test_reader = network(False)
|
||||
|
||||
fluid.Executor(fluid.CUDAPlace(0)).run(startup_prog)
|
||||
fluid.Executor(fluid.CUDAPlace(0)).run(test_startup)
|
||||
|
||||
trainer = fluid.ParallelExecutor(
|
||||
use_cuda=True, loss_name=loss.name, main_program=train_prog)
|
||||
|
||||
tester = fluid.ParallelExecutor(
|
||||
use_cuda=True, share_vars_from=trainer, main_program=test_prog)
|
||||
|
||||
train_reader.decorate_paddle_reader(
|
||||
paddle.v2.reader.shuffle(
|
||||
paddle.batch(mnist.train(), 512), buf_size=8192))
|
||||
|
||||
test_reader.decorate_paddle_reader(paddle.batch(mnist.test(), 512))
|
||||
|
||||
for epoch_id in xrange(10):
|
||||
train_reader.start()
|
||||
try:
|
||||
while True:
|
||||
print 'train_loss', numpy.array(
|
||||
trainer.run(fetch_list=[loss.name]))
|
||||
except fluid.core.EOFException:
|
||||
print 'End of epoch', epoch_id
|
||||
train_reader.reset()
|
||||
|
||||
test_reader.start()
|
||||
try:
|
||||
while True:
|
||||
print 'test loss', numpy.array(
|
||||
tester.run(fetch_list=[test_loss.name]))
|
||||
except fluid.core.EOFException:
|
||||
print 'End of testing'
|
||||
test_reader.reset()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue