Merge pull request #15799 from sneaxiy/feature/decoupled_reader
Try to decouple reader with program_descmove-code
Zeng Jinle
6 years ago
committed by
GitHub
commit
4cc9809cae
@ -0,0 +1,42 @@
|
|||||||
|
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "paddle/fluid/operators/reader/py_reader.h"
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
namespace reader {
|
||||||
|
|
||||||
|
PyReader::PyReader(const std::shared_ptr<LoDTensorBlockingQueue>& queue)
|
||||||
|
: framework::FileReader() {
|
||||||
|
PADDLE_ENFORCE(queue != nullptr, "LoDTensorBlockingQueue must not be null");
|
||||||
|
queue_ = queue;
|
||||||
|
}
|
||||||
|
|
||||||
|
void PyReader::ReadNext(std::vector<framework::LoDTensor>* out) {
|
||||||
|
bool success;
|
||||||
|
*out = queue_->Pop(&success);
|
||||||
|
if (!success) out->clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
PyReader::~PyReader() { queue_->Close(); }
|
||||||
|
|
||||||
|
void PyReader::Shutdown() { queue_->Close(); }
|
||||||
|
|
||||||
|
void PyReader::Start() { queue_->ReOpen(); }
|
||||||
|
|
||||||
|
} // namespace reader
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
|
#include <memory>
|
||||||
|
#include <vector>
|
||||||
|
#include "paddle/fluid/framework/reader.h"
|
||||||
|
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
namespace reader {
|
||||||
|
|
||||||
|
class PyReader : public framework::FileReader {
|
||||||
|
public:
|
||||||
|
explicit PyReader(const std::shared_ptr<LoDTensorBlockingQueue>& queue);
|
||||||
|
|
||||||
|
void ReadNext(std::vector<framework::LoDTensor>* out) override;
|
||||||
|
|
||||||
|
~PyReader();
|
||||||
|
|
||||||
|
void Shutdown() override;
|
||||||
|
|
||||||
|
void Start() override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::shared_ptr<LoDTensorBlockingQueue> queue_;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace reader
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,161 @@
|
|||||||
|
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "paddle/fluid/pybind/reader_py.h"
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
#include "paddle/fluid/framework/reader.h"
|
||||||
|
#include "paddle/fluid/operators/reader/buffered_reader.h"
|
||||||
|
#include "paddle/fluid/operators/reader/py_reader.h"
|
||||||
|
#include "paddle/fluid/platform/place.h"
|
||||||
|
#include "pybind11/stl.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace pybind {
|
||||||
|
|
||||||
|
class MultiDeviceFeedReader {
|
||||||
|
public:
|
||||||
|
using ResultDictList =
|
||||||
|
std::vector<std::unordered_map<std::string, framework::LoDTensor>>;
|
||||||
|
|
||||||
|
MultiDeviceFeedReader(
|
||||||
|
const std::shared_ptr<operators::reader::LoDTensorBlockingQueue> &queue,
|
||||||
|
const std::vector<std::string> &names,
|
||||||
|
const std::vector<platform::Place> &dst_places, bool use_double_buffer)
|
||||||
|
: queue_(queue),
|
||||||
|
names_(names),
|
||||||
|
pool_(new ::ThreadPool(dst_places.size())) {
|
||||||
|
std::shared_ptr<framework::ReaderBase> reader(
|
||||||
|
new operators::reader::PyReader(queue));
|
||||||
|
|
||||||
|
readers_.reserve(dst_places.size());
|
||||||
|
for (auto &p : dst_places) {
|
||||||
|
auto *holder = new framework::ReaderHolder();
|
||||||
|
if (use_double_buffer) {
|
||||||
|
holder->Reset(
|
||||||
|
framework::MakeDecoratedReader<operators::reader::BufferedReader>(
|
||||||
|
reader, p, 2));
|
||||||
|
} else {
|
||||||
|
if (platform::is_gpu_place(p)) {
|
||||||
|
PADDLE_THROW(
|
||||||
|
"Place cannot be CUDAPlace when use_double_buffer is False");
|
||||||
|
}
|
||||||
|
holder->Reset(reader);
|
||||||
|
}
|
||||||
|
readers_.emplace_back(holder);
|
||||||
|
}
|
||||||
|
|
||||||
|
futures_.resize(dst_places.size());
|
||||||
|
ret_.resize(dst_places.size());
|
||||||
|
ReadAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
ResultDictList ReadNext() {
|
||||||
|
bool success = WaitFutures();
|
||||||
|
|
||||||
|
if (!success) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
ResultDictList result(ret_.size());
|
||||||
|
for (size_t i = 0; i < ret_.size(); ++i) {
|
||||||
|
for (size_t j = 0; j < names_.size(); ++j) {
|
||||||
|
result[i].emplace(names_[j], std::move(ret_[i][j]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ReadAsync();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
void Reset() {
|
||||||
|
Shutdown();
|
||||||
|
Start();
|
||||||
|
ReadAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
~MultiDeviceFeedReader() {
|
||||||
|
queue_->Close();
|
||||||
|
pool_.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool WaitFutures() {
|
||||||
|
bool success = true;
|
||||||
|
for (auto &f : futures_) {
|
||||||
|
success &= f.get();
|
||||||
|
}
|
||||||
|
return success;
|
||||||
|
}
|
||||||
|
|
||||||
|
void Shutdown() {
|
||||||
|
for (auto &r : readers_) r->Shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
void Start() {
|
||||||
|
for (auto &r : readers_) r->Start();
|
||||||
|
}
|
||||||
|
|
||||||
|
void ReadAsync() {
|
||||||
|
for (size_t i = 0; i < readers_.size(); ++i) {
|
||||||
|
futures_[i] = pool_->enqueue([this, i] {
|
||||||
|
readers_[i]->ReadNext(&ret_[i]);
|
||||||
|
return !ret_[i].empty();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::shared_ptr<operators::reader::LoDTensorBlockingQueue> queue_;
|
||||||
|
std::vector<std::string> names_;
|
||||||
|
std::unique_ptr<::ThreadPool> pool_;
|
||||||
|
|
||||||
|
std::vector<std::unique_ptr<framework::ReaderHolder>> readers_;
|
||||||
|
|
||||||
|
std::vector<std::future<bool>> futures_;
|
||||||
|
std::vector<std::vector<framework::LoDTensor>> ret_;
|
||||||
|
};
|
||||||
|
|
||||||
|
namespace py = pybind11;
|
||||||
|
|
||||||
|
void BindReader(py::module *module) {
|
||||||
|
auto &m = *module;
|
||||||
|
|
||||||
|
namespace reader = ::paddle::operators::reader;
|
||||||
|
|
||||||
|
py::class_<framework::ReaderHolder>(m, "Reader", "")
|
||||||
|
.def("start", &framework::ReaderHolder::Start)
|
||||||
|
.def("reset", &framework::ReaderHolder::ResetAll);
|
||||||
|
|
||||||
|
py::class_<MultiDeviceFeedReader>(m, "MultiDeviceFeedReader", "")
|
||||||
|
.def("read_next", &MultiDeviceFeedReader::ReadNext,
|
||||||
|
py::call_guard<py::gil_scoped_release>())
|
||||||
|
.def("reset", &MultiDeviceFeedReader::Reset,
|
||||||
|
py::call_guard<py::gil_scoped_release>());
|
||||||
|
|
||||||
|
m.def("create_py_reader",
|
||||||
|
[](const std::shared_ptr<operators::reader::LoDTensorBlockingQueue>
|
||||||
|
&queue,
|
||||||
|
const std::vector<std::string> &names,
|
||||||
|
const std::vector<platform::Place> &dst_places,
|
||||||
|
bool use_double_buffer) {
|
||||||
|
return new MultiDeviceFeedReader(queue, names, dst_places,
|
||||||
|
use_double_buffer);
|
||||||
|
},
|
||||||
|
py::return_value_policy::take_ownership);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace pybind
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "pybind11/pybind11.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace pybind {
|
||||||
|
|
||||||
|
void BindReader(pybind11::module *module);
|
||||||
|
|
||||||
|
} // namespace pybind
|
||||||
|
} // namespace paddle
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,175 @@
|
|||||||
|
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import paddle
|
||||||
|
import paddle.fluid as fluid
|
||||||
|
import numpy as np
|
||||||
|
import time
|
||||||
|
import six
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
EPOCH_NUM = 60
|
||||||
|
BATCH_SIZE = 32
|
||||||
|
CLASS_NUM = 10
|
||||||
|
|
||||||
|
|
||||||
|
def random_reader():
|
||||||
|
np.random.seed(1)
|
||||||
|
for i in range(BATCH_SIZE * 40):
|
||||||
|
image = np.random.random([784])
|
||||||
|
label = np.random.random_integers(low=0, high=CLASS_NUM - 1)
|
||||||
|
yield image, label
|
||||||
|
|
||||||
|
|
||||||
|
def simple_fc_net(places, use_legacy_py_reader, use_double_buffer):
|
||||||
|
startup_prog = fluid.Program()
|
||||||
|
main_prog = fluid.Program()
|
||||||
|
startup_prog.random_seed = 1
|
||||||
|
main_prog.random_seed = 1
|
||||||
|
|
||||||
|
with fluid.unique_name.guard():
|
||||||
|
with fluid.program_guard(main_prog, startup_prog):
|
||||||
|
image = fluid.layers.data(
|
||||||
|
name='image', shape=[784], dtype='float32')
|
||||||
|
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
|
||||||
|
py_reader = fluid.io.PyReader(
|
||||||
|
feed_list=[image, label],
|
||||||
|
capacity=4,
|
||||||
|
iterable=not use_legacy_py_reader,
|
||||||
|
use_double_buffer=use_double_buffer)
|
||||||
|
hidden = image
|
||||||
|
for hidden_size in [10, 20, 30]:
|
||||||
|
hidden = fluid.layers.fc(
|
||||||
|
hidden,
|
||||||
|
size=hidden_size,
|
||||||
|
act='tanh',
|
||||||
|
bias_attr=fluid.ParamAttr(
|
||||||
|
initializer=fluid.initializer.Constant(value=1.0)))
|
||||||
|
|
||||||
|
predict_label = fluid.layers.fc(hidden,
|
||||||
|
size=CLASS_NUM,
|
||||||
|
act='softmax')
|
||||||
|
loss = fluid.layers.mean(
|
||||||
|
fluid.layers.cross_entropy(
|
||||||
|
input=predict_label, label=label))
|
||||||
|
|
||||||
|
optimizer = fluid.optimizer.Adam()
|
||||||
|
optimizer.minimize(loss)
|
||||||
|
return startup_prog, main_prog, py_reader, loss
|
||||||
|
|
||||||
|
|
||||||
|
class TestBase(unittest.TestCase):
|
||||||
|
def run_main(self, use_legacy_py_reader, with_data_parallel, places,
|
||||||
|
use_double_buffer):
|
||||||
|
scope = fluid.Scope()
|
||||||
|
with fluid.scope_guard(scope):
|
||||||
|
startup_prog, main_prog, py_reader, loss = simple_fc_net(
|
||||||
|
places, use_legacy_py_reader, use_double_buffer)
|
||||||
|
|
||||||
|
reader = paddle.batch(random_reader, batch_size=BATCH_SIZE)
|
||||||
|
|
||||||
|
ps = places if use_double_buffer else fluid.cpu_places(len(places))
|
||||||
|
|
||||||
|
py_reader.decorate_sample_list_generator(
|
||||||
|
reader, places=ps if py_reader.iterable else None)
|
||||||
|
|
||||||
|
exe = fluid.Executor(place=places[0])
|
||||||
|
exe.run(startup_prog)
|
||||||
|
|
||||||
|
prog = fluid.CompiledProgram(main_prog)
|
||||||
|
if with_data_parallel:
|
||||||
|
prog = prog.with_data_parallel(
|
||||||
|
loss_name=loss.name, places=places)
|
||||||
|
|
||||||
|
step = 0
|
||||||
|
step_list = []
|
||||||
|
loss_list = []
|
||||||
|
start_t = time.time()
|
||||||
|
if not py_reader.iterable:
|
||||||
|
for _ in six.moves.range(EPOCH_NUM):
|
||||||
|
step = 0
|
||||||
|
py_reader.start()
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
L, = exe.run(program=prog,
|
||||||
|
fetch_list=[loss],
|
||||||
|
use_program_cache=True)
|
||||||
|
loss_list.append(np.mean(L))
|
||||||
|
step += 1
|
||||||
|
except fluid.core.EOFException:
|
||||||
|
py_reader.reset()
|
||||||
|
break
|
||||||
|
step_list.append(step)
|
||||||
|
else:
|
||||||
|
for _ in six.moves.range(EPOCH_NUM):
|
||||||
|
step = 0
|
||||||
|
for d in py_reader():
|
||||||
|
assert len(d) == len(places)
|
||||||
|
for i, item in enumerate(d):
|
||||||
|
image = item['image']
|
||||||
|
label = item['label']
|
||||||
|
assert image.shape() == [BATCH_SIZE, 784]
|
||||||
|
assert label.shape() == [BATCH_SIZE, 1]
|
||||||
|
assert image._place()._equals(ps[i])
|
||||||
|
assert label._place()._equals(ps[i])
|
||||||
|
L, = exe.run(program=prog,
|
||||||
|
feed=d,
|
||||||
|
fetch_list=[loss],
|
||||||
|
use_program_cache=True)
|
||||||
|
loss_list.append(np.mean(L))
|
||||||
|
step += 1
|
||||||
|
step_list.append(step)
|
||||||
|
end_t = time.time()
|
||||||
|
ret = {
|
||||||
|
"time": end_t - start_t,
|
||||||
|
"step": step_list,
|
||||||
|
"loss": np.array(loss_list)
|
||||||
|
}
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def prepare_places(self, with_data_parallel, with_cpu=True, with_gpu=True):
|
||||||
|
places = []
|
||||||
|
if with_cpu:
|
||||||
|
places.append([fluid.CPUPlace()])
|
||||||
|
if with_data_parallel:
|
||||||
|
places.append([fluid.CPUPlace()] * 2)
|
||||||
|
|
||||||
|
if with_gpu and fluid.core.is_compiled_with_cuda():
|
||||||
|
tmp = fluid.cuda_places()
|
||||||
|
assert len(tmp) > 0, "no gpu detected"
|
||||||
|
if with_data_parallel:
|
||||||
|
places.append(tmp)
|
||||||
|
places.append([tmp[0]])
|
||||||
|
return places
|
||||||
|
|
||||||
|
def test_main(self):
|
||||||
|
for with_data_parallel in [True, False]:
|
||||||
|
for p in self.prepare_places(with_data_parallel):
|
||||||
|
for use_double_buffer in [False, True]:
|
||||||
|
results = []
|
||||||
|
for use_legacy_py_reader in [False, True]:
|
||||||
|
ret = self.run_main(
|
||||||
|
use_legacy_py_reader=use_legacy_py_reader,
|
||||||
|
with_data_parallel=with_data_parallel,
|
||||||
|
places=p,
|
||||||
|
use_double_buffer=use_double_buffer)
|
||||||
|
results.append(ret)
|
||||||
|
if not use_double_buffer:
|
||||||
|
diff = np.max(
|
||||||
|
np.abs(results[0]['loss'] - results[1]['loss']))
|
||||||
|
self.assertLess(diff, 1e-3)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@ -0,0 +1,137 @@
|
|||||||
|
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import paddle
|
||||||
|
import paddle.fluid as fluid
|
||||||
|
import math
|
||||||
|
import unittest
|
||||||
|
import numpy as np
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.environ['CPU_NUM'] = '1'
|
||||||
|
|
||||||
|
|
||||||
|
def random_reader(sample_num):
|
||||||
|
def __impl__():
|
||||||
|
for _ in range(sample_num):
|
||||||
|
yield np.random.random(
|
||||||
|
size=[784]).astype('float32'), np.random.random_integers(
|
||||||
|
low=0, high=9, size=[1]).astype('int64')
|
||||||
|
|
||||||
|
return paddle.reader.cache(__impl__)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCaseBase(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.batch_size = 32
|
||||||
|
self.epoch_num = 2
|
||||||
|
self.sample_num = 165
|
||||||
|
|
||||||
|
def generate_all_data(self, reader):
|
||||||
|
ret = []
|
||||||
|
for d in reader():
|
||||||
|
slots = [[], []]
|
||||||
|
for item in d:
|
||||||
|
slots[0].append(item[0])
|
||||||
|
slots[1].append(item[1])
|
||||||
|
slots = [np.array(slot) for slot in slots]
|
||||||
|
ret.append(slots)
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def run_main(self, reader, use_sample_generator, iterable, drop_last):
|
||||||
|
image = fluid.layers.data(name='image', dtype='float32', shape=[784])
|
||||||
|
label = fluid.layers.data(name='label', dtype='int64', shape=[1])
|
||||||
|
py_reader = fluid.io.PyReader(
|
||||||
|
feed_list=[image, label],
|
||||||
|
capacity=16,
|
||||||
|
iterable=iterable,
|
||||||
|
use_double_buffer=False)
|
||||||
|
|
||||||
|
batch_reader = paddle.batch(reader, self.batch_size, drop_last)
|
||||||
|
all_datas = self.generate_all_data(batch_reader)
|
||||||
|
|
||||||
|
if not use_sample_generator:
|
||||||
|
py_reader.decorate_sample_list_generator(
|
||||||
|
batch_reader, places=fluid.cpu_places())
|
||||||
|
else:
|
||||||
|
py_reader.decorate_sample_generator(
|
||||||
|
reader, self.batch_size, drop_last, places=fluid.cpu_places())
|
||||||
|
|
||||||
|
if drop_last:
|
||||||
|
batch_num = int(self.sample_num / self.batch_size)
|
||||||
|
else:
|
||||||
|
batch_num = math.ceil(float(self.sample_num) / self.batch_size)
|
||||||
|
|
||||||
|
exe = fluid.Executor(fluid.CPUPlace())
|
||||||
|
exe.run(fluid.default_startup_program())
|
||||||
|
for _ in range(self.epoch_num):
|
||||||
|
if py_reader.iterable:
|
||||||
|
step = 0
|
||||||
|
for data in py_reader():
|
||||||
|
img, lbl = exe.run(feed=data, fetch_list=[image, label])
|
||||||
|
self.assertArrayEqual(img, all_datas[step][0])
|
||||||
|
self.assertArrayEqual(lbl, all_datas[step][1])
|
||||||
|
step += 1
|
||||||
|
self.assertEqual(step, len(all_datas))
|
||||||
|
else:
|
||||||
|
step = 0
|
||||||
|
try:
|
||||||
|
py_reader.start()
|
||||||
|
while True:
|
||||||
|
img, lbl = exe.run(fetch_list=[image, label])
|
||||||
|
self.assertArrayEqual(img, all_datas[step][0])
|
||||||
|
self.assertArrayEqual(lbl, all_datas[step][1])
|
||||||
|
step += 1
|
||||||
|
except fluid.core.EOFException:
|
||||||
|
py_reader.reset()
|
||||||
|
self.assertEqual(step, len(all_datas))
|
||||||
|
break
|
||||||
|
|
||||||
|
def assertArrayEqual(self, arr1, arr2):
|
||||||
|
self.assertEqual(arr1.shape, arr2.shape)
|
||||||
|
self.assertTrue((arr1 == arr2).all())
|
||||||
|
|
||||||
|
def test_main(self):
|
||||||
|
reader = random_reader(self.sample_num)
|
||||||
|
for use_sample_generator in [False, True]:
|
||||||
|
for iterable in [False, True]:
|
||||||
|
for drop_last in [False, True]:
|
||||||
|
with fluid.program_guard(fluid.Program(), fluid.Program()):
|
||||||
|
self.run_main(reader, use_sample_generator, iterable,
|
||||||
|
drop_last)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCase1(TestCaseBase):
|
||||||
|
def setUp(self):
|
||||||
|
self.batch_size = 32
|
||||||
|
self.epoch_num = 10
|
||||||
|
self.sample_num = 160
|
||||||
|
|
||||||
|
|
||||||
|
class TestCase2(TestCaseBase):
|
||||||
|
def setUp(self):
|
||||||
|
self.batch_size = 32
|
||||||
|
self.epoch_num = 2
|
||||||
|
self.sample_num = 200
|
||||||
|
|
||||||
|
|
||||||
|
class TestCase3(TestCaseBase):
|
||||||
|
def setUp(self):
|
||||||
|
self.batch_size = 32
|
||||||
|
self.epoch_num = 2
|
||||||
|
self.sample_num = 159
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in new issue