Add iterable dataset support for multiprocess DataLoader (#25558)
* add IterableDataset support in multiprocess DataLoader. test=developrevert-24895-update_cub
Kaipeng Deng
5 years ago
committed by
GitHub
parent
54003b873e
commit
dbc88bb900
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,53 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class _DatasetFetcher(object):
|
||||
def __init__(self, dataset, collate_fn, drop_last):
|
||||
self.dataset = dataset
|
||||
self.collate_fn = collate_fn
|
||||
self.drop_last = drop_last
|
||||
|
||||
def fetch(self, batch_indices):
|
||||
raise NotImplementedError("'fetch' not implement for class {}".format(
|
||||
self.__class__.__name__))
|
||||
|
||||
|
||||
class _IterableDatasetFetcher(_DatasetFetcher):
|
||||
def __init__(self, dataset, collate_fn, drop_last):
|
||||
super(_IterableDatasetFetcher, self).__init__(dataset, collate_fn,
|
||||
drop_last)
|
||||
self.dataset_iter = iter(dataset)
|
||||
|
||||
def fetch(self, batch_indices):
|
||||
data = []
|
||||
for _ in batch_indices:
|
||||
try:
|
||||
data.append(next(self.dataset_iter))
|
||||
except StopIteration:
|
||||
break
|
||||
if len(data) == 0 or (self.drop_last and
|
||||
len(data) < len(batch_indices)):
|
||||
raise StopIteration
|
||||
|
||||
return self.collate_fn(data)
|
||||
|
||||
|
||||
class _MapDatasetFetcher(_DatasetFetcher):
|
||||
def __init__(self, dataset, collate_fn, drop_last):
|
||||
super(_MapDatasetFetcher, self).__init__(dataset, collate_fn, drop_last)
|
||||
|
||||
def fetch(self, batch_indices):
|
||||
data = [self.dataset[idx] for idx in batch_indices]
|
||||
return self.collate_fn(data)
|
||||
@ -0,0 +1,124 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import os
|
||||
import sys
|
||||
import six
|
||||
import time
|
||||
import unittest
|
||||
import multiprocessing
|
||||
import numpy as np
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.io import Dataset, BatchSampler, DataLoader
|
||||
from paddle.fluid.dygraph.nn import Linear
|
||||
from paddle.fluid.dygraph.base import to_variable
|
||||
|
||||
from test_multiprocess_dataloader_iterable_dataset_static import RandomDataset, prepare_places
|
||||
from test_multiprocess_dataloader_iterable_dataset_static import EPOCH_NUM, BATCH_SIZE, IMAGE_SIZE, SAMPLE_NUM, CLASS_NUM
|
||||
|
||||
|
||||
class SimpleFCNet(fluid.dygraph.Layer):
|
||||
def __init__(self):
|
||||
super(SimpleFCNet, self).__init__()
|
||||
|
||||
param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant(
|
||||
value=0.8))
|
||||
bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant(
|
||||
value=0.5))
|
||||
self._fcs = []
|
||||
in_channel = IMAGE_SIZE
|
||||
for hidden_size in [10, 20, 30]:
|
||||
self._fcs.append(
|
||||
Linear(
|
||||
in_channel,
|
||||
hidden_size,
|
||||
act='tanh',
|
||||
param_attr=param_attr,
|
||||
bias_attr=bias_attr))
|
||||
in_channel = hidden_size
|
||||
self._fcs.append(
|
||||
Linear(
|
||||
in_channel,
|
||||
CLASS_NUM,
|
||||
act='softmax',
|
||||
param_attr=param_attr,
|
||||
bias_attr=bias_attr))
|
||||
|
||||
def forward(self, image):
|
||||
out = image
|
||||
for fc in self._fcs:
|
||||
out = fc(out)
|
||||
return out
|
||||
|
||||
|
||||
class TestDygraphDataLoader(unittest.TestCase):
|
||||
def run_main(self, num_workers, places):
|
||||
fluid.default_startup_program().random_seed = 1
|
||||
fluid.default_main_program().random_seed = 1
|
||||
with fluid.dygraph.guard(places[0]):
|
||||
fc_net = SimpleFCNet()
|
||||
optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters())
|
||||
|
||||
dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
places=places,
|
||||
num_workers=num_workers,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_last=True)
|
||||
|
||||
step_list = []
|
||||
loss_list = []
|
||||
start_t = time.time()
|
||||
for _ in six.moves.range(EPOCH_NUM):
|
||||
step = 0
|
||||
for image, label in dataloader():
|
||||
out = fc_net(image)
|
||||
loss = fluid.layers.cross_entropy(out, label)
|
||||
avg_loss = fluid.layers.reduce_mean(loss)
|
||||
avg_loss.backward()
|
||||
optimizer.minimize(avg_loss)
|
||||
fc_net.clear_gradients()
|
||||
|
||||
loss_list.append(np.mean(avg_loss.numpy()))
|
||||
step += 1
|
||||
step_list.append(step)
|
||||
|
||||
end_t = time.time()
|
||||
ret = {
|
||||
"time": end_t - start_t,
|
||||
"step": step_list,
|
||||
"loss": np.array(loss_list)
|
||||
}
|
||||
print("time cost", ret['time'], 'step_list', ret['step'])
|
||||
return ret
|
||||
|
||||
def test_main(self):
|
||||
# dynamic graph do not run with_data_parallel
|
||||
for p in prepare_places(False):
|
||||
results = []
|
||||
for num_workers in [0, 2]:
|
||||
print(self.__class__.__name__, p, num_workers)
|
||||
sys.stdout.flush()
|
||||
ret = self.run_main(num_workers=num_workers, places=p)
|
||||
results.append(ret)
|
||||
assert results[0]['loss'].shape[0] * 2 == results[1]['loss'].shape[
|
||||
0]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,111 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import math
|
||||
import unittest
|
||||
import numpy as np
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.io import IterableDataset, BatchSampler, DataLoader, get_worker_info
|
||||
|
||||
|
||||
class RangeIterableDatasetSplit(IterableDataset):
|
||||
def __init__(self, start, end):
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
||||
def __iter__(self):
|
||||
worker_info = get_worker_info()
|
||||
if worker_info is None:
|
||||
iter_start = self.start
|
||||
iter_end = self.end
|
||||
else:
|
||||
per_worker = int(
|
||||
math.ceil((self.end - self.start) / float(
|
||||
worker_info.num_workers)))
|
||||
worker_id = worker_info.id
|
||||
iter_start = self.start + worker_id * per_worker
|
||||
iter_end = min(iter_start + per_worker, self.end)
|
||||
|
||||
for i in range(iter_start, iter_end):
|
||||
yield np.array([i])
|
||||
|
||||
|
||||
class TestDynamicDataLoaderIterSplit(unittest.TestCase):
|
||||
def test_main(self):
|
||||
place = fluid.CPUPlace()
|
||||
with fluid.dygraph.guard(place):
|
||||
dataset = RangeIterableDatasetSplit(0, 10)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
places=place,
|
||||
num_workers=2,
|
||||
batch_size=1,
|
||||
drop_last=True)
|
||||
|
||||
rets = []
|
||||
for d in dataloader:
|
||||
rets.append(d[0].numpy()[0][0])
|
||||
|
||||
assert tuple(sorted(rets)) == tuple(range(0, 10))
|
||||
|
||||
|
||||
class RangeIterableDataset(IterableDataset):
|
||||
def __init__(self, start, end):
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
||||
def __iter__(self):
|
||||
for i in range(self.start, self.end):
|
||||
yield np.array([i])
|
||||
|
||||
|
||||
class TestDynamicDataLoaderIterInitFuncSplit(unittest.TestCase):
|
||||
def test_main(self):
|
||||
place = fluid.CPUPlace()
|
||||
with fluid.dygraph.guard(place):
|
||||
dataset = RangeIterableDataset(0, 10)
|
||||
|
||||
def worker_spliter(worker_id):
|
||||
worker_info = get_worker_info()
|
||||
|
||||
dataset = worker_info.dataset
|
||||
start = dataset.start
|
||||
end = dataset.end
|
||||
num_per_worker = int(
|
||||
math.ceil((end - start) / float(worker_info.num_workers)))
|
||||
|
||||
worker_id = worker_info.id
|
||||
dataset.start = start + worker_id * num_per_worker
|
||||
dataset.end = min(dataset.start + num_per_worker, end)
|
||||
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
places=place,
|
||||
num_workers=1,
|
||||
batch_size=1,
|
||||
drop_last=True,
|
||||
worker_init_fn=worker_spliter)
|
||||
|
||||
rets = []
|
||||
for d in dataloader:
|
||||
rets.append(d[0].numpy()[0][0])
|
||||
|
||||
assert tuple(sorted(rets)) == tuple(range(0, 10))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,171 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import os
|
||||
import sys
|
||||
import six
|
||||
import time
|
||||
import unittest
|
||||
import multiprocessing
|
||||
import numpy as np
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.io import IterableDataset, BatchSampler, DataLoader, get_worker_info
|
||||
|
||||
EPOCH_NUM = 2
|
||||
BATCH_SIZE = 8
|
||||
IMAGE_SIZE = 32
|
||||
SAMPLE_NUM = 80
|
||||
CLASS_NUM = 10
|
||||
|
||||
|
||||
class RandomDataset(IterableDataset):
|
||||
def __init__(self, sample_num, class_num):
|
||||
self.sample_num = sample_num
|
||||
self.class_num = class_num
|
||||
|
||||
def __iter__(self):
|
||||
for i in range(self.sample_num):
|
||||
np.random.seed(i)
|
||||
image = np.random.random([IMAGE_SIZE]).astype('float32')
|
||||
label = np.random.randint(0, self.class_num - 1,
|
||||
(1, )).astype('int64')
|
||||
yield image, label
|
||||
|
||||
|
||||
def simple_fc_net_static():
|
||||
startup_prog = fluid.Program()
|
||||
main_prog = fluid.Program()
|
||||
startup_prog.random_seed = 1
|
||||
main_prog.random_seed = 1
|
||||
|
||||
with fluid.unique_name.guard():
|
||||
with fluid.program_guard(main_prog, startup_prog):
|
||||
image = fluid.data(
|
||||
name='image', shape=[None, IMAGE_SIZE], dtype='float32')
|
||||
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
|
||||
hidden = image
|
||||
param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant(
|
||||
value=0.8))
|
||||
bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant(
|
||||
value=0.5))
|
||||
for hidden_size in [10, 20, 30]:
|
||||
hidden = fluid.layers.fc(hidden,
|
||||
size=hidden_size,
|
||||
act='tanh',
|
||||
param_attr=param_attr,
|
||||
bias_attr=bias_attr)
|
||||
|
||||
predict_label = fluid.layers.fc(hidden,
|
||||
size=CLASS_NUM,
|
||||
act='softmax',
|
||||
param_attr=param_attr,
|
||||
bias_attr=bias_attr)
|
||||
loss = fluid.layers.reduce_mean(
|
||||
fluid.layers.cross_entropy(
|
||||
input=predict_label, label=label))
|
||||
|
||||
optimizer = fluid.optimizer.Adam()
|
||||
optimizer.minimize(loss)
|
||||
return startup_prog, main_prog, image, label, loss
|
||||
|
||||
|
||||
def prepare_places(with_data_parallel, with_cpu=False, with_gpu=True):
|
||||
places = []
|
||||
if with_cpu:
|
||||
places.append([fluid.CPUPlace()])
|
||||
if with_data_parallel:
|
||||
places.append([fluid.CPUPlace()] * 2)
|
||||
|
||||
if with_gpu and fluid.core.is_compiled_with_cuda():
|
||||
tmp = fluid.cuda_places()[:2]
|
||||
assert len(tmp) > 0, "no gpu detected"
|
||||
if with_data_parallel:
|
||||
places.append(tmp)
|
||||
places.append([tmp[0]])
|
||||
return places
|
||||
|
||||
|
||||
class TestStaticDataLoader(unittest.TestCase):
|
||||
def run_main(self, num_workers, places):
|
||||
scope = fluid.Scope()
|
||||
with fluid.scope_guard(scope):
|
||||
startup_prog, main_prog, image, label, loss = simple_fc_net_static()
|
||||
|
||||
dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM)
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
feed_list=[image, label],
|
||||
places=places,
|
||||
num_workers=num_workers,
|
||||
batch_size=BATCH_SIZE,
|
||||
drop_last=True)
|
||||
# assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE)
|
||||
|
||||
exe = fluid.Executor(place=places[0])
|
||||
exe.run(startup_prog)
|
||||
|
||||
prog = fluid.CompiledProgram(main_prog)
|
||||
if len(places) > 1:
|
||||
prog = prog.with_data_parallel(
|
||||
loss_name=loss.name, places=places)
|
||||
|
||||
step_list = []
|
||||
loss_list = []
|
||||
start_t = time.time()
|
||||
for i in six.moves.range(EPOCH_NUM):
|
||||
step = 0
|
||||
for d in dataloader:
|
||||
assert len(d) == len(places), "{} != {}".format(
|
||||
len(d), len(places))
|
||||
for i, item in enumerate(d):
|
||||
image = item['image']
|
||||
label = item['label']
|
||||
assert image.shape() == [BATCH_SIZE, IMAGE_SIZE]
|
||||
assert label.shape() == [BATCH_SIZE, 1]
|
||||
assert image._place()._equals(places[i])
|
||||
assert label._place()._equals(places[i])
|
||||
L, = exe.run(program=prog,
|
||||
feed=d,
|
||||
fetch_list=[loss],
|
||||
use_program_cache=True)
|
||||
loss_list.append(np.mean(L))
|
||||
step += 1
|
||||
step_list.append(step)
|
||||
|
||||
end_t = time.time()
|
||||
ret = {
|
||||
"time": end_t - start_t,
|
||||
"step": step_list,
|
||||
"loss": np.array(loss_list)
|
||||
}
|
||||
print("time cost", ret['time'], 'step_list', ret['step'])
|
||||
return ret
|
||||
|
||||
def test_main(self):
|
||||
for p in prepare_places(True):
|
||||
results = []
|
||||
for num_workers in [0, 2]:
|
||||
print(self.__class__.__name__, p, num_workers)
|
||||
sys.stdout.flush()
|
||||
ret = self.run_main(num_workers=num_workers, places=p)
|
||||
results.append(ret)
|
||||
assert results[0]['loss'].shape[0] * 2 == results[1]['loss'].shape[
|
||||
0]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue