Refine DataLoader support multi-processing (#23107)
* add DataLoader, Dataset, BatchSamplerrevert-22778-infer_var_type
Kaipeng Deng
5 years ago
committed by
GitHub
parent
76d78c6387
commit
80cf3c3c4d
@ -0,0 +1,24 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
from . import dataset
|
||||
from .dataset import *
|
||||
|
||||
from . import batch_sampler
|
||||
from .batch_sampler import *
|
||||
|
||||
__all__ = dataset.__all__ \
|
||||
+ batch_sampler.__all__
|
||||
@ -0,0 +1,143 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import division
|
||||
|
||||
import numpy as np
|
||||
from .dataset import Dataset
|
||||
|
||||
__all__ = ["BatchSampler"]
|
||||
|
||||
|
||||
class BatchSampler(object):
|
||||
"""
|
||||
A base implement of batch sampler used by `paddle.io.DataLoader`
|
||||
which yield mini-batch indices(a list/tuple with length as
|
||||
mini-batch size and holds sample indices) iterably.
|
||||
|
||||
Batch sampler used by :code:`paddle.io.DataLoader` should be a subclass
|
||||
of :code:`paddle.io.BatchSampler`, BatchSampler subclasses should
|
||||
implement following methods:
|
||||
|
||||
:code:`__iter__`: return mini-batch indices iterably.
|
||||
|
||||
:code:`__len__`: get mini-batch number in an epoch.
|
||||
|
||||
|
||||
Args:
|
||||
dataset(Dataset): this could be a :code:`paddle.io.Dataset`
|
||||
implement or other python object which implemented
|
||||
:code:`__len__` for BatchSampler to get indices as the
|
||||
range of :attr:`dataset` length. Default None.
|
||||
indices (list|tuple): a substitution parameter for
|
||||
:attr:`dataset` either :attr:`dataset` or
|
||||
:attr:`indices` should be set, give the whole
|
||||
indices to sampler from directly. Default None.
|
||||
shuffle(bool): whether to shuffle indices order before genrating
|
||||
batch indices. Default False.
|
||||
batch_size(int): sample indice number in a mini-batch indices.
|
||||
drop_last(bool): whether drop the last incomplete batch dataset size
|
||||
is not divisible by the batch size. Default False
|
||||
|
||||
Returns:
|
||||
BatchSampler: an iterable object for indices iterating
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from paddle.io import BatchSampler, Dataset
|
||||
|
||||
# init with indices
|
||||
bs = BatchSampler(indices=list(range(100)),
|
||||
shuffle=True,
|
||||
batch_size=8,
|
||||
drop_last=True)
|
||||
|
||||
for batch_indices in bs:
|
||||
print(batch_indices)
|
||||
|
||||
# init with dataset
|
||||
class RandomDataset(Dataset):
|
||||
def __init__(self, num_samples):
|
||||
self.num_samples = num_samples
|
||||
|
||||
def __getitem__(self, idx):
|
||||
image = np.random.random([784]).astype('float32')
|
||||
label = np.random.randint(0, 9, (1, )).astype('int64')
|
||||
return image, label
|
||||
|
||||
def __len__(self):
|
||||
return self.num_samples
|
||||
|
||||
bs = BatchSampler(dataset=RandomDataset(100),
|
||||
shuffle=False,
|
||||
batch_size=16,
|
||||
drop_last=False)
|
||||
|
||||
for batch_indices in bs:
|
||||
print(batch_indices)
|
||||
|
||||
see `paddle.io.DataLoader`
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
dataset=None,
|
||||
indices=None,
|
||||
shuffle=False,
|
||||
batch_size=1,
|
||||
drop_last=False):
|
||||
if dataset is None:
|
||||
assert indices is not None, \
|
||||
"either dataset or indices should be set"
|
||||
assert isinstance(indices, list) or isinstance(indices, tuple), \
|
||||
"indices should be a list or tuple, but got {}".format(type(indices))
|
||||
self.indices = indices
|
||||
else:
|
||||
assert isinstance(dataset, Dataset), \
|
||||
"dataset should be an instance of paddle.io.Dataset"
|
||||
assert indices is None, \
|
||||
"should not set both dataset and indices"
|
||||
self.indices = list(range(len(dataset)))
|
||||
|
||||
assert isinstance(batch_size, int) and batch_size > 0, \
|
||||
"batch_size should be a positive integer, but got {}".format(batch_size)
|
||||
self.batch_size = batch_size
|
||||
assert isinstance(shuffle, bool), \
|
||||
"shuffle should be a boolean value, but got {}".format(type(shuffle))
|
||||
self.shuffle = shuffle
|
||||
assert isinstance(drop_last, bool), \
|
||||
"drop_last should be a boolean value, but got {}".format(type(drop_last))
|
||||
self.drop_last = drop_last
|
||||
|
||||
def __iter__(self):
|
||||
if self.shuffle:
|
||||
np.random.shuffle(self.indices)
|
||||
_iter = iter(self.indices)
|
||||
|
||||
batch_indices = []
|
||||
for idx in _iter:
|
||||
batch_indices.append(idx)
|
||||
if len(batch_indices) == self.batch_size:
|
||||
yield batch_indices
|
||||
batch_indices = []
|
||||
if not self.drop_last and len(batch_indices) > 0:
|
||||
yield batch_indices
|
||||
|
||||
def __len__(self):
|
||||
num_samples = len(self.indices)
|
||||
num_samples += int(not self.drop_last) * (self.batch_size - 1)
|
||||
return num_samples // self.batch_size
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,73 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import paddle.dataset.common
|
||||
|
||||
__all__ = ["Dataset"]
|
||||
|
||||
|
||||
class Dataset(object):
|
||||
"""
|
||||
An abstract class to encapsulates methods and behaviors of datasets.
|
||||
|
||||
All datasets in map-style(dataset samples can be get by a given key)
|
||||
should be a subclass of `paddle.io.Dataset`. All subclasses should
|
||||
implement following methods:
|
||||
|
||||
:code:`__getitem__`: get sample from dataset with a given index. This
|
||||
method is required by reading dataset sample in :code:`paddle.io.DataLoader`.
|
||||
|
||||
:code:`__len__`: return dataset sample number. This method is required
|
||||
by some implements of :code:`paddle.io.BatchSampler`
|
||||
|
||||
see :code:`paddle.io.DataLoader`.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import numpy as np
|
||||
from paddle.io import Dataset
|
||||
|
||||
# define a random dataset
|
||||
class RandomDataset(Dataset):
|
||||
def __init__(self, num_samples):
|
||||
self.num_samples = num_samples
|
||||
|
||||
def __getitem__(self, idx):
|
||||
image = np.random.random([784]).astype('float32')
|
||||
label = np.random.randint(0, 9, (1, )).astype('int64')
|
||||
return image, label
|
||||
|
||||
def __len__(self):
|
||||
return self.num_samples
|
||||
|
||||
dataset = RandomDataset(10)
|
||||
for i in range(len(dataset)):
|
||||
print(dataset[i])
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getitem__(self, idx):
|
||||
raise NotImplementedError("'{}' not implement in class "\
|
||||
"{}".format('__getitem__', self.__class__.__name__))
|
||||
|
||||
def __len__(self):
|
||||
raise NotImplementedError("'{}' not implement in class "\
|
||||
"{}".format('__len__', self.__class__.__name__))
|
||||
@ -0,0 +1,139 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import six
|
||||
import sys
|
||||
import signal
|
||||
import atexit
|
||||
|
||||
from . import core
|
||||
|
||||
# NOTE: queue has a different name in python2 and python3
|
||||
if six.PY2:
|
||||
import Queue as queue
|
||||
else:
|
||||
import queue
|
||||
|
||||
# NOTE: [ mmap files clear ] If there is still data in the multiprocess queue when the main process finishes reading,
|
||||
# the data in the queue needs to be popped. Then the LoDTensor read by the main process
|
||||
# from the child process will automatically clear the memory-mapped file.
|
||||
multiprocess_queue_set = set()
|
||||
|
||||
|
||||
def _clear_multiprocess_queue_set():
|
||||
global multiprocess_queue_set
|
||||
for data_queue in multiprocess_queue_set:
|
||||
while True:
|
||||
try:
|
||||
data_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
|
||||
# NOTE: main process clear function at exit
|
||||
def _cleanup():
|
||||
# NOTE: inter-process Queue shared memory objects clear function
|
||||
_clear_multiprocess_queue_set()
|
||||
# NOTE: main process memory map files clear funciton
|
||||
core._cleanup_mmap_fds()
|
||||
|
||||
|
||||
# NOTE: for child process clear function at exit
|
||||
def _cleanup_mmap():
|
||||
# clear memory map files in child process
|
||||
core._cleanup_mmap_fds()
|
||||
|
||||
|
||||
# NOTE used for register a function to be executed at interpreter exit.
|
||||
class CleanupFuncRegistrar():
|
||||
# Record the cleanup functions that have been executed
|
||||
_executed_func_set = set()
|
||||
# Record the cleanup functions that have been registered
|
||||
_registered_func_set = set()
|
||||
|
||||
@classmethod
|
||||
def register(cls, function, signals=[]):
|
||||
def _func_exectuor():
|
||||
if function not in cls._executed_func_set:
|
||||
try:
|
||||
function()
|
||||
finally:
|
||||
cls._executed_func_set.add(function)
|
||||
|
||||
def _func_register(function):
|
||||
if not callable(function):
|
||||
raise TypeError("%s is not callable object." % (function))
|
||||
# check function object whether hash-able
|
||||
set([function])
|
||||
if function not in cls._registered_func_set:
|
||||
atexit.register(_func_exectuor)
|
||||
cls._registered_func_set.add(function)
|
||||
|
||||
def _signal_handler(signum=None, frame=None):
|
||||
_func_exectuor()
|
||||
if signum is not None:
|
||||
if signum == signal.SIGINT:
|
||||
raise KeyboardInterrupt
|
||||
sys.exit(signum)
|
||||
|
||||
def _signal_register(signals):
|
||||
signals = set(signals)
|
||||
for sig in signals:
|
||||
orig_handler = signal.signal(sig, _signal_handler)
|
||||
if orig_handler not in (signal.SIG_DFL, signal.SIG_IGN):
|
||||
if (sig == signal.SIGINT and
|
||||
orig_handler is signal.default_int_handler):
|
||||
continue
|
||||
if orig_handler not in cls._registered_func_set:
|
||||
atexit.register(orig_handler)
|
||||
cls._registered_func_set.add(orig_handler)
|
||||
|
||||
# deal with signals
|
||||
_signal_register(signals)
|
||||
# deal with function
|
||||
_func_register(function)
|
||||
|
||||
|
||||
# NOTE: [ mmap files clear ] When the main process exits unexpectedly, the remaining
|
||||
# shared memory objects in the inter-process Queue and the main process (mostly in the
|
||||
# BlockingQueue) may not be completely released, resulting in the corresponding
|
||||
# memory-mapped file remaining on the disk (/dev/shm), so register this function
|
||||
# to clean up shared memory objects in these two queues before the python interpreter exits.
|
||||
# NOTE: Currently multi-process DataLoader only supports Linux platform
|
||||
if not (sys.platform == 'darwin' or sys.platform == 'win32'):
|
||||
CleanupFuncRegistrar.register(_cleanup)
|
||||
|
||||
# ------------ SIGCHLD handler setting --------------
|
||||
_SIGCHLD_handler_set = False
|
||||
|
||||
|
||||
def _set_SIGCHLD_handler():
|
||||
global _SIGCHLD_handler_set
|
||||
if _SIGCHLD_handler_set:
|
||||
return
|
||||
|
||||
current_handler = signal.getsignal(signal.SIGCHLD)
|
||||
if not callable(current_handler):
|
||||
current_handler = None
|
||||
|
||||
def __handler__(signum, frame):
|
||||
# NOTE: Here the signum is SIGCHLD, when the child process exits,
|
||||
# this handler will be called whenever the child process exits
|
||||
# normally or abnormally.
|
||||
core._throw_error_if_process_failed()
|
||||
if current_handler is not None:
|
||||
current_handler(signum, frame)
|
||||
|
||||
signal.signal(signal.SIGCHLD, __handler__)
|
||||
_SIGCHLD_handler_set = True
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,120 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import unittest
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.io import BatchSampler, Dataset
|
||||
|
||||
|
||||
class RandomDataset(Dataset):
|
||||
def __init__(self, sample_num, class_num):
|
||||
self.sample_num = sample_num
|
||||
self.class_num = class_num
|
||||
|
||||
def __getitem__(self, idx):
|
||||
np.random.seed(idx)
|
||||
image = np.random.random([IMAGE_SIZE]).astype('float32')
|
||||
label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
|
||||
return image, label
|
||||
|
||||
def __len__(self):
|
||||
return self.sample_num
|
||||
|
||||
|
||||
class TestBatchSampler(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.num_samples = 1000
|
||||
self.num_classes = 10
|
||||
self.batch_size = 32
|
||||
self.shuffle = False
|
||||
self.drop_last = False
|
||||
|
||||
def init_batch_sampler(self):
|
||||
dataset = RandomDataset(self.num_samples, self.num_classes)
|
||||
bs = BatchSampler(
|
||||
dataset=dataset,
|
||||
batch_size=self.batch_size,
|
||||
shuffle=self.shuffle,
|
||||
drop_last=self.drop_last)
|
||||
return bs
|
||||
|
||||
def test_main(self):
|
||||
bs = self.init_batch_sampler()
|
||||
# length check
|
||||
bs_len = (self.num_samples + int(not self.drop_last) \
|
||||
* (self.batch_size - 1)) // self.batch_size
|
||||
self.assertTrue(bs_len == len(bs))
|
||||
|
||||
# output indices check
|
||||
if not self.shuffle:
|
||||
index = 0
|
||||
for indices in bs:
|
||||
for idx in indices:
|
||||
self.assertTrue(index == idx)
|
||||
index += 1
|
||||
|
||||
|
||||
class TestBatchSamplerDropLast(TestBatchSampler):
|
||||
def setUp(self):
|
||||
self.num_samples = 1000
|
||||
self.num_classes = 10
|
||||
self.batch_size = 32
|
||||
self.shuffle = False
|
||||
self.drop_last = True
|
||||
|
||||
|
||||
class TestBatchSamplerShuffle(TestBatchSampler):
|
||||
def setUp(self):
|
||||
self.num_samples = 1000
|
||||
self.num_classes = 10
|
||||
self.batch_size = 32
|
||||
self.shuffle = True
|
||||
self.drop_last = True
|
||||
|
||||
|
||||
class TestBatchSamplerWithIndices(TestBatchSampler):
|
||||
def init_batch_sampler(self):
|
||||
bs = BatchSampler(
|
||||
indices=list(range(self.num_samples)),
|
||||
batch_size=self.batch_size,
|
||||
drop_last=self.drop_last)
|
||||
return bs
|
||||
|
||||
|
||||
class TestBatchSamplerWithIndicesAndDataSource(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.num_samples = 1000
|
||||
self.num_classes = 10
|
||||
self.batch_size = 32
|
||||
self.shuffle = False
|
||||
self.drop_last = True
|
||||
|
||||
def test_main(self):
|
||||
try:
|
||||
dataset = RandomDataset(self.num_samples, self.num_classes)
|
||||
bs = BatchSampler(
|
||||
dataset=dataset,
|
||||
indices=list(range(self.num_samples)),
|
||||
batch_size=self.batch_size,
|
||||
drop_last=self.drop_last)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,41 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import unittest
|
||||
import numpy as np
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.io import *
|
||||
|
||||
|
||||
class TestDatasetAbstract(unittest.TestCase):
|
||||
def test_main(self):
|
||||
dataset = Dataset()
|
||||
try:
|
||||
d = dataset[0]
|
||||
self.assertTrue(False)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
try:
|
||||
l = len(dataset)
|
||||
self.assertTrue(False)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,199 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import os
|
||||
import sys
|
||||
import six
|
||||
import time
|
||||
import unittest
|
||||
import multiprocessing
|
||||
import numpy as np
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.io import Dataset, BatchSampler, DataLoader
|
||||
from paddle.fluid.dygraph.nn import Linear
|
||||
from paddle.fluid.dygraph.base import to_variable
|
||||
|
||||
|
||||
class RandomDataset(Dataset):
|
||||
def __init__(self, sample_num):
|
||||
self.sample_num = sample_num
|
||||
|
||||
def __getitem__(self, idx):
|
||||
np.random.seed(idx)
|
||||
image = np.random.random([784]).astype('float32')
|
||||
label = np.random.randint(0, 9, (1, )).astype('int64')
|
||||
return image, label
|
||||
|
||||
def __len__(self):
|
||||
return self.sample_num
|
||||
|
||||
|
||||
class TestDataLoaderAssert(unittest.TestCase):
|
||||
def test_main(self):
|
||||
place = fluid.cpu_places()[0]
|
||||
with fluid.dygraph.guard(place):
|
||||
dataset = RandomDataset(100)
|
||||
batch_sampler = BatchSampler(dataset=dataset, batch_size=4)
|
||||
|
||||
# dataset is not instance of Dataset
|
||||
try:
|
||||
loader = DataLoader(dataset=batch_sampler, places=place)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
# places is None
|
||||
try:
|
||||
loader = DataLoader(dataset=dataset, places=None)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
# num_workers < 0
|
||||
try:
|
||||
loader = DataLoader(
|
||||
dataset=dataset, places=place, num_workers=-1)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
# timeout < 0
|
||||
try:
|
||||
loader = DataLoader(dataset=dataset, places=place, timeout=-1)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
# batch_sampler is not instance of BatchSampler
|
||||
try:
|
||||
loader = DataLoader(
|
||||
dataset=dataset, places=place, batch_sampler=dataset)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
# set batch_sampler and shuffle/batch_size/drop_last
|
||||
try:
|
||||
loader = DataLoader(
|
||||
dataset=dataset,
|
||||
places=place,
|
||||
batch_sampler=batch_sampler,
|
||||
shuffle=True,
|
||||
drop_last=True)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
# set batch_sampler correctly
|
||||
try:
|
||||
loader = DataLoader(
|
||||
dataset=dataset, places=place, batch_sampler=batch_sampler)
|
||||
self.assertTrue(True)
|
||||
except AssertionError:
|
||||
self.assertTrue(False)
|
||||
|
||||
|
||||
# CI Converage cannot record stub in subprocess,
|
||||
# HACK a _worker_loop in main process call here
|
||||
class TestDataLoaderWorkerLoop(unittest.TestCase):
|
||||
def run_without_worker_done(self, use_shared_memory=True):
|
||||
try:
|
||||
place = fluid.cpu_places()[0]
|
||||
with fluid.dygraph.guard(place):
|
||||
dataset = RandomDataset(800)
|
||||
|
||||
# test init_fn
|
||||
def _init_fn(worker_id):
|
||||
pass
|
||||
|
||||
# test collate_fn
|
||||
def _collate_fn(sample_list):
|
||||
return [
|
||||
np.stack(
|
||||
s, axis=0) for s in list(zip(*sample_list))
|
||||
]
|
||||
|
||||
loader = DataLoader(
|
||||
dataset,
|
||||
num_workers=1,
|
||||
places=place,
|
||||
use_shared_memory=use_shared_memory)
|
||||
assert loader.num_workers > 0, \
|
||||
"go to AssertionError and pass in Mac and Windows"
|
||||
loader = iter(loader)
|
||||
print("loader length", len(loader))
|
||||
indices_queue = multiprocessing.Queue()
|
||||
for i in range(10):
|
||||
indices_queue.put([i, i + 10])
|
||||
indices_queue.put(None)
|
||||
loader._worker_loop(
|
||||
loader._dataset, indices_queue, loader._data_queue,
|
||||
loader._workers_done_event, _collate_fn, _init_fn, 0)
|
||||
self.assertTrue(False)
|
||||
except AssertionError:
|
||||
pass
|
||||
except Exception:
|
||||
self.assertTrue(False)
|
||||
|
||||
def run_with_worker_done(self, use_shared_memory=True):
|
||||
try:
|
||||
place = fluid.cpu_places()[0]
|
||||
with fluid.dygraph.guard(place):
|
||||
dataset = RandomDataset(800)
|
||||
|
||||
# test init_fn
|
||||
def _init_fn(worker_id):
|
||||
pass
|
||||
|
||||
# test collate_fn
|
||||
def _collate_fn(sample_list):
|
||||
return [
|
||||
np.stack(
|
||||
s, axis=0) for s in list(zip(*sample_list))
|
||||
]
|
||||
|
||||
loader = DataLoader(
|
||||
dataset,
|
||||
num_workers=1,
|
||||
places=place,
|
||||
use_shared_memory=use_shared_memory)
|
||||
assert loader.num_workers > 0, \
|
||||
"go to AssertionError and pass in Mac and Windows"
|
||||
loader = iter(loader)
|
||||
print("loader length", len(loader))
|
||||
indices_queue = multiprocessing.Queue()
|
||||
for i in range(10):
|
||||
indices_queue.put([i, i + 10])
|
||||
indices_queue.put(None)
|
||||
loader._workers_done_event.set()
|
||||
loader._worker_loop(
|
||||
loader._dataset, indices_queue, loader._data_queue,
|
||||
loader._workers_done_event, _collate_fn, _init_fn, 0)
|
||||
self.assertTrue(True)
|
||||
except AssertionError:
|
||||
pass
|
||||
except Exception:
|
||||
self.assertTrue(False)
|
||||
|
||||
def test_main(self):
|
||||
for use_shared_memory in [True, False]:
|
||||
self.run_without_worker_done(use_shared_memory)
|
||||
self.run_with_worker_done(use_shared_memory)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue