# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle
import paddle.fluid as fluid
import numpy as np
import six
import os
import unittest
from simple_nets import simple_fc_net_with_inputs
BATCH_SIZE = 32
BATCH_NUM = 10
EPOCH_NUM = 4
IMAGE_SHAPE = [2, 3]
LABEL_SHAPE = [1]
ALL_WRITTEN_FILES = set()
def get_place_string(p):
if isinstance(p, (fluid.CPUPlace or fluid.CUDAPlace)):
tmp = fluid.core.Place()
tmp.set_place(p)
p = tmp
if p._type() == fluid.CPUPlace()._type():
return 'CPUPlace()'
else:
return 'CUDAPlace()'
def remove_all_written_files():
for filename in ALL_WRITTEN_FILES:
os.remove(filename)
def write_reader_data_to_file(filename, reader):
ALL_WRITTEN_FILES.add(filename)
with open(filename, 'w') as fid:
for instance_list in reader():
for i, instance in enumerate(instance_list):
instance = np.reshape(instance, [instance.size, ])
fid.write(str(instance.size) + ' ')
fid.write(' '.join(map(str, instance)))
fid.write(' ')
fid.write('\n')
def fake_reader(batch_size=BATCH_SIZE, batch_num=BATCH_NUM):
def __reader__():
iteration = BATCH_SIZE * BATCH_NUM
iteration = int(iteration + BATCH_SIZE / 2)
for _ in six.moves.range(iteration):
image = np.random.random(size=IMAGE_SHAPE).astype('float32')
label = np.random.random_integers(
size=LABEL_SHAPE, low=0, high=9).astype('int64')
yield image, label
return __reader__
class DatasetLoaderTestBase(unittest.TestCase):
def setUp(self):
self.dataset_name = "QueueDataset"
self.drop_last = False
def tearDown(self):
return
remove_all_written_files()
def build_network(self):
main_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(main_prog, startup_prog):
image = fluid.layers.data(
name='image', shape=IMAGE_SHAPE, dtype='float32')
label = fluid.layers.data(
name='label', shape=LABEL_SHAPE, dtype='int64')
simple_fc_net_with_inputs(image, label)
return main_prog, startup_prog, [image, label]
def check_batch_number(self, place, randomize_batch_num=False):
main_prog, startup_prog, feeds = self.build_network()
if self.dataset_name == "QueueDataset":
dataset = paddle.distributed.QueueDataset()
else:
dataset = paddle.distributed.InMemoryDataset()
dataset._set_batch_size(BATCH_SIZE)
if isinstance(place, fluid.CPUPlace):
file_num = 10
os.environ['CPU_NUM'] = str(file_num)
places = fluid.cpu_places()
use_cuda = False
else:
file_num = fluid.core.get_cuda_device_count()
places = fluid.cuda_places()
use_cuda = True
filelist = []
if file_num > 1 and randomize_batch_num:
random_delta_batch_size = np.random.random_integers(
low=-BATCH_NUM / 2, high=BATCH_NUM / 2, size=[file_num])
random_delta_batch_size[-1] = -int(
np.sum(random_delta_batch_size[0:-1]))
else:
random_delta_batch_size = np.zeros(shape=[file_num])
for i in six.moves.range(file_num):
filename = 'dataset_test_{}.txt'.format(i)
filelist.append(filename)
write_reader_data_to_file(
filename,
fake_reader(batch_num=BATCH_NUM + random_delta_batch_size[i]))
dataset.set_filelist(filelist)
dataset._set_use_var(feeds)
dataset._set_pipe_command("cat")
if self.dataset_name == 'InMemoryDataset':
dataset.load_into_memory()
dataloader = fluid.io.DataLoader.from_dataset(
dataset=dataset, places=places, drop_last=self.drop_last)
prog = fluid.CompiledProgram(main_prog).with_data_parallel()
exe = fluid.Executor(place)
exe.run(startup_prog)
for _ in six.moves.range(EPOCH_NUM):
has_complete_batch = False
for batch_id, data in enumerate(dataloader):
self.assertEquals(len(places), len(data))
for idx, data_on_each_device in enumerate(data):
image = data_on_each_device["image"]
label = data_on_each_device["label"]
if self.drop_last:
batch_size = BATCH_SIZE
else:
if batch_id == BATCH_NUM:
batch_size = BATCH_SIZE / 2
else:
batch_size = BATCH_SIZE
self.assertEquals(image.shape()[1:], IMAGE_SHAPE)
self.assertTrue(
image._place()._equals(places[idx]),
msg=get_place_string(image._place()) + ' vs ' +
get_place_string(places[idx]))
if self.drop_last:
self.assertEquals(image.shape()[0], BATCH_SIZE)
else:
self.assertTrue(image.shape()[0] == BATCH_SIZE or
image.shape()[0] == BATCH_SIZE / 2)
self.assertEquals(label.shape()[1:], LABEL_SHAPE)
self.assertTrue(label._place()._equals(places[idx]))
if self.drop_last:
self.assertEquals(label.shape()[0], BATCH_SIZE)
else:
self.assertTrue(label.shape()[0] == BATCH_SIZE or
label.shape()[0] == BATCH_SIZE / 2)
self.assertEquals(image.shape()[0], label.shape()[0])
if image.shape()[0] == BATCH_SIZE:
has_complete_batch = True
exe.run(prog, feed=data)
self.assertTrue(has_complete_batch)
def get_all_places(self):
p = [fluid.CPUPlace()]
if fluid.is_compiled_with_cuda():
p.append(fluid.CUDAPlace(0))
return p
def test_batch_number_with_same_length_files(self):
for p in self.get_all_places():
with fluid.scope_guard(fluid.Scope()):
self.check_batch_number(place=p, randomize_batch_num=False)
def test_batch_number_with_different_length_files(self):
for p in self.get_all_places():
with fluid.scope_guard(fluid.Scope()):
self.check_batch_number(place=p, randomize_batch_num=True)
class QueueDatasetTestWithoutDropLast(DatasetLoaderTestBase):
def setUp(self):
self.dataset_name = "QueueDataset"
self.drop_last = True
class InMemoryDatasetTestWithoutDropLast(DatasetLoaderTestBase):
def setUp(self):
self.dataset_name = "InMemoryDataset"
self.drop_last = False
class InMemoryDatasetTestWithDropLast(DatasetLoaderTestBase):
def setUp(self):
self.dataset_name = "InMemoryDataset"
self.drop_last = True
if __name__ == '__main__':
unittest.main()