optimize pyreader

revert-13637-optimize-opyreader
qiaolongfei 6 years ago
parent 1ab7b55162
commit accf3f7505

@ -12,6 +12,5 @@ endif(NOT WIN32)
if(WITH_INFERENCE) if(WITH_INFERENCE)
# NOTE: please add subdirectory inference at last. # NOTE: please add subdirectory inference at last.
add_subdirectory(inference) add_subdirectory(inference)
endif()
add_subdirectory(train) add_subdirectory(train)
endif()

File diff suppressed because it is too large Load Diff

@ -53,7 +53,16 @@ def simple_fc_net(in_size,
hidden_sizes, hidden_sizes,
batch_size, batch_size,
queue_capacity, queue_capacity,
use_double_buffer=False): use_double_buffer=False,
use_feed_list=True):
if use_feed_list:
data = fluid.layers.data(name="data", dtype='float32', shape=[in_size])
label = fluid.layers.data(name='label', dtype='int64', shape=[1])
reader = fluid.layers.py_reader_by_data(
capacity=queue_capacity,
use_double_buffer=False,
feed_list=[data, label])
else:
reader = fluid.layers.py_reader( reader = fluid.layers.py_reader(
capacity=queue_capacity, capacity=queue_capacity,
shapes=[[-1, in_size], [-1, 1]], shapes=[[-1, in_size], [-1, 1]],
@ -100,6 +109,7 @@ class TestPyReaderUsingExecutor(unittest.TestCase):
if core.is_compiled_with_cuda() else [False]): if core.is_compiled_with_cuda() else [False]):
for use_parallel_executor in [False, True]: for use_parallel_executor in [False, True]:
for use_double_buffer in [False, True]: for use_double_buffer in [False, True]:
for use_feed_list in [False, True]:
print('Test Parameters:'), print('Test Parameters:'),
print({ print({
'use_cuda': use_cuda, 'use_cuda': use_cuda,
@ -107,7 +117,7 @@ class TestPyReaderUsingExecutor(unittest.TestCase):
'use_double_buffer': use_double_buffer 'use_double_buffer': use_double_buffer
}) })
self.main(use_cuda, use_parallel_executor, self.main(use_cuda, use_parallel_executor,
use_double_buffer) use_double_buffer, use_feed_list)
def random_reader(self): def random_reader(self):
def reader(): def reader():
@ -143,12 +153,14 @@ class TestPyReaderUsingExecutor(unittest.TestCase):
def main(self, def main(self,
use_cuda=True, use_cuda=True,
use_parallel_executor=False, use_parallel_executor=False,
use_double_buffer=False): use_double_buffer=False,
use_feed_list=False):
assert not use_cuda or use_cuda and core.is_compiled_with_cuda() assert not use_cuda or use_cuda and core.is_compiled_with_cuda()
self.use_cuda = use_cuda self.use_cuda = use_cuda
self.use_parallel_executor = use_parallel_executor self.use_parallel_executor = use_parallel_executor
self.use_double_buffer = use_double_buffer self.use_double_buffer = use_double_buffer
self.use_feed_list = use_feed_list
startup_program = fluid.Program() startup_program = fluid.Program()
main_program = fluid.Program() main_program = fluid.Program()
@ -160,7 +172,8 @@ class TestPyReaderUsingExecutor(unittest.TestCase):
hidden_sizes=self.hidden_sizes, hidden_sizes=self.hidden_sizes,
batch_size=self.batch_size, batch_size=self.batch_size,
queue_capacity=self.queue_capacity, queue_capacity=self.queue_capacity,
use_double_buffer=self.use_double_buffer) use_double_buffer=self.use_double_buffer,
use_feed_list=self.use_feed_list)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()

Loading…
Cancel
Save