From 62f51fce00cab139c50480a96ced51500e97243f Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Wed, 15 Feb 2017 17:38:37 -0800 Subject: [PATCH 01/12] add data provider design doc proposal --- doc/design/reader/README.md | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 doc/design/reader/README.md diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md new file mode 100644 index 0000000000..fa37ad7b29 --- /dev/null +++ b/doc/design/reader/README.md @@ -0,0 +1,79 @@ +# Python Data Provider Design Doc + +Data provider provides data for training. It will be passed into `paddle.train` as a parameter. + +## Data Provider Interface + +Data provider is a function with no parameter that creates a iterable (anything can be used in `for x in iterable`): + +``` +iterable = data_provider() +``` + +Element produced for the iterable should be a **single** entry of data, in format `[column_0_item, column_1_item, ...]`. Each element of the list needs to be supported data type (e.g., numpy 1d array of float32, list of int). + +For example, `column_0_item` could be image pixels of format numpy 1d array of float32, and `column_1_item` could be image label of format single int value: + +``` +for single_entry in iterable: + pixel = entry[0] + label = entry[1] +``` + +## Usage + +data provider, mapping from data provider column to data layer, batch size and number of total pass will be passed into `paddle.train`: + +```python +# two data layer is created: +image_layer = paddle.layer.data("image", ...) +label_layer = paddle.layer.data("label", ...) + +# ... + +paddle.train(paddle.data.mnist, ["image", "label"], 128, 10, ...) +``` +## Q & A + +### Why return only a single entry, but not a mini batch? + +If return a mini batch, data provider need to take care of batch size. But batch size is a concept for training, it makes more sense for user to specify batch size as a parameter for `train`. + +Concretely, always return a single entry make reusing existing data providers much easier (e.g., if existing data provider return not a single entry but 3 entries, training code will be more complex because it need to handle cases like batch size 2). + +### How to create custom data provider + +```python +def image_provider(path): + f = open(path) + images = numpy.fromfile( + f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)).astype('float32') + images = images / 255.0 * 2.0 - 1.0 + labels = numpy.fromfile(l, 'ubyte', count=n).astype("int") + for i in xrange(n): + yield [images[i, :], labels[i]] # a single entry of data is created each time + f.close() + +# use python lambda to change image_provier into a function with no parameters. +provider = lambda : image_provier("/path/to/data/") +paddle.train(provider, ["image", "label"], ...) +``` + +### How is `paddle.train` implemented + +An example implementation of paddle.train could be: + +```python +def make_minibatch_generator(reader, minibatch_size): + buf = [reader.next() for x in xrange(minibatch_size)] + while len(buf) > 0: + yield buf + buf = [reader.next() for x in xrange(minibatch_size)] + +def train(provider, mapping, batch_size, total_pass): + for pass_idx in range(total_pass): + for mini_batch in make_minibatch_generator(provider(pass_idx)): # this loop will never end in online learning. + do_forward_backward(mini_batch, mapping) +``` + +This is just an example implementation, more complicated logic like data prefetch could be implemented. From a9902a300bd35a1a497414ea3475cf79c7058f5c Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Thu, 16 Feb 2017 17:34:10 -0800 Subject: [PATCH 02/12] fix according to discussion --- doc/design/reader/README.md | 114 ++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 26 deletions(-) diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md index fa37ad7b29..7fa62350c4 100644 --- a/doc/design/reader/README.md +++ b/doc/design/reader/README.md @@ -1,28 +1,86 @@ -# Python Data Provider Design Doc +# Python Data Reader Design Doc -Data provider provides data for training. It will be passed into `paddle.train` as a parameter. +Paddle reads data from data reader during training. It will be passed into `paddle.train` as a parameter. -## Data Provider Interface +## Data Reader Interface -Data provider is a function with no parameter that creates a iterable (anything can be used in `for x in iterable`): +Data reader is a function with no parameter that creates a iterable (anything can be used in `for x in iterable`): ``` -iterable = data_provider() +iterable = data_reader() ``` -Element produced for the iterable should be a **single** entry of data, in format `[column_0_item, column_1_item, ...]`. Each element of the list needs to be supported data type (e.g., numpy 1d array of float32, list of int). +Element produced for the iterable should be a **single** entry of data, **not** a mini batch. That entry of data could be a single item, or a tuple of items. Item should be of [supported type](http://www.paddlepaddle.org/doc/ui/data_provider/pydataprovider2.html?highlight=dense_vector#input-types) (e.g., numpy 1d array of float32, int, list of int) -For example, `column_0_item` could be image pixels of format numpy 1d array of float32, and `column_1_item` could be image label of format single int value: +An example implementation for single item data reader: +```python +def data_reader_fake_image(): + while True: + yield numpy.random.uniform(-1, 1, size=20*20) +``` + +An example implementation for multiple item data reader: +```python +def data_reader_fake_image_and_label(): + while True: + yield numpy.random.uniform(-1, 1, size=20*20), False ``` -for single_entry in iterable: - pixel = entry[0] - label = entry[1] + +## Data Reader Decorators + +Data Reader Decorators takes a single or multiple data reader, returns a new data reader. It is similar to a [python decorator](https://wiki.python.org/moin/PythonDecorators), but it does not use `@` syntax. + +Since we have a strict interface for data readers (no parameter, return a single data item). Data reader can be used flexiable via data reader decorators. Following a few examples: + +### Prefetch Data + +Since reading data may take time and training can not proceed without data. It is generally a good idea to prefetch data. + +Use `paddle.reader.buffered` to prefetch data: + +```python +buffered_reader = paddle.reader.buffered(paddle.dataset.mnist, 100) +``` + +`buffered_reader` will try to buffer (prefetch) `100` data entries. + +### Compose Multiple Data Readers + +For example, we want to use a source of real images (reusing mnist dataset), and a source of fake images as input for [Generative Adversarial Networks](https://arxiv.org/abs/1406.2661). + +We can do: + +```python +def data_reader_fake_image(): + while True: + yield numpy.random.uniform(-1, 1, size=20*20) + +def data_reader_bool(t): + while True: + yield t + +true_reader = lambda : data_reader_bool(True) +false_reader = lambda : data_reader_bool(False) + +reader = paddle.reader.combine(paddle.dataset.mnist, data_reader_fake_image, true_reader, false_reader) +# skipped 1 because paddle.dataset.mnist produces two items per data entry. +# We don't care second item at this time. +paddle.train(reader, {"true_image":0, "fake_image": 2, "true_label": 3, "false_label": 4}, ...) +``` + +### Shuffle + +Given shuffle buffer size `n`, `paddle.reader.shuffle` will return a data reader decorater that buffers `n` data entries and shuffle them before a data entry is read. + +Example: +```python +reader = paddle.reader.shuffle(paddle.dataset.mnist, 512) ``` ## Usage -data provider, mapping from data provider column to data layer, batch size and number of total pass will be passed into `paddle.train`: +data reader, mapping from item(s) read to data layer, batch size and number of total pass will be passed into `paddle.train`: ```python # two data layer is created: @@ -31,32 +89,38 @@ label_layer = paddle.layer.data("label", ...) # ... -paddle.train(paddle.data.mnist, ["image", "label"], 128, 10, ...) +paddle.train(paddle.dataset.mnist, {"image":0, "label":1}, 128, 10, ...) ``` + ## Q & A ### Why return only a single entry, but not a mini batch? -If return a mini batch, data provider need to take care of batch size. But batch size is a concept for training, it makes more sense for user to specify batch size as a parameter for `train`. +If return a mini batch, data reader need to take care of batch size. But batch size is a concept for training, it makes more sense for user to specify batch size as a parameter for `train`. -Concretely, always return a single entry make reusing existing data providers much easier (e.g., if existing data provider return not a single entry but 3 entries, training code will be more complex because it need to handle cases like batch size 2). +Practically, always return a single entry make reusing existing data reader much easier (e.g., if existing data reader return not a single entry but 3 entries, training code will be more complex because it need to handle cases like batch size 2). -### How to create custom data provider +### Why use a dictionary but not a list to provide mapping? + +We decided to use dictionary (`{"image":0, "label":1}`) instead of list (`["image", "label"]`) is because that user can easily resue item (e.g., using `{"image_a":0, "image_b":0, "label":1}`) or skip item (e.g., using `{"image_a":0, "label":2}`). + +### How to create custom data reader ```python -def image_provider(path): - f = open(path) +def image_reader(image_path, label_path): + f = open(image_path) + l = open(label_path) images = numpy.fromfile( f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)).astype('float32') images = images / 255.0 * 2.0 - 1.0 labels = numpy.fromfile(l, 'ubyte', count=n).astype("int") for i in xrange(n): - yield [images[i, :], labels[i]] # a single entry of data is created each time + yield images[i, :], labels[i] # a single entry of data is created each time f.close() -# use python lambda to change image_provier into a function with no parameters. -provider = lambda : image_provier("/path/to/data/") -paddle.train(provider, ["image", "label"], ...) +# use python lambda to change image_reader into a function with no parameters. +reader = lambda : image_reader("/path/to/image_file", "/path/to/label_file") +paddle.train(reader, {"image":0, "label":1}, ...) ``` ### How is `paddle.train` implemented @@ -64,16 +128,14 @@ paddle.train(provider, ["image", "label"], ...) An example implementation of paddle.train could be: ```python -def make_minibatch_generator(reader, minibatch_size): +def minibatch_decorater(reader, minibatch_size): buf = [reader.next() for x in xrange(minibatch_size)] while len(buf) > 0: yield buf buf = [reader.next() for x in xrange(minibatch_size)] -def train(provider, mapping, batch_size, total_pass): +def train(reader, mapping, batch_size, total_pass): for pass_idx in range(total_pass): - for mini_batch in make_minibatch_generator(provider(pass_idx)): # this loop will never end in online learning. + for mini_batch in minibatch_decorater(reader()): # this loop will never end in online learning. do_forward_backward(mini_batch, mapping) ``` - -This is just an example implementation, more complicated logic like data prefetch could be implemented. From 9048311c4afc50ee8277a3f897648eab2bb7db0e Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Thu, 16 Feb 2017 17:45:48 -0800 Subject: [PATCH 03/12] fix gramma, typo, code and rearrange paragraph --- doc/design/reader/README.md | 51 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md index 7fa62350c4..be337a800f 100644 --- a/doc/design/reader/README.md +++ b/doc/design/reader/README.md @@ -27,11 +27,25 @@ def data_reader_fake_image_and_label(): yield numpy.random.uniform(-1, 1, size=20*20), False ``` +## Usage + +data reader, mapping from item(s) read to data layer, batch size and number of total pass will be passed into `paddle.train`: + +```python +# two data layer is created: +image_layer = paddle.layer.data("image", ...) +label_layer = paddle.layer.data("label", ...) + +# ... + +paddle.train(paddle.dataset.mnist, {"image":0, "label":1}, 128, 10, ...) +``` + ## Data Reader Decorators Data Reader Decorators takes a single or multiple data reader, returns a new data reader. It is similar to a [python decorator](https://wiki.python.org/moin/PythonDecorators), but it does not use `@` syntax. -Since we have a strict interface for data readers (no parameter, return a single data item). Data reader can be used flexiable via data reader decorators. Following a few examples: +Since we have a strict interface for data readers (no parameter, return a single data item). Data reader can be used flexiable via data reader decorators. Following are a few examples: ### Prefetch Data @@ -64,39 +78,25 @@ true_reader = lambda : data_reader_bool(True) false_reader = lambda : data_reader_bool(False) reader = paddle.reader.combine(paddle.dataset.mnist, data_reader_fake_image, true_reader, false_reader) -# skipped 1 because paddle.dataset.mnist produces two items per data entry. -# We don't care second item at this time. +# Skipped 1 because paddle.dataset.mnist produces two items per data entry. +# And we don't care second item at this time. paddle.train(reader, {"true_image":0, "fake_image": 2, "true_label": 3, "false_label": 4}, ...) ``` ### Shuffle -Given shuffle buffer size `n`, `paddle.reader.shuffle` will return a data reader decorater that buffers `n` data entries and shuffle them before a data entry is read. +Given shuffle buffer size `n`, `paddle.reader.shuffle` will return a data reader that buffers `n` data entries and shuffle them before a data entry is read. Example: ```python reader = paddle.reader.shuffle(paddle.dataset.mnist, 512) ``` -## Usage - -data reader, mapping from item(s) read to data layer, batch size and number of total pass will be passed into `paddle.train`: - -```python -# two data layer is created: -image_layer = paddle.layer.data("image", ...) -label_layer = paddle.layer.data("label", ...) - -# ... - -paddle.train(paddle.dataset.mnist, {"image":0, "label":1}, 128, 10, ...) -``` - ## Q & A ### Why return only a single entry, but not a mini batch? -If return a mini batch, data reader need to take care of batch size. But batch size is a concept for training, it makes more sense for user to specify batch size as a parameter for `train`. +If a mini batch is returned, data reader need to take care of batch size. But batch size is a concept for training, it makes more sense for user to specify batch size as a parameter for `train`. Practically, always return a single entry make reusing existing data reader much easier (e.g., if existing data reader return not a single entry but 3 entries, training code will be more complex because it need to handle cases like batch size 2). @@ -129,13 +129,16 @@ An example implementation of paddle.train could be: ```python def minibatch_decorater(reader, minibatch_size): - buf = [reader.next() for x in xrange(minibatch_size)] - while len(buf) > 0: - yield buf - buf = [reader.next() for x in xrange(minibatch_size)] + def ret(): + r = reader() + buf = [r.next() for x in xrange(minibatch_size)] + while len(buf) > 0: + yield buf + buf = [r.next() for x in xrange(minibatch_size)] + return ret def train(reader, mapping, batch_size, total_pass): for pass_idx in range(total_pass): - for mini_batch in minibatch_decorater(reader()): # this loop will never end in online learning. + for mini_batch in minibatch_decorater(reader): # this loop will never end in online learning. do_forward_backward(mini_batch, mapping) ``` From 7f19ca4fbe61f4a454eb55b018f222305f87ce7a Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Thu, 16 Feb 2017 17:53:46 -0800 Subject: [PATCH 04/12] fix capitalization --- doc/design/reader/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md index be337a800f..8f7abf12f7 100644 --- a/doc/design/reader/README.md +++ b/doc/design/reader/README.md @@ -43,7 +43,7 @@ paddle.train(paddle.dataset.mnist, {"image":0, "label":1}, 128, 10, ...) ## Data Reader Decorators -Data Reader Decorators takes a single or multiple data reader, returns a new data reader. It is similar to a [python decorator](https://wiki.python.org/moin/PythonDecorators), but it does not use `@` syntax. +Data reader decorators takes a single or multiple data reader, returns a new data reader. It is similar to a [python decorator](https://wiki.python.org/moin/PythonDecorators), but it does not use `@` syntax. Since we have a strict interface for data readers (no parameter, return a single data item). Data reader can be used flexiable via data reader decorators. Following are a few examples: From 695b5a7fcd42e4e5678fdb4288cc8dd23240aac4 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 17 Feb 2017 18:09:55 +0800 Subject: [PATCH 05/12] change topology to layer --- demo/mnist/api_train_v2.py | 5 ++--- python/paddle/v2/parameters.py | 26 +++++++++++++------------- python/paddle/v2/trainer.py | 9 ++++++--- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index 59486ed1b3..b5cc74ce67 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -25,8 +25,7 @@ def main(): act=paddle.activation.Softmax()) cost = paddle.layer.classification_cost(input=inference, label=label) - topology = paddle.layer.parse_network(cost) - parameters = paddle.parameters.create(topology) + parameters = paddle.parameters.create(cost) for param_name in parameters.keys(): array = parameters.get(param_name) array[:] = numpy.random.uniform(low=-1.0, high=1.0, size=array.shape) @@ -46,7 +45,7 @@ def main(): trainer = paddle.trainer.SGD(update_equation=adam_optimizer) trainer.train(train_data_reader=train_reader, - topology=topology, + topology=cost, parameters=parameters, event_handler=event_handler, batch_size=32, # batch size should be refactor in Data reader diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py index e5b7dabcb8..ea504d5104 100644 --- a/python/paddle/v2/parameters.py +++ b/python/paddle/v2/parameters.py @@ -1,27 +1,27 @@ import numpy as np - -from paddle.proto.ModelConfig_pb2 import ModelConfig -from paddle.proto.ParameterConfig_pb2 import ParameterConfig +from . import layer as v2_layer import py_paddle.swig_paddle as api +from paddle.proto.ParameterConfig_pb2 import ParameterConfig __all__ = ['Parameters', 'create'] -def create(*topologies): +def create(*layers): """ - Create parameter pool by topologies. + Create parameter pool by layers. In paddle, layer can be represent a + model config. - :param topologies: + :param layers: :return: """ - pool = Parameters() - for topo in topologies: - if not isinstance(topo, ModelConfig): + for layer in layers: + if not isinstance(layer, v2_layer.Layer): raise ValueError( - 'create must pass a topologies which type is ModelConfig') - - for param in topo.parameters: - pool.__append_config__(param) + 'create must pass a topologies which type is paddle.layer.Layer') + model_config = v2_layer.parse_network(*layers) + pool = Parameters() + for param in model_config.parameters: + pool.__append_config__(param) return pool diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index 9ba13dc5c8..4365bd41e7 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -1,12 +1,13 @@ import collections import py_paddle.swig_paddle as api +from paddle.proto.ModelConfig_pb2 import ModelConfig from py_paddle import DataProviderConverter -from paddle.proto.ModelConfig_pb2 import ModelConfig +from . import event as v2_event +from . import layer as v2_layer from . import optimizer as v2_optimizer from . import parameters as v2_parameters -from . import event as v2_event __all__ = ['ITrainer', 'SGD'] @@ -73,7 +74,7 @@ class SGD(ITrainer): Training method. Will train num_passes of input data. :param train_data_reader: - :param topology: Network Topology, a protobuf ModelConfig message. + :param topology: Network Topology, use one or more Layers to represent it. :param parameters: The parameter pools. :param num_passes: The total train passes. :param test_data_reader: @@ -87,6 +88,8 @@ class SGD(ITrainer): if event_handler is None: event_handler = default_event_handler + topology = v2_layer.parse_network(topology) + __check_train_args__(**locals()) gm = api.GradientMachine.createFromConfigProto( From 701d02459d36ee254e20f5a934205aa0530478e8 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Fri, 17 Feb 2017 19:58:16 +0800 Subject: [PATCH 06/12] Fix bug in processing input with static parameter in seqconcat layer --- paddle/gserver/layers/SequenceConcatLayer.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/paddle/gserver/layers/SequenceConcatLayer.cpp b/paddle/gserver/layers/SequenceConcatLayer.cpp index b4677687a6..599706eb41 100644 --- a/paddle/gserver/layers/SequenceConcatLayer.cpp +++ b/paddle/gserver/layers/SequenceConcatLayer.cpp @@ -168,13 +168,17 @@ void SequenceConcatLayer::backward(const UpdateCallback& callback) { size_t rightNumIns = 0; for (size_t seqId = 0; seqId < numSequences1; ++seqId) { leftNumIns = starts1[seqId + 1] - starts1[seqId]; - inputGrad1->subMatrix(starts1[seqId], leftNumIns) - ->add(*(outputGrad->subMatrix(offset, leftNumIns))); + if (inputGrad1) { + inputGrad1->subMatrix(starts1[seqId], leftNumIns) + ->add(*(outputGrad->subMatrix(offset, leftNumIns))); + } offset += leftNumIns; rightNumIns = starts2[seqId + 1] - starts2[seqId]; - inputGrad2->subMatrix(starts2[seqId], rightNumIns) - ->add(*(outputGrad->subMatrix(offset, rightNumIns))); + if (inputGrad2) { + inputGrad2->subMatrix(starts2[seqId], rightNumIns) + ->add(*(outputGrad->subMatrix(offset, rightNumIns))); + } offset += rightNumIns; } } From f846362927fa9ba6baf0c54db5051e2616af0a39 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Fri, 17 Feb 2017 10:06:59 -0800 Subject: [PATCH 07/12] create buffered data reader decorator and tests --- python/CMakeLists.txt | 1 + python/paddle/reader/__init__.py | 15 +++++ python/paddle/reader/decorator.py | 60 ++++++++++++++++++++ python/paddle/reader/tests/CMakeLists.txt | 4 ++ python/paddle/reader/tests/decorator_test.py | 50 ++++++++++++++++ 5 files changed, 130 insertions(+) create mode 100644 python/paddle/reader/__init__.py create mode 100644 python/paddle/reader/decorator.py create mode 100644 python/paddle/reader/tests/CMakeLists.txt create mode 100644 python/paddle/reader/tests/decorator_test.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index ee7a5bff84..357637e203 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -24,6 +24,7 @@ add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) add_subdirectory(paddle/trainer_config_helpers/tests) +add_subdirectory(paddle/reader/tests) install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/ DESTINATION opt/paddle/share/wheels diff --git a/python/paddle/reader/__init__.py b/python/paddle/reader/__init__.py new file mode 100644 index 0000000000..28a69d8370 --- /dev/null +++ b/python/paddle/reader/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from decorator import * diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py new file mode 100644 index 0000000000..f0ddb0ff81 --- /dev/null +++ b/python/paddle/reader/decorator.py @@ -0,0 +1,60 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['buffered'] + +from Queue import Queue +from threading import Thread + + +def buffered(reader, size): + """Creates a buffered data reader. + + The buffered data reader will read and save data entries into a buffer. + Reading from the buffered data reader will proceed as long as the buffer + is not empty. + + Args: + reader: the data reader to read from. + size: max buffer size. + + Returns: + The buffered data reader. + """ + + class EndSignal(): + pass + + end = EndSignal() + + def read_worker(r, q): + for d in r: + q.put(d) + q.put(end) + + def create_reader(): + r = reader() + q = Queue(maxsize=size) + t = Thread( + target=read_worker, args=( + r, + q, )) + t.daemon = True + t.start() + e = q.get() + while e != end: + yield e + e = q.get() + + return create_reader diff --git a/python/paddle/reader/tests/CMakeLists.txt b/python/paddle/reader/tests/CMakeLists.txt new file mode 100644 index 0000000000..4768d0738b --- /dev/null +++ b/python/paddle/reader/tests/CMakeLists.txt @@ -0,0 +1,4 @@ +add_test(NAME reader_decorator_test + COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ + ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/reader/decorator_test.py + WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) diff --git a/python/paddle/reader/tests/decorator_test.py b/python/paddle/reader/tests/decorator_test.py new file mode 100644 index 0000000000..879d1d9c1d --- /dev/null +++ b/python/paddle/reader/tests/decorator_test.py @@ -0,0 +1,50 @@ +# Copyright PaddlePaddle contributors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import paddle.reader +import time + + +def reader_10(dur): + for i in range(10): + time.sleep(dur) + yield i + + +class TestBuffered(unittest.TestCase): + def test_read(self): + for size in range(20): + b = paddle.reader.buffered(lambda: reader_10(0), size) + c = 0 + for i in b(): + self.assertEqual(i, c) + c += 1 + self.assertEqual(c, 10) + + def test_buffering(self): + # read have 30ms delay. + b = paddle.reader.buffered(lambda: reader_10(0.03), 10) + last_time = time.time() + for idx, i in enumerate(b()): + elapsed_time = time.time() - last_time + if i == 0: + time.sleep(0.3) + else: + # read time should be short, meaning already buffered. + self.assertLess(elapsed_time, 0.01) + last_time = time.time() + + +if __name__ == '__main__': + unittest.main() From 7c1d10e6cbfd4bfd4f9ac1d0c49418755fab541e Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Fri, 17 Feb 2017 10:46:26 -0800 Subject: [PATCH 08/12] travis ci: cache brew directory Fixes #1365 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5d82d9729b..4fb2ca9387 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ cache: - $HOME/third_party - $HOME/.ccache - $HOME/.cache/pip + - $HOME/Library/Caches/Homebrew sudo: required dist: trusty os: From 7b50594f6d61b80631b8700d0ba158bb7ef5363a Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Sat, 18 Feb 2017 10:23:22 -0800 Subject: [PATCH 09/12] fix test path --- python/paddle/reader/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/reader/tests/CMakeLists.txt b/python/paddle/reader/tests/CMakeLists.txt index 4768d0738b..502c897d89 100644 --- a/python/paddle/reader/tests/CMakeLists.txt +++ b/python/paddle/reader/tests/CMakeLists.txt @@ -1,4 +1,4 @@ add_test(NAME reader_decorator_test COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/reader/decorator_test.py + ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/reader/tests/decorator_test.py WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) From b2a969b7b3227a089ac5b482dfd5f9b340a3f483 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Sat, 18 Feb 2017 13:27:25 -0800 Subject: [PATCH 10/12] fix according to comment --- python/paddle/reader/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/paddle/reader/__init__.py b/python/paddle/reader/__init__.py index 28a69d8370..493b410e82 100644 --- a/python/paddle/reader/__init__.py +++ b/python/paddle/reader/__init__.py @@ -12,4 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# It would be too lengthy to require our users to prefix decorators with `decorator`. +# For example, we want the following line +# +# r = paddle.reader.decorator.bufferd(paddle.reader.creator.text("hello.txt")) +# +# to be a shorter version: +# +# r = paddle.reader.buffered(paddle.reader.creator.text("hello.txt")) from decorator import * From 948218dab96369b24027c801b61cad699db7b4c6 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 20 Feb 2017 14:08:35 +0800 Subject: [PATCH 11/12] Unify PrintLogic in PrintLayer/ValuePrinter. --- paddle/gserver/evaluators/Evaluator.cpp | 13 ++----------- paddle/gserver/layers/PrintLayer.cpp | 24 ++++++++---------------- paddle/parameter/Argument.cpp | 12 ++++++++++++ paddle/parameter/Argument.h | 3 +++ 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 1a43d3bd99..1b06519713 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -888,19 +888,10 @@ Evaluator* Evaluator::create(const EvaluatorConfig& config) { */ class ValuePrinter : public Evaluator { public: - ValuePrinter() {} - virtual void eval(const NeuralNetwork& nn) { for (const std::string& name : config_.input_layers()) { - auto& argu = nn.getLayer(name)->getOutput(); - std::unordered_map out; - argu.getValueString(&out); - for (auto field : {"value", "id", "sequence pos", "sub-sequence pos"}) { - auto it = out.find(field); - if (it != out.end()) { - LOG(INFO) << "layer=" << name << " " << field << ":\n" << it->second; - } - } + nn.getLayer(name)->getOutput().printValueString(LOG(INFO), + "layer=" + name + " "); } } diff --git a/paddle/gserver/layers/PrintLayer.cpp b/paddle/gserver/layers/PrintLayer.cpp index f1f3dd412c..de198af111 100644 --- a/paddle/gserver/layers/PrintLayer.cpp +++ b/paddle/gserver/layers/PrintLayer.cpp @@ -19,25 +19,17 @@ namespace paddle { class PrintLayer : public Layer { public: explicit PrintLayer(const LayerConfig& config) : Layer(config) {} - void forward(PassType passType) override; - void backward(const UpdateCallback& callback) override {} -}; -void PrintLayer::forward(PassType passType) { - Layer::forward(passType); - for (size_t i = 0; i != inputLayers_.size(); ++i) { - auto& argu = getInput(i); - const std::string& name = inputLayers_[i]->getName(); - std::unordered_map out; - argu.getValueString(&out); - for (auto field : {"value", "id", "sequence pos", "sub-sequence pos"}) { - auto it = out.find(field); - if (it != out.end()) { - LOG(INFO) << "layer=" << name << " " << field << ":\n" << it->second; - } + void forward(PassType passType) override { + Layer::forward(passType); + for (size_t i = 0; i != inputLayers_.size(); ++i) { + getInput(i).printValueString(LOG(INFO), + "layer=" + inputLayers_[i]->getName() + " "); } } -} + + void backward(const UpdateCallback& callback) override {} +}; REGISTER_LAYER(print, PrintLayer); diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index e9de0f6698..7a343cca33 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -628,6 +628,18 @@ void Argument::getValueString( } } +void Argument::printValueString(std::ostream& stream, + const std::string& prefix) const { + std::unordered_map out; + getValueString(&out); + for (auto field : {"value", "id", "sequence pos", "sub-sequence pos"}) { + auto it = out.find(field); + if (it != out.end()) { + stream << prefix << field << ":\n" << it->second; + } + } +} + void Argument::subArgFrom(const Argument& input, size_t offset, size_t height, diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index c751dbb855..e2a8500c3f 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -305,6 +305,9 @@ struct Argument { * @param out [out]: the return values. */ void getValueString(std::unordered_map* out) const; + + void printValueString(std::ostream& stream, + const std::string& prefix = "") const; }; } // namespace paddle From b5b9a9cfb797db957d51b2dc5fc6a7d45d90e46b Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 20 Feb 2017 14:17:18 +0800 Subject: [PATCH 12/12] Add comments --- paddle/parameter/Argument.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index e2a8500c3f..178c068b93 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -306,6 +306,12 @@ struct Argument { */ void getValueString(std::unordered_map* out) const; + /** + * @brief printValueString will print the argument's output in order of + * 'value', 'id', 'sequence pos', 'sub-sequence pos'. + * @param stream: Output stream + * @param prefix: line prefix for printing. + */ void printValueString(std::ostream& stream, const std::string& prefix = "") const; };