From a219455516f77b735c93621db16d26e8f230f5e7 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 13:45:47 +0800 Subject: [PATCH 01/14] Rise type error because of python. --- python/paddle/v2/trainer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index 023ab5e42d..f65c631f7b 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -164,20 +164,20 @@ def __check_train_args__(train_data_reader, topology, parameters, """ if not callable(train_data_reader) or not isinstance(train_data_reader(), collections.Iterator): - raise ValueError('train_data_reader should be a function, ' - 'which can return a iterator') + raise TypeError('train_data_reader should be a function, ' + 'which can return a iterator') if test_data_reader is not None: if not callable(test_data_reader) or not isinstance( test_data_reader(), collections.Iterator): - raise ValueError('test_data_reader should be a function, which can ' - 'return a iterator') + raise TypeError('test_data_reader should be a function, which can ' + 'return a iterator') if not isinstance(topology, ModelConfig): - raise ValueError('topology should be a model config') + raise TypeError('topology should be a model config') if not isinstance(parameters, v2_parameters.Parameters): - raise ValueError('parameters should be a parameter pool') + raise TypeError('parameters should be a parameter pool') if not callable(event_handler): - raise ValueError('event handler should be a function') + raise TypeError('event handler should be a function') From c26431bacf25a6053999af791ab66d17d3c62c1b Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 14:03:34 +0800 Subject: [PATCH 02/14] Reordered package for feature usage. --- python/CMakeLists.txt | 4 +-- python/paddle/v2/__init__.py | 3 ++- python/paddle/v2/dataset/__init__.py | 3 +++ python/paddle/{ => v2}/reader/__init__.py | 0 python/paddle/{ => v2}/reader/creator.py | 0 python/paddle/{ => v2}/reader/decorator.py | 0 .../{ => v2}/reader/tests/CMakeLists.txt | 0 python/paddle/v2/reader/tests/__init__.py | 0 .../{ => v2}/reader/tests/creator_test.py | 10 ++++--- .../{ => v2}/reader/tests/decorator_test.py | 27 ++++++++++--------- .../reader/tests/test_data_creator.txt | 0 python/paddle/v2/trainer.py | 6 ++--- python/setup.py.in | 4 ++- 13 files changed, 33 insertions(+), 24 deletions(-) rename python/paddle/{ => v2}/reader/__init__.py (100%) rename python/paddle/{ => v2}/reader/creator.py (100%) rename python/paddle/{ => v2}/reader/decorator.py (100%) rename python/paddle/{ => v2}/reader/tests/CMakeLists.txt (100%) create mode 100644 python/paddle/v2/reader/tests/__init__.py rename python/paddle/{ => v2}/reader/tests/creator_test.py (89%) rename python/paddle/{ => v2}/reader/tests/decorator_test.py (81%) rename python/paddle/{ => v2}/reader/tests/test_data_creator.txt (100%) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 71af50a9a4..48e0a1993d 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,7 +4,7 @@ set(OUTPUT_DIR file(GLOB TRAINER_PY_FILES . ./paddle/trainer/*.py) file(GLOB HELPERS_PY_FILES . ./paddle/trainer_config_helpers/*.py) file(GLOB UTILS_PY_FILES . ./paddle/utils/*.py) -file(GLOB V2_PY_FILES . ./paddle/v2/*.py) +file(GLOB_RECURSE V2_PY_FILES ./paddle/v2/ *.py) set(PY_FILES paddle/__init__.py ${TRAINER_PY_FILES} @@ -24,7 +24,7 @@ add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) add_subdirectory(paddle/trainer_config_helpers/tests) -add_subdirectory(paddle/reader/tests) +add_subdirectory(paddle/v2/reader/tests) add_subdirectory(paddle/v2/tests) install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/ diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 49d1983a2a..1122bcb5e4 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -19,12 +19,13 @@ import trainer import event import data_type import data_feeder +from . import dataset import attr import py_paddle.swig_paddle as api __all__ = [ 'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer', - 'event', 'data_type', 'attr', 'data_feeder' + 'event', 'data_type', 'attr', 'data_feeder', 'dataset' ] diff --git a/python/paddle/v2/dataset/__init__.py b/python/paddle/v2/dataset/__init__.py index e69de29bb2..9647e98503 100644 --- a/python/paddle/v2/dataset/__init__.py +++ b/python/paddle/v2/dataset/__init__.py @@ -0,0 +1,3 @@ +import mnist + +__all__ = ['mnist'] diff --git a/python/paddle/reader/__init__.py b/python/paddle/v2/reader/__init__.py similarity index 100% rename from python/paddle/reader/__init__.py rename to python/paddle/v2/reader/__init__.py diff --git a/python/paddle/reader/creator.py b/python/paddle/v2/reader/creator.py similarity index 100% rename from python/paddle/reader/creator.py rename to python/paddle/v2/reader/creator.py diff --git a/python/paddle/reader/decorator.py b/python/paddle/v2/reader/decorator.py similarity index 100% rename from python/paddle/reader/decorator.py rename to python/paddle/v2/reader/decorator.py diff --git a/python/paddle/reader/tests/CMakeLists.txt b/python/paddle/v2/reader/tests/CMakeLists.txt similarity index 100% rename from python/paddle/reader/tests/CMakeLists.txt rename to python/paddle/v2/reader/tests/CMakeLists.txt diff --git a/python/paddle/v2/reader/tests/__init__.py b/python/paddle/v2/reader/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/paddle/reader/tests/creator_test.py b/python/paddle/v2/reader/tests/creator_test.py similarity index 89% rename from python/paddle/reader/tests/creator_test.py rename to python/paddle/v2/reader/tests/creator_test.py index eda8ab6715..9f8d7133b8 100644 --- a/python/paddle/reader/tests/creator_test.py +++ b/python/paddle/v2/reader/tests/creator_test.py @@ -11,17 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest -import paddle.reader.creator + import numpy as np -import os + +import paddle.v2.reader.creator class TestNumpyArray(unittest.TestCase): def test_numpy_array(self): l = [[1, 2, 3], [4, 5, 6]] x = np.array(l, np.int32) - reader = paddle.reader.creator.np_array(x) + reader = paddle.v2.reader.creator.np_array(x) for idx, e in enumerate(reader()): self.assertItemsEqual(e, l[idx]) @@ -29,7 +31,7 @@ class TestNumpyArray(unittest.TestCase): class TestTextFile(unittest.TestCase): def test_text_file(self): path = os.path.join(os.path.dirname(__file__), "test_data_creator.txt") - reader = paddle.reader.creator.text_file(path) + reader = paddle.v2.reader.creator.text_file(path) for idx, e in enumerate(reader()): self.assertEqual(e, str(idx * 2) + " " + str(idx * 2 + 1)) diff --git a/python/paddle/reader/tests/decorator_test.py b/python/paddle/v2/reader/tests/decorator_test.py similarity index 81% rename from python/paddle/reader/tests/decorator_test.py rename to python/paddle/v2/reader/tests/decorator_test.py index 0396a61786..734154b979 100644 --- a/python/paddle/reader/tests/decorator_test.py +++ b/python/paddle/v2/reader/tests/decorator_test.py @@ -11,9 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import unittest -import paddle.reader import time +import unittest + +import paddle.v2.reader def reader_creator_10(dur): @@ -37,7 +38,7 @@ class TestMap(unittest.TestCase): yield "h" yield "i" - r = paddle.reader.map_readers(tokenize, read) + r = paddle.v2.reader.map_readers(tokenize, read) for i, e in enumerate(r()): self.assertEqual(e, i) @@ -45,7 +46,7 @@ class TestMap(unittest.TestCase): class TestBuffered(unittest.TestCase): def test_read(self): for size in range(20): - b = paddle.reader.buffered(reader_creator_10(0), size) + b = paddle.v2.reader.buffered(reader_creator_10(0), size) c = 0 for i in b(): self.assertEqual(i, c) @@ -54,7 +55,7 @@ class TestBuffered(unittest.TestCase): def test_buffering(self): # read have 30ms delay. - b = paddle.reader.buffered(reader_creator_10(0.03), 10) + b = paddle.v2.reader.buffered(reader_creator_10(0.03), 10) last_time = time.time() for idx, i in enumerate(b()): elapsed_time = time.time() - last_time @@ -68,17 +69,17 @@ class TestBuffered(unittest.TestCase): class TestCompose(unittest.TestCase): def test_compse(self): - reader = paddle.reader.compose( + reader = paddle.v2.reader.compose( reader_creator_10(0), reader_creator_10(0)) for idx, e in enumerate(reader()): self.assertEqual(e, (idx, idx)) def test_compose_not_aligned(self): total = 0 - reader = paddle.reader.compose( - paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)), + reader = paddle.v2.reader.compose( + paddle.v2.reader.chain(reader_creator_10(0), reader_creator_10(0)), reader_creator_10(0)) - with self.assertRaises(paddle.reader.ComposeNotAligned): + with self.assertRaises(paddle.v2.reader.ComposeNotAligned): for e in reader(): total += 1 # expecting 10, not 20 @@ -86,8 +87,8 @@ class TestCompose(unittest.TestCase): def test_compose_not_aligned_no_check(self): total = 0 - reader = paddle.reader.compose( - paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)), + reader = paddle.v2.reader.compose( + paddle.v2.reader.chain(reader_creator_10(0), reader_creator_10(0)), reader_creator_10(0), check_alignment=False) for e in reader(): @@ -98,7 +99,7 @@ class TestCompose(unittest.TestCase): class TestChain(unittest.TestCase): def test_chain(self): - c = paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)) + c = paddle.v2.reader.chain(reader_creator_10(0), reader_creator_10(0)) idx = 0 for e in c(): self.assertEqual(e, idx % 10) @@ -111,7 +112,7 @@ class TestShuffle(unittest.TestCase): case = [(0, True), (1, True), (10, False), (100, False)] a = reader_creator_10(0) for size, checkEq in case: - s = paddle.reader.shuffle(a, size) + s = paddle.v2.reader.shuffle(a, size) total = 0 for idx, e in enumerate(s()): if checkEq: diff --git a/python/paddle/reader/tests/test_data_creator.txt b/python/paddle/v2/reader/tests/test_data_creator.txt similarity index 100% rename from python/paddle/reader/tests/test_data_creator.txt rename to python/paddle/v2/reader/tests/test_data_creator.txt diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index f65c631f7b..89415787eb 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -62,7 +62,7 @@ class SGD(ITrainer): self.__optimizer__ = update_equation def train(self, - train_data_reader, + train_reader_creator, topology, parameters, num_passes=1, @@ -74,7 +74,7 @@ class SGD(ITrainer): """ Training method. Will train num_passes of input data. - :param train_data_reader: + :param train_reader_creator: :param topology: Network Topology, use one or more Layers to represent it. :param parameters: The parameter pools. :param num_passes: The total train passes. @@ -109,7 +109,7 @@ class SGD(ITrainer): for pass_id in xrange(num_passes): updater.startPass() for batch_id, data_batch in enumerate( - __data_reader_to_batch__(train_data_reader, batch_size, + __data_reader_to_batch__(train_reader_creator, batch_size, topology)): pass_type = updater.startBatch(len(data_batch)) gm.forwardBackward(feeder(data_batch), out_args, pass_type) diff --git a/python/setup.py.in b/python/setup.py.in index 1e1324eea8..68ca35265c 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -5,7 +5,9 @@ packages=['paddle', 'paddle.trainer', 'paddle.trainer_config_helpers', 'paddle.utils', - 'paddle.v2'] + 'paddle.v2', + 'paddle.v2.dataset', + 'paddle.v2.reader'] setup(name='paddle', version='${PADDLE_VERSION}', From 46d2ca575b98fef6d2506a66651c51400f189617 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 14:44:59 +0800 Subject: [PATCH 03/14] Combine Reader=>Feeder together. --- demo/mnist/api_train_v2.py | 38 +++++++++--------- python/paddle/v2/__init__.py | 3 +- python/paddle/v2/reader/decorator.py | 24 ++++++++++- python/paddle/v2/trainer.py | 59 +++++----------------------- 4 files changed, 52 insertions(+), 72 deletions(-) diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index 650bf392bb..67e36e6b20 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -1,15 +1,6 @@ import numpy import paddle.v2 as paddle -import mnist_util - - -def train_reader(): - train_file = './data/raw_data/train' - generator = mnist_util.read_from_mnist(train_file) - for item in generator: - yield item - def main(): paddle.init(use_gpu=False, trainer_count=1) @@ -45,17 +36,24 @@ def main(): trainer = paddle.trainer.SGD(update_equation=adam_optimizer) - trainer.train(train_data_reader=train_reader, - topology=cost, - parameters=parameters, - event_handler=event_handler, - batch_size=32, # batch size should be refactor in Data reader - data_types=[ # data_types will be removed, It should be in - # network topology - ('pixel', images.type), - ('label', label.type)], - reader_dict={'pixel':0, 'label':1} - ) + reader = paddle.reader.batched( + paddle.reader.shuffle( + paddle.dataset.mnist.train_creator(), buf_size=8192), + batch_size=32) + + trainer.train( + train_reader=paddle.reader.batched( + paddle.reader.shuffle(paddle.dataset.mnist.train_creator(), + buf_size=8192), batch_size=32), + topology=cost, + parameters=parameters, + event_handler=event_handler, + data_types=[ # data_types will be removed, It should be in + # network topology + ('pixel', images.type), + ('label', label.type)], + reader_dict={'pixel': 0, 'label': 1} + ) if __name__ == '__main__': diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 1122bcb5e4..cc130caa15 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -20,12 +20,13 @@ import event import data_type import data_feeder from . import dataset +from . import reader import attr import py_paddle.swig_paddle as api __all__ = [ 'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer', - 'event', 'data_type', 'attr', 'data_feeder', 'dataset' + 'event', 'data_type', 'attr', 'data_feeder', 'dataset', 'reader' ] diff --git a/python/paddle/v2/reader/decorator.py b/python/paddle/v2/reader/decorator.py index 9f4234358f..d83e2f4577 100644 --- a/python/paddle/v2/reader/decorator.py +++ b/python/paddle/v2/reader/decorator.py @@ -14,7 +14,7 @@ __all__ = [ 'map_readers', 'buffered', 'compose', 'chain', 'shuffle', - 'ComposeNotAligned' + 'ComposeNotAligned', 'batched' ] from Queue import Queue @@ -191,3 +191,25 @@ def buffered(reader, size): e = q.get() return data_reader + + +def batched(reader, batch_size): + """ + Create a batched reader. + :param reader: the data reader to read from. + :param batch_size: batch_size + :return: the batched reader. + """ + + def __impl__(): + r = reader() + batch = [] + for instance in r: + batch.append(instance) + if len(batch) == batch_size: + yield batch + batch = [] + if batch: + yield batch + + return __impl__ diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index 89415787eb..9b45ee69f3 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -29,7 +29,7 @@ class ITrainer(object): """ def train(self, - train_data_reader, + train_reader_creator, topology, parameters, test_data_reader=None, @@ -37,7 +37,7 @@ class ITrainer(object): """ train method. - :param train_data_reader: + :param train_reader_creator: :param topology: :param parameters: :param test_data_reader: @@ -62,27 +62,23 @@ class SGD(ITrainer): self.__optimizer__ = update_equation def train(self, - train_reader_creator, + train_reader, topology, parameters, num_passes=1, - test_data_reader=None, event_handler=None, - batch_size=32, data_types=None, reader_dict=None): """ Training method. Will train num_passes of input data. - :param train_reader_creator: + :param train_reader: :param topology: Network Topology, use one or more Layers to represent it. :param parameters: The parameter pools. :param num_passes: The total train passes. - :param test_data_reader: :param event_handler: Event handler. A method will be invoked when event occurred. :type event_handler: (BaseEvent) => None - :param batch_size: Not important, will be removed after data refactor. :param data_types: Not important, will be removed after data refactor. :return: """ @@ -108,9 +104,7 @@ class SGD(ITrainer): for pass_id in xrange(num_passes): updater.startPass() - for batch_id, data_batch in enumerate( - __data_reader_to_batch__(train_reader_creator, batch_size, - topology)): + for batch_id, data_batch in enumerate(train_reader()): pass_type = updater.startBatch(len(data_batch)) gm.forwardBackward(feeder(data_batch), out_args, pass_type) for each_param in gm.getParameters(): @@ -128,51 +122,16 @@ class SGD(ITrainer): gm.finish() -def __data_reader_to_batch__(reader, batch_size, topology): - """ - This function is not important, and will be removed when data refactored. - """ - - def input_reorder(func): - for item in func(): - retv = [] - for __layer_name__ in topology.input_layer_names: - retv.append(item[__layer_name__]) - yield retv - - return __generator_to_batch__(input_reorder(reader), batch_size=batch_size) - - -def __generator_to_batch__(generator, batch_size): - """ - This function is not important, and will be removed when data refactored. - """ - ret_val = list() - for each_item in generator: - ret_val.append(each_item) - if len(ret_val) == batch_size: - yield ret_val - ret_val = list() - if len(ret_val) != 0: - yield ret_val - - -def __check_train_args__(train_data_reader, topology, parameters, - test_data_reader, event_handler, **kwargs): +def __check_train_args__(train_reader, topology, parameters, event_handler, + **kwargs): """ Check train function's argument types """ - if not callable(train_data_reader) or not isinstance(train_data_reader(), - collections.Iterator): + if not callable(train_reader) or not isinstance(train_reader(), + collections.Iterator): raise TypeError('train_data_reader should be a function, ' 'which can return a iterator') - if test_data_reader is not None: - if not callable(test_data_reader) or not isinstance( - test_data_reader(), collections.Iterator): - raise TypeError('test_data_reader should be a function, which can ' - 'return a iterator') - if not isinstance(topology, ModelConfig): raise TypeError('topology should be a model config') From f02698675d0ce833bee27057aec17b86e46a00c0 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 14:51:37 +0800 Subject: [PATCH 04/14] Remove test code --- demo/mnist/api_train_v2.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index 67e36e6b20..d5eeb05318 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -36,11 +36,6 @@ def main(): trainer = paddle.trainer.SGD(update_equation=adam_optimizer) - reader = paddle.reader.batched( - paddle.reader.shuffle( - paddle.dataset.mnist.train_creator(), buf_size=8192), - batch_size=32) - trainer.train( train_reader=paddle.reader.batched( paddle.reader.shuffle(paddle.dataset.mnist.train_creator(), From f3f24604350cedb62bc05b02449958b4e9aca4d4 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 15:08:58 +0800 Subject: [PATCH 05/14] Make unittest ok. --- python/paddle/v2/reader/tests/CMakeLists.txt | 12 ++----- python/paddle/v2/reader/tests/run_tests.sh | 35 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) create mode 100755 python/paddle/v2/reader/tests/run_tests.sh diff --git a/python/paddle/v2/reader/tests/CMakeLists.txt b/python/paddle/v2/reader/tests/CMakeLists.txt index da072fb3db..a51f700406 100644 --- a/python/paddle/v2/reader/tests/CMakeLists.txt +++ b/python/paddle/v2/reader/tests/CMakeLists.txt @@ -1,9 +1,3 @@ -add_test(NAME reader_decorator_test - COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/reader/tests/decorator_test.py - WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) - -add_test(NAME reader_creator_test - COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/reader/tests/creator_test.py - WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) +add_test(NAME reader_tests + COMMAND bash ${PROJ_ROOT}/python/paddle/v2/reader/tests/run_tests.sh + ${PYTHON_EXECUTABLE}) diff --git a/python/paddle/v2/reader/tests/run_tests.sh b/python/paddle/v2/reader/tests/run_tests.sh new file mode 100755 index 0000000000..a544a56360 --- /dev/null +++ b/python/paddle/v2/reader/tests/run_tests.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +pushd `dirname $0` > /dev/null +SCRIPTPATH=$PWD +popd > /dev/null + +cd $SCRIPTPATH +$1 -m pip install ../../../../../paddle/dist/*.whl + +test_list="creator_test.py decorator_test.py" + +export PYTHONPATH=$PWD/../../../../../python/ + +for fn in $test_list +do + echo "test $fn" + $1 $fn + if [ $? -ne 0 ]; then + exit 1 + fi +done From dcc332a89001a01fb982938cc6c408b50ebd895a Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 15:12:53 +0800 Subject: [PATCH 06/14] Follow qq's comments --- demo/mnist/api_train_v2.py | 2 +- python/paddle/v2/trainer.py | 22 +++++++--------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index d5eeb05318..4d95780f06 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -37,7 +37,7 @@ def main(): trainer = paddle.trainer.SGD(update_equation=adam_optimizer) trainer.train( - train_reader=paddle.reader.batched( + reader=paddle.reader.batched( paddle.reader.shuffle(paddle.dataset.mnist.train_creator(), buf_size=8192), batch_size=32), topology=cost, diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index 9b45ee69f3..cbeef0306b 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -28,19 +28,13 @@ class ITrainer(object): The interface of Trainer. The only exposed method is `train`. """ - def train(self, - train_reader_creator, - topology, - parameters, - test_data_reader=None, - event_handler=None): + def train(self, reader, topology, parameters, event_handler=None): """ train method. - :param train_reader_creator: + :param reader: :param topology: :param parameters: - :param test_data_reader: :param event_handler: :return: """ @@ -62,7 +56,7 @@ class SGD(ITrainer): self.__optimizer__ = update_equation def train(self, - train_reader, + reader, topology, parameters, num_passes=1, @@ -72,7 +66,7 @@ class SGD(ITrainer): """ Training method. Will train num_passes of input data. - :param train_reader: + :param reader: :param topology: Network Topology, use one or more Layers to represent it. :param parameters: The parameter pools. :param num_passes: The total train passes. @@ -104,7 +98,7 @@ class SGD(ITrainer): for pass_id in xrange(num_passes): updater.startPass() - for batch_id, data_batch in enumerate(train_reader()): + for batch_id, data_batch in enumerate(reader()): pass_type = updater.startBatch(len(data_batch)) gm.forwardBackward(feeder(data_batch), out_args, pass_type) for each_param in gm.getParameters(): @@ -122,13 +116,11 @@ class SGD(ITrainer): gm.finish() -def __check_train_args__(train_reader, topology, parameters, event_handler, - **kwargs): +def __check_train_args__(reader, topology, parameters, event_handler, **kwargs): """ Check train function's argument types """ - if not callable(train_reader) or not isinstance(train_reader(), - collections.Iterator): + if not callable(reader) or not isinstance(reader(), collections.Iterator): raise TypeError('train_data_reader should be a function, ' 'which can return a iterator') From 692df52ef801ce79180ada37dc0fbae94282ae34 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 16:34:54 +0800 Subject: [PATCH 07/14] Fix unittest --- .travis.yml | 2 +- python/setup.py.in | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fb2ca9387..3993dc31fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ before_install: - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python # protobuf version. - - pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker + - pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker 'scikit-learn>=0.18.0' script: - paddle/scripts/travis/main.sh notifications: diff --git a/python/setup.py.in b/python/setup.py.in index 68ca35265c..fbee7bca90 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -15,5 +15,8 @@ setup(name='paddle', packages=packages, package_dir={ '': '${CMAKE_CURRENT_SOURCE_DIR}' - } + }, + install_requires = [ + 'scikit-learn>=0.18.0' + ] ) From 9435025b8e267c84431e3b606da8197d221062e3 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 27 Feb 2017 20:39:04 +0800 Subject: [PATCH 08/14] Add dependencies --- .travis.yml | 2 +- python/setup.py.in | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3993dc31fd..31d76ecb87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ before_install: - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python # protobuf version. - - pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker 'scikit-learn>=0.18.0' + - pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker 'scikit-learn>=0.18.0' 'scipy>=0.18.0' script: - paddle/scripts/travis/main.sh notifications: diff --git a/python/setup.py.in b/python/setup.py.in index fbee7bca90..c7e46293b7 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -17,6 +17,7 @@ setup(name='paddle', '': '${CMAKE_CURRENT_SOURCE_DIR}' }, install_requires = [ - 'scikit-learn>=0.18.0' + 'scikit-learn>=0.18.0', + 'scipy>=0.18.0', ] ) From 792875e3eaa0467e40748f0ed97f022fe7fdcd0b Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 28 Feb 2017 09:56:27 +0800 Subject: [PATCH 09/14] Lazy initialize mnist dataset. Fix unittest --- python/paddle/v2/dataset/mnist.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index db84f37aa4..faae818a5d 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -16,18 +16,29 @@ def __mnist_reader_creator__(data, target): TEST_SIZE = 10000 +X_train = None +X_test = None +y_train = None +y_test = None -data = sklearn.datasets.mldata.fetch_mldata( - "MNIST original", data_home=DATA_HOME) -X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - data.data, data.target, test_size=TEST_SIZE, random_state=0) + +def __initialize_dataset__(): + global X_train, X_test, y_train, y_test + if X_train is not None: + return + data = sklearn.datasets.mldata.fetch_mldata( + "MNIST original", data_home=DATA_HOME) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + data.data, data.target, test_size=TEST_SIZE, random_state=0) def train_creator(): + __initialize_dataset__() return __mnist_reader_creator__(X_train, y_train) def test_creator(): + __initialize_dataset__() return __mnist_reader_creator__(X_test, y_test) From 0432e9073984d0c7ce3329043cb3d6fe6ef65a8a Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 28 Feb 2017 13:44:38 +0800 Subject: [PATCH 10/14] remove unused python package in TravisCI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 31d76ecb87..4fb2ca9387 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ before_install: - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python # protobuf version. - - pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker 'scikit-learn>=0.18.0' 'scipy>=0.18.0' + - pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker script: - paddle/scripts/travis/main.sh notifications: From 01ad3bfdb9daf494967a0789bcad09057c11bdb6 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 28 Feb 2017 13:52:57 +0800 Subject: [PATCH 11/14] Fix reader creator in mnist --- demo/mnist/api_train_v2.py | 2 +- python/paddle/v2/dataset/mnist.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index c9fd7c72df..1c2c831bec 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -33,7 +33,7 @@ def main(): trainer.train( reader=paddle.reader.batched( paddle.reader.shuffle( - paddle.dataset.mnist.train_creator(), buf_size=8192), + paddle.dataset.mnist.train(), buf_size=8192), batch_size=32), cost=cost, parameters=parameters, diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index 8ba11ca5ec..653c91aacd 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -45,7 +45,7 @@ def reader_creator(image_filename, label_filename, buffer_size): m.terminate() l.terminate() - return reader() + return reader def train(): From c852003d6b2e11c8d0f75d5c0adf9d332dc429f1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 28 Feb 2017 14:00:22 +0800 Subject: [PATCH 12/14] Fix errors in mnist dataset --- demo/mnist/api_train_v2.py | 1 - python/paddle/v2/dataset/mnist.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index 1c2c831bec..612b0d218f 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -38,7 +38,6 @@ def main(): cost=cost, parameters=parameters, event_handler=event_handler, - batch_size=32, # batch size should be refactor in Data reader reader_dict={images.name: 0, label.name: 1}) diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index 653c91aacd..045bcfcc80 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -1,6 +1,7 @@ import paddle.v2.dataset.common import subprocess import numpy +import platform __all__ = ['train', 'test'] @@ -18,12 +19,19 @@ TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432' def reader_creator(image_filename, label_filename, buffer_size): def reader(): + if platform.system() == 'Darwin': + zcat_cmd = 'gzcat' + elif platform.system() == 'Linux': + zcat_cmd = 'zcat' + else: + raise NotImplementedError() + # According to http://stackoverflow.com/a/38061619/724872, we # cannot use standard package gzip here. - m = subprocess.Popen(["zcat", image_filename], stdout=subprocess.PIPE) + m = subprocess.Popen([zcat_cmd, image_filename], stdout=subprocess.PIPE) m.stdout.read(16) # skip some magic bytes - l = subprocess.Popen(["zcat", label_filename], stdout=subprocess.PIPE) + l = subprocess.Popen([zcat_cmd, label_filename], stdout=subprocess.PIPE) l.stdout.read(8) # skip some magic bytes while True: From faa43e3682a993450653904c9a66ad7f37c182f3 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 28 Feb 2017 14:02:09 +0800 Subject: [PATCH 13/14] Remove unused dependencies --- python/setup.py.in | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index c7e46293b7..68ca35265c 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -15,9 +15,5 @@ setup(name='paddle', packages=packages, package_dir={ '': '${CMAKE_CURRENT_SOURCE_DIR}' - }, - install_requires = [ - 'scikit-learn>=0.18.0', - 'scipy>=0.18.0', - ] + } ) From eee1320b873f4dcfd78857d80aa06a99cca90916 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 28 Feb 2017 14:55:09 +0800 Subject: [PATCH 14/14] Change code style --- python/paddle/v2/reader/decorator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/reader/decorator.py b/python/paddle/v2/reader/decorator.py index d83e2f4577..5687f118ce 100644 --- a/python/paddle/v2/reader/decorator.py +++ b/python/paddle/v2/reader/decorator.py @@ -201,7 +201,7 @@ def batched(reader, batch_size): :return: the batched reader. """ - def __impl__(): + def batched_reader(): r = reader() batch = [] for instance in r: @@ -212,4 +212,4 @@ def batched(reader, batch_size): if batch: yield batch - return __impl__ + return batched_reader