From 7fe21602f6b767c64e058ab99ad71a1db6bd5d74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=9B=8A?= Date: Mon, 27 Feb 2017 14:43:43 -0800 Subject: [PATCH 1/9] Rename config.py into common.py --- python/paddle/v2/dataset/cifar.py | 2 +- python/paddle/v2/dataset/{config.py => common.py} | 0 python/paddle/v2/dataset/mnist.py | 2 +- python/paddle/v2/dataset/movielens.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename python/paddle/v2/dataset/{config.py => common.py} (100%) diff --git a/python/paddle/v2/dataset/cifar.py b/python/paddle/v2/dataset/cifar.py index 2ac71c6eff..accb32f117 100644 --- a/python/paddle/v2/dataset/cifar.py +++ b/python/paddle/v2/dataset/cifar.py @@ -11,7 +11,7 @@ import tarfile import numpy -from config import download +from common import download __all__ = [ 'cifar_100_train_creator', 'cifar_100_test_creator', 'train_creator', diff --git a/python/paddle/v2/dataset/config.py b/python/paddle/v2/dataset/common.py similarity index 100% rename from python/paddle/v2/dataset/config.py rename to python/paddle/v2/dataset/common.py diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index db84f37aa4..2f195bfb96 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -1,7 +1,7 @@ import sklearn.datasets.mldata import sklearn.model_selection import numpy -from config import DATA_HOME +from common import DATA_HOME __all__ = ['train_creator', 'test_creator'] diff --git a/python/paddle/v2/dataset/movielens.py b/python/paddle/v2/dataset/movielens.py index 314329e91c..dcffcff2f5 100644 --- a/python/paddle/v2/dataset/movielens.py +++ b/python/paddle/v2/dataset/movielens.py @@ -1,5 +1,5 @@ import zipfile -from config import download +from common import download import re import random import functools From b93722df95d3907782cdff034df360b79d1fd093 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Mon, 27 Feb 2017 23:07:33 +0000 Subject: [PATCH 2/9] Set data cache home directory to ~/.cache/paddle/dataset --- python/paddle/v2/dataset/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py index 02a009f09c..ae4a5383b0 100644 --- a/python/paddle/v2/dataset/common.py +++ b/python/paddle/v2/dataset/common.py @@ -5,7 +5,7 @@ import urllib2 __all__ = ['DATA_HOME', 'download'] -DATA_HOME = os.path.expanduser('~/.cache/paddle_data_set') +DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset') if not os.path.exists(DATA_HOME): os.makedirs(DATA_HOME) From 37e2b92089ed583ba9e73f615444c3b080cd1b63 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Mon, 27 Feb 2017 23:41:32 +0000 Subject: [PATCH 3/9] Add md5file into dataset/common.py, and unit test in tests/common_test.py --- python/paddle/v2/dataset/common.py | 13 +++++++++++-- python/paddle/v2/dataset/tests/common_test.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 python/paddle/v2/dataset/tests/common_test.py diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py index ae4a5383b0..ff5ed76c0f 100644 --- a/python/paddle/v2/dataset/common.py +++ b/python/paddle/v2/dataset/common.py @@ -3,7 +3,7 @@ import os import shutil import urllib2 -__all__ = ['DATA_HOME', 'download'] +__all__ = ['DATA_HOME', 'download', 'md5file'] DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset') @@ -11,7 +11,7 @@ if not os.path.exists(DATA_HOME): os.makedirs(DATA_HOME) -def download(url, md5): +def download(url, package_name, md5): filename = os.path.split(url)[-1] assert DATA_HOME is not None filepath = os.path.join(DATA_HOME, md5) @@ -34,3 +34,12 @@ def download(url, md5): with open(__full_file__, mode='wb') as of: shutil.copyfileobj(fsrc=response, fdst=of) return __full_file__ + + +def md5file(fname): + hash_md5 = hashlib.md5() + f = open(fname, "rb") + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + f.close() + return hash_md5.hexdigest() diff --git a/python/paddle/v2/dataset/tests/common_test.py b/python/paddle/v2/dataset/tests/common_test.py new file mode 100644 index 0000000000..d2f97f06de --- /dev/null +++ b/python/paddle/v2/dataset/tests/common_test.py @@ -0,0 +1,16 @@ +import paddle.v2.dataset.common +import unittest +import tempfile + +class TestCommon(unittest.TestCase): + def test_md5file(self): + _, temp_path =tempfile.mkstemp() + f = open(temp_path, 'w') + f.write("Hello\n") + f.close() + self.assertEqual( + '09f7e02f1290be211da707a266f153b3', + paddle.v2.dataset.common.md5file(temp_path)) + +if __name__ == '__main__': + unittest.main() From 91115ab6de016806a5d3ad168b114f54b1eaac87 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 28 Feb 2017 00:17:52 +0000 Subject: [PATCH 4/9] Use module name and raw data filename as the local filename --- python/paddle/v2/dataset/common.py | 42 +++++++------------ python/paddle/v2/dataset/tests/common_test.py | 12 ++++-- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py index ff5ed76c0f..b1831f38af 100644 --- a/python/paddle/v2/dataset/common.py +++ b/python/paddle/v2/dataset/common.py @@ -1,7 +1,7 @@ +import requests import hashlib import os import shutil -import urllib2 __all__ = ['DATA_HOME', 'download', 'md5file'] @@ -11,31 +11,6 @@ if not os.path.exists(DATA_HOME): os.makedirs(DATA_HOME) -def download(url, package_name, md5): - filename = os.path.split(url)[-1] - assert DATA_HOME is not None - filepath = os.path.join(DATA_HOME, md5) - if not os.path.exists(filepath): - os.makedirs(filepath) - __full_file__ = os.path.join(filepath, filename) - - def __file_ok__(): - if not os.path.exists(__full_file__): - return False - md5_hash = hashlib.md5() - with open(__full_file__, 'rb') as f: - for chunk in iter(lambda: f.read(4096), b""): - md5_hash.update(chunk) - - return md5_hash.hexdigest() == md5 - - while not __file_ok__(): - response = urllib2.urlopen(url) - with open(__full_file__, mode='wb') as of: - shutil.copyfileobj(fsrc=response, fdst=of) - return __full_file__ - - def md5file(fname): hash_md5 = hashlib.md5() f = open(fname, "rb") @@ -43,3 +18,18 @@ def md5file(fname): hash_md5.update(chunk) f.close() return hash_md5.hexdigest() + + +def download(url, module_name, md5sum): + dirname = os.path.join(DATA_HOME, module_name) + if not os.path.exists(dirname): + os.makedirs(dirname) + + filename = os.path.join(dirname, url.split('/')[-1]) + if not (os.path.exists(filename) and md5file(filename) == md5sum): + # If file doesn't exist or MD5 doesn't match, then download. + r = requests.get(url, stream=True) + with open(filename, 'w') as f: + shutil.copyfileobj(r.raw, f) + + return filename diff --git a/python/paddle/v2/dataset/tests/common_test.py b/python/paddle/v2/dataset/tests/common_test.py index d2f97f06de..0672a46714 100644 --- a/python/paddle/v2/dataset/tests/common_test.py +++ b/python/paddle/v2/dataset/tests/common_test.py @@ -5,12 +5,18 @@ import tempfile class TestCommon(unittest.TestCase): def test_md5file(self): _, temp_path =tempfile.mkstemp() - f = open(temp_path, 'w') - f.write("Hello\n") - f.close() + with open(temp_path, 'w') as f: + f.write("Hello\n") self.assertEqual( '09f7e02f1290be211da707a266f153b3', paddle.v2.dataset.common.md5file(temp_path)) + def test_download(self): + yi_avatar = 'https://avatars0.githubusercontent.com/u/1548775?v=3&s=460' + self.assertEqual( + paddle.v2.dataset.common.DATA_HOME + '/test/1548775?v=3&s=460', + paddle.v2.dataset.common.download( + yi_avatar, 'test', 'f75287202d6622414c706c36c16f8e0d')) + if __name__ == '__main__': unittest.main() From d6c62e852d7788ba81e704323995874b85f89c3e Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 28 Feb 2017 02:26:30 +0000 Subject: [PATCH 5/9] Rewrite mnist.py and add mnist_test.py --- python/paddle/v2/dataset/mnist.py | 74 ++++++++++++++------ python/paddle/v2/dataset/tests/mnist_test.py | 27 +++++++ 2 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 python/paddle/v2/dataset/tests/mnist_test.py diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index 2f195bfb96..29fc20eae9 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -1,39 +1,67 @@ -import sklearn.datasets.mldata -import sklearn.model_selection +import paddle.v2.dataset.common +import subprocess import numpy -from common import DATA_HOME -__all__ = ['train_creator', 'test_creator'] +URL_PREFIX = 'http://yann.lecun.com/exdb/mnist/' +TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' +TEST_IMAGE_MD5 = '25e3cc63507ef6e98d5dc541e8672bb6' -def __mnist_reader_creator__(data, target): - def reader(): - n_samples = data.shape[0] - for i in xrange(n_samples): - yield (data[i] / 255.0).astype(numpy.float32), int(target[i]) +TEST_LABEL_URL = URL_PREFIX + 't10k-labels-idx1-ubyte.gz' +TEST_LABEL_MD5 = '4e9511fe019b2189026bd0421ba7b688' + +TRAIN_IMAGE_URL = URL_PREFIX + 'train-images-idx3-ubyte.gz' +TRAIN_IMAGE_MD5 = 'f68b3c2dcbeaaa9fbdd348bbdeb94873' - return reader +TRAIN_LABEL_URL = URL_PREFIX + 'train-labels-idx1-ubyte.gz' +TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432' -TEST_SIZE = 10000 +def reader_creator(image_filename, label_filename, buffer_size): + def reader(): + # According to http://stackoverflow.com/a/38061619/724872, we + # cannot use standard package gzip here. + m = subprocess.Popen(["zcat", image_filename], stdout=subprocess.PIPE) + m.stdout.read(16) # skip some magic bytes + + l = subprocess.Popen(["zcat", label_filename], stdout=subprocess.PIPE) + l.stdout.read(8) # skip some magic bytes -data = sklearn.datasets.mldata.fetch_mldata( - "MNIST original", data_home=DATA_HOME) -X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - data.data, data.target, test_size=TEST_SIZE, random_state=0) + while True: + labels = numpy.fromfile( + l.stdout, 'ubyte', count=buffer_size + ).astype("int") + if labels.size != buffer_size: + break # numpy.fromfile returns empty slice after EOF. -def train_creator(): - return __mnist_reader_creator__(X_train, y_train) + images = numpy.fromfile( + m.stdout, 'ubyte', count=buffer_size * 28 * 28 + ).reshape((buffer_size, 28 * 28) + ).astype('float32') + images = images / 255.0 * 2.0 - 1.0 -def test_creator(): - return __mnist_reader_creator__(X_test, y_test) + for i in xrange(buffer_size): + yield images[i, :], labels[i] + m.terminate() + l.terminate() -def unittest(): - assert len(list(test_creator()())) == TEST_SIZE + return reader() +def train(): + return reader_creator( + paddle.v2.dataset.common.download( + TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5), + paddle.v2.dataset.common.download( + TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5), + 100) -if __name__ == '__main__': - unittest() +def test(): + return reader_creator( + paddle.v2.dataset.common.download( + TEST_IMAGE_URL, 'mnist', TEST_IMAGE_MD5), + paddle.v2.dataset.common.download( + TEST_LABEL_URL, 'mnist', TEST_LABEL_MD5), + 100) diff --git a/python/paddle/v2/dataset/tests/mnist_test.py b/python/paddle/v2/dataset/tests/mnist_test.py new file mode 100644 index 0000000000..23ed2eaba8 --- /dev/null +++ b/python/paddle/v2/dataset/tests/mnist_test.py @@ -0,0 +1,27 @@ +import paddle.v2.dataset.mnist +import unittest + +class TestMNIST(unittest.TestCase): + def check_reader(self, reader): + sum = 0 + for l in reader: + self.assertEqual(l[0].size, 784) + self.assertEqual(l[1].size, 1) + self.assertLess(l[1], 10) + self.assertGreaterEqual(l[1], 0) + sum += 1 + return sum + + def test_train(self): + self.assertEqual( + self.check_reader(paddle.v2.dataset.mnist.train()), + 60000) + + def test_test(self): + self.assertEqual( + self.check_reader(paddle.v2.dataset.mnist.test()), + 10000) + + +if __name__ == '__main__': + unittest.main() From dcbfbb15338e0ca0f195e12ce0e0275995622ca1 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 28 Feb 2017 02:46:31 +0000 Subject: [PATCH 6/9] yapf format --- python/paddle/v2/dataset/mnist.py | 34 +++++++++---------- python/paddle/v2/dataset/tests/common_test.py | 9 ++--- python/paddle/v2/dataset/tests/mnist_test.py | 7 ++-- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index 29fc20eae9..ec334d39e6 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -22,23 +22,21 @@ def reader_creator(image_filename, label_filename, buffer_size): # According to http://stackoverflow.com/a/38061619/724872, we # cannot use standard package gzip here. m = subprocess.Popen(["zcat", image_filename], stdout=subprocess.PIPE) - m.stdout.read(16) # skip some magic bytes + m.stdout.read(16) # skip some magic bytes l = subprocess.Popen(["zcat", label_filename], stdout=subprocess.PIPE) - l.stdout.read(8) # skip some magic bytes + l.stdout.read(8) # skip some magic bytes while True: labels = numpy.fromfile( - l.stdout, 'ubyte', count=buffer_size - ).astype("int") + l.stdout, 'ubyte', count=buffer_size).astype("int") if labels.size != buffer_size: - break # numpy.fromfile returns empty slice after EOF. + break # numpy.fromfile returns empty slice after EOF. images = numpy.fromfile( - m.stdout, 'ubyte', count=buffer_size * 28 * 28 - ).reshape((buffer_size, 28 * 28) - ).astype('float32') + m.stdout, 'ubyte', count=buffer_size * 28 * 28).reshape( + (buffer_size, 28 * 28)).astype('float32') images = images / 255.0 * 2.0 - 1.0 @@ -50,18 +48,18 @@ def reader_creator(image_filename, label_filename, buffer_size): return reader() + def train(): return reader_creator( - paddle.v2.dataset.common.download( - TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5), - paddle.v2.dataset.common.download( - TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5), - 100) + paddle.v2.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', + TRAIN_IMAGE_MD5), + paddle.v2.dataset.common.download(TRAIN_LABEL_URL, 'mnist', + TRAIN_LABEL_MD5), 100) + def test(): return reader_creator( - paddle.v2.dataset.common.download( - TEST_IMAGE_URL, 'mnist', TEST_IMAGE_MD5), - paddle.v2.dataset.common.download( - TEST_LABEL_URL, 'mnist', TEST_LABEL_MD5), - 100) + paddle.v2.dataset.common.download(TEST_IMAGE_URL, 'mnist', + TEST_IMAGE_MD5), + paddle.v2.dataset.common.download(TEST_LABEL_URL, 'mnist', + TEST_LABEL_MD5), 100) diff --git a/python/paddle/v2/dataset/tests/common_test.py b/python/paddle/v2/dataset/tests/common_test.py index 0672a46714..7d8406171b 100644 --- a/python/paddle/v2/dataset/tests/common_test.py +++ b/python/paddle/v2/dataset/tests/common_test.py @@ -2,14 +2,14 @@ import paddle.v2.dataset.common import unittest import tempfile + class TestCommon(unittest.TestCase): def test_md5file(self): - _, temp_path =tempfile.mkstemp() + _, temp_path = tempfile.mkstemp() with open(temp_path, 'w') as f: f.write("Hello\n") - self.assertEqual( - '09f7e02f1290be211da707a266f153b3', - paddle.v2.dataset.common.md5file(temp_path)) + self.assertEqual('09f7e02f1290be211da707a266f153b3', + paddle.v2.dataset.common.md5file(temp_path)) def test_download(self): yi_avatar = 'https://avatars0.githubusercontent.com/u/1548775?v=3&s=460' @@ -18,5 +18,6 @@ class TestCommon(unittest.TestCase): paddle.v2.dataset.common.download( yi_avatar, 'test', 'f75287202d6622414c706c36c16f8e0d')) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/dataset/tests/mnist_test.py b/python/paddle/v2/dataset/tests/mnist_test.py index 23ed2eaba8..e4f0b33d52 100644 --- a/python/paddle/v2/dataset/tests/mnist_test.py +++ b/python/paddle/v2/dataset/tests/mnist_test.py @@ -1,6 +1,7 @@ import paddle.v2.dataset.mnist import unittest + class TestMNIST(unittest.TestCase): def check_reader(self, reader): sum = 0 @@ -14,13 +15,11 @@ class TestMNIST(unittest.TestCase): def test_train(self): self.assertEqual( - self.check_reader(paddle.v2.dataset.mnist.train()), - 60000) + self.check_reader(paddle.v2.dataset.mnist.train()), 60000) def test_test(self): self.assertEqual( - self.check_reader(paddle.v2.dataset.mnist.test()), - 10000) + self.check_reader(paddle.v2.dataset.mnist.test()), 10000) if __name__ == '__main__': From 6bc82c8eb8478e2fe0911a8e43ddc7ed3539a372 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 28 Feb 2017 02:56:48 +0000 Subject: [PATCH 7/9] Add __all__ to mnist.py --- python/paddle/v2/dataset/mnist.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index ec334d39e6..8ba11ca5ec 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -2,17 +2,16 @@ import paddle.v2.dataset.common import subprocess import numpy +__all__ = ['train', 'test'] + URL_PREFIX = 'http://yann.lecun.com/exdb/mnist/' TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' TEST_IMAGE_MD5 = '25e3cc63507ef6e98d5dc541e8672bb6' - TEST_LABEL_URL = URL_PREFIX + 't10k-labels-idx1-ubyte.gz' TEST_LABEL_MD5 = '4e9511fe019b2189026bd0421ba7b688' - TRAIN_IMAGE_URL = URL_PREFIX + 'train-images-idx3-ubyte.gz' TRAIN_IMAGE_MD5 = 'f68b3c2dcbeaaa9fbdd348bbdeb94873' - TRAIN_LABEL_URL = URL_PREFIX + 'train-labels-idx1-ubyte.gz' TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432' From 4eb54c2437aef7c857151a2a3e4fd24aa51d5336 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 28 Feb 2017 04:30:30 +0000 Subject: [PATCH 8/9] Debug unit tests --- python/paddle/v2/dataset/cifar.py | 85 ++++++++------------ python/paddle/v2/dataset/common.py | 1 - python/paddle/v2/dataset/mnist.py | 10 ++- python/paddle/v2/dataset/tests/cifar_test.py | 42 ++++++++++ python/paddle/v2/dataset/tests/mnist_test.py | 22 ++--- 5 files changed, 93 insertions(+), 67 deletions(-) create mode 100644 python/paddle/v2/dataset/tests/cifar_test.py diff --git a/python/paddle/v2/dataset/cifar.py b/python/paddle/v2/dataset/cifar.py index accb32f117..77c54bd268 100644 --- a/python/paddle/v2/dataset/cifar.py +++ b/python/paddle/v2/dataset/cifar.py @@ -1,82 +1,61 @@ """ -CIFAR Dataset. - -URL: https://www.cs.toronto.edu/~kriz/cifar.html - -the default train_creator, test_creator used for CIFAR-10 dataset. +CIFAR dataset: https://www.cs.toronto.edu/~kriz/cifar.html """ import cPickle import itertools -import tarfile - import numpy +import paddle.v2.dataset.common +import tarfile -from common import download - -__all__ = [ - 'cifar_100_train_creator', 'cifar_100_test_creator', 'train_creator', - 'test_creator' -] +__all__ = ['train100', 'test100', 'train10', 'test10'] -CIFAR10_URL = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' +URL_PREFIX = 'https://www.cs.toronto.edu/~kriz/' +CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz' CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a' -CIFAR100_URL = 'https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz' +CIFAR100_URL = URL_PREFIX + 'cifar-100-python.tar.gz' CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85' -def __read_batch__(filename, sub_name): - def reader(): - def __read_one_batch_impl__(batch): - data = batch['data'] - labels = batch.get('labels', batch.get('fine_labels', None)) - assert labels is not None - for sample, label in itertools.izip(data, labels): - yield (sample / 255.0).astype(numpy.float32), int(label) +def reader_creator(filename, sub_name): + def read_batch(batch): + data = batch['data'] + labels = batch.get('labels', batch.get('fine_labels', None)) + assert labels is not None + for sample, label in itertools.izip(data, labels): + yield (sample / 255.0).astype(numpy.float32), int(label) + def reader(): with tarfile.open(filename, mode='r') as f: names = (each_item.name for each_item in f if sub_name in each_item.name) for name in names: batch = cPickle.load(f.extractfile(name)) - for item in __read_one_batch_impl__(batch): + for item in read_batch(batch): yield item return reader -def cifar_100_train_creator(): - fn = download(url=CIFAR100_URL, md5=CIFAR100_MD5) - return __read_batch__(fn, 'train') - - -def cifar_100_test_creator(): - fn = download(url=CIFAR100_URL, md5=CIFAR100_MD5) - return __read_batch__(fn, 'test') - - -def train_creator(): - """ - Default train reader creator. Use CIFAR-10 dataset. - """ - fn = download(url=CIFAR10_URL, md5=CIFAR10_MD5) - return __read_batch__(fn, 'data_batch') +def train100(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), + 'train') -def test_creator(): - """ - Default test reader creator. Use CIFAR-10 dataset. - """ - fn = download(url=CIFAR10_URL, md5=CIFAR10_MD5) - return __read_batch__(fn, 'test_batch') +def test100(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), + 'test') -def unittest(): - for _ in train_creator()(): - pass - for _ in test_creator()(): - pass +def train10(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'data_batch') -if __name__ == '__main__': - unittest() +def test10(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'test_batch') diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py index b1831f38af..a5ffe25a11 100644 --- a/python/paddle/v2/dataset/common.py +++ b/python/paddle/v2/dataset/common.py @@ -27,7 +27,6 @@ def download(url, module_name, md5sum): filename = os.path.join(dirname, url.split('/')[-1]) if not (os.path.exists(filename) and md5file(filename) == md5sum): - # If file doesn't exist or MD5 doesn't match, then download. r = requests.get(url, stream=True) with open(filename, 'w') as f: shutil.copyfileobj(r.raw, f) diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index 8ba11ca5ec..a36c20e3fa 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -1,11 +1,13 @@ +""" +MNIST dataset. +""" +import numpy import paddle.v2.dataset.common import subprocess -import numpy __all__ = ['train', 'test'] URL_PREFIX = 'http://yann.lecun.com/exdb/mnist/' - TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' TEST_IMAGE_MD5 = '25e3cc63507ef6e98d5dc541e8672bb6' TEST_LABEL_URL = URL_PREFIX + 't10k-labels-idx1-ubyte.gz' @@ -40,12 +42,12 @@ def reader_creator(image_filename, label_filename, buffer_size): images = images / 255.0 * 2.0 - 1.0 for i in xrange(buffer_size): - yield images[i, :], labels[i] + yield images[i, :], int(labels[i]) m.terminate() l.terminate() - return reader() + return reader def train(): diff --git a/python/paddle/v2/dataset/tests/cifar_test.py b/python/paddle/v2/dataset/tests/cifar_test.py new file mode 100644 index 0000000000..a2af45ecf5 --- /dev/null +++ b/python/paddle/v2/dataset/tests/cifar_test.py @@ -0,0 +1,42 @@ +import paddle.v2.dataset.cifar +import unittest + + +class TestCIFAR(unittest.TestCase): + def check_reader(self, reader): + sum = 0 + label = 0 + for l in reader(): + self.assertEqual(l[0].size, 3072) + if l[1] > label: + label = l[1] + sum += 1 + return sum, label + + def test_test10(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.test10()) + self.assertEqual(instances, 10000) + self.assertEqual(max_label_value, 9) + + def test_train10(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.train10()) + self.assertEqual(instances, 50000) + self.assertEqual(max_label_value, 9) + + def test_test100(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.test100()) + self.assertEqual(instances, 10000) + self.assertEqual(max_label_value, 99) + + def test_train100(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.train100()) + self.assertEqual(instances, 50000) + self.assertEqual(max_label_value, 99) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/dataset/tests/mnist_test.py b/python/paddle/v2/dataset/tests/mnist_test.py index e4f0b33d52..b4408cc2f5 100644 --- a/python/paddle/v2/dataset/tests/mnist_test.py +++ b/python/paddle/v2/dataset/tests/mnist_test.py @@ -5,21 +5,25 @@ import unittest class TestMNIST(unittest.TestCase): def check_reader(self, reader): sum = 0 - for l in reader: + label = 0 + for l in reader(): self.assertEqual(l[0].size, 784) - self.assertEqual(l[1].size, 1) - self.assertLess(l[1], 10) - self.assertGreaterEqual(l[1], 0) + if l[1] > label: + label = l[1] sum += 1 - return sum + return sum, label def test_train(self): - self.assertEqual( - self.check_reader(paddle.v2.dataset.mnist.train()), 60000) + instances, max_label_value = self.check_reader( + paddle.v2.dataset.mnist.train()) + self.assertEqual(instances, 60000) + self.assertEqual(max_label_value, 9) def test_test(self): - self.assertEqual( - self.check_reader(paddle.v2.dataset.mnist.test()), 10000) + instances, max_label_value = self.check_reader( + paddle.v2.dataset.mnist.test()) + self.assertEqual(instances, 10000) + self.assertEqual(max_label_value, 9) if __name__ == '__main__': From 375899339330d00852433ced1b7f02663040c7fb Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 28 Feb 2017 13:29:24 +0800 Subject: [PATCH 9/9] Simplify layer.v2 --- .../default_decorators.py | 4 + .../paddle/trainer_config_helpers/layers.py | 6 + python/paddle/v2/layer.py | 143 ++++++------------ python/paddle/v2/tests/test_layer.py | 6 +- 4 files changed, 61 insertions(+), 98 deletions(-) diff --git a/python/paddle/trainer_config_helpers/default_decorators.py b/python/paddle/trainer_config_helpers/default_decorators.py index ad3efcbf36..2f25579fcd 100644 --- a/python/paddle/trainer_config_helpers/default_decorators.py +++ b/python/paddle/trainer_config_helpers/default_decorators.py @@ -52,6 +52,10 @@ def wrap_param_default(param_names=None, kwargs[name] = default_factory(func) return func(*args, **kwargs) + if hasattr(func, 'argspec'): + __wrapper__.argspec = func.argspec + else: + __wrapper__.argspec = inspect.getargspec(func) return __wrapper__ return __impl__ diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 1bb1a01d50..b68460b6a3 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -14,6 +14,7 @@ import functools import collections +import inspect from paddle.trainer.config_parser import * from .activations import LinearActivation, SigmoidActivation, TanhActivation, \ @@ -316,6 +317,11 @@ def layer_support(*attrs): val.check(method.__name__) return method(*args, **kwargs) + if hasattr(method, 'argspec'): + wrapper.argspec = method.argspec + else: + wrapper.argspec = inspect.getargspec(method) + return wrapper return decorator diff --git a/python/paddle/v2/layer.py b/python/paddle/v2/layer.py index d15e6398f5..d68b66cf02 100644 --- a/python/paddle/v2/layer.py +++ b/python/paddle/v2/layer.py @@ -67,6 +67,7 @@ paddle.v2.parameters.create, no longer exposed to users. """ import collections +import inspect import paddle.trainer_config_helpers as conf_helps from paddle.trainer_config_helpers.config_parser_utils import \ @@ -74,26 +75,14 @@ from paddle.trainer_config_helpers.config_parser_utils import \ from paddle.trainer_config_helpers.default_decorators import wrap_name_default from paddle.trainer_config_helpers.default_decorators import wrap_act_default -from paddle.trainer_config_helpers.default_decorators import wrap_bias_attr_default +from paddle.trainer_config_helpers.default_decorators import \ + wrap_bias_attr_default from paddle.trainer_config_helpers.layers import layer_support import data_type import activation -import attr - -__all__ = [ - 'parse_network', 'data', 'fc', 'conv_shift', 'img_conv', 'img_pool', 'spp', - 'maxout', 'img_cmrnorm', 'batch_norm', 'sum_to_one_norm', 'recurrent', - 'lstmemory', 'grumemory', 'pool', 'last_seq', 'first_seq', 'concat', - 'seq_concat', 'block_expand', 'expand', 'repeat', 'seq_reshape', 'addto', - 'linear_comb', 'interpolation', 'bilinear_interp', 'power', 'scaling', - 'slope_intercept', 'tensor', 'cos_sim', 'trans', 'max_id', 'sampling_id', - 'pad', 'classification_cost', 'cross_entropy_cost', - 'cross_entropy_with_selfnorm_cost', 'regression_cost', - 'multi_binary_label_cross_entropy_cost', 'rank_cost', 'lambda_cost', - 'sum_cost', 'huber_cost', 'crf', 'crf_decoding', 'ctc', 'warp_ctc', 'nce', - 'hsigmoid', 'eos' -] + +__all__ = ['parse_network', 'data'] __projection_names__ = filter(lambda x: x.endswith('_projection'), dir(conf_helps)) @@ -288,83 +277,51 @@ data = DataLayerV2 AggregateLevel = conf_helps.layers.AggregateLevel ExpandLevel = conf_helps.layers.ExpandLevel -layer_list = [ - # [V2LayerImpl, V1_method_name, parent_names] - # fully connected layers - ['fc', 'fc_layer', ['input']], - # conv layers - ['conv_shift', 'conv_shift_layer', ['a', 'b']], - ['img_conv', 'img_conv_layer', ['input']], - # image pooling layers - ['img_pool', 'img_pool_layer', ['input']], - ['spp', 'spp_layer', ['input']], - ['maxout', 'maxout_layer', ['input']], - # norm layers - ['img_cmrnorm', 'img_cmrnorm_layer', ['input']], - ['batch_norm', 'batch_norm_layer', ['input']], - ['sum_to_one_norm', 'sum_to_one_norm_layer', ['input']], - # recurrent layers - ['recurrent', 'recurrent_layer', ['input']], - ['lstmemory', 'lstmemory', ['input']], - ['grumemory', 'grumemory', ['input']], - # aggregate layers - ['pool', 'pooling_layer', ['input']], - ['last_seq', 'last_seq', ['input']], - ['first_seq', 'first_seq', ['input']], - ['concat', 'concat_layer', ['input']], - ['seq_concat', 'seq_concat_layer', ['a', 'b']], - # reshaping layers - ['block_expand', 'block_expand_layer', ['input']], - ['expand', 'expand_layer', ['input', 'expand_as']], - ['repeat', 'repeat_layer', ['input']], - ['rotate', 'rotate_layer', ['input']], - ['seq_reshape', 'seq_reshape_layer', ['input']], - # math layers - ['addto', 'addto_layer', ['input']], - ['linear_comb', 'linear_comb_layer', ['weights', 'vectors']], - ['interpolation', 'interpolation_layer', ['input', 'weight']], - ['bilinear_interp', 'bilinear_interp_layer', ['input']], - ['power', 'power_layer', ['input', 'weight']], - ['scaling', 'scaling_layer', ['input', 'weight']], - ['slope_intercept', 'slope_intercept_layer', ['input']], - ['tensor', 'tensor_layer', ['a', 'b']], - ['cos_sim', 'cos_sim', ['a', 'b']], - ['trans', 'trans_layer', ['input']], - # sampling layers - ['max_id', 'maxid_layer', ['input']], - ['sampling_id', 'sampling_id_layer', ['input']], - # slicing and joining layers - ['pad', 'pad_layer', ['input']], - # cost layers - [ - 'classification_cost', 'classification_cost', - ['input', 'label', 'weight'] - ], - ['regression_cost', 'regression_cost', ['input', 'label', 'weight']], - ['cross_entropy_cost', 'cross_entropy', ['input', 'label']], - [ - 'cross_entropy_with_selfnorm_cost', 'cross_entropy_with_selfnorm', - ['input', 'label'] - ], - [ - 'multi_binary_label_cross_entropy_cost', - 'multi_binary_label_cross_entropy', ['input', 'label'] - ], - ['rank_cost', 'rank_cost', ['left', 'right', 'label', 'weight']], - ['lambda_cost', 'lambda_cost', ['input', 'score']], - ['sum_cost', 'sum_cost', ['input']], - ['huber_cost', 'huber_cost', ['input', 'label']], - ['crf', 'crf_layer', ['input', 'label']], - ['crf_decoding', 'crf_decoding_layer', ['input']], - ['ctc', 'ctc_layer', ['input', 'label']], - ['warp_ctc', 'warp_ctc_layer', ['input', 'label']], - ['nce', 'nce_layer', ['input', 'label']], - ['hsigmoid', 'hsigmoid', ['input', 'label']], - # check layers - ['eos', 'eos_layer', ['input']] -] -for l in layer_list: - globals()[l[0]] = __convert_to_v2__(l[1], l[2]) + +def __layer_name_mapping__(inname): + if inname in ['data_layer', 'memory', 'mixed_layer']: + # Do Not handle these layers + return + elif inname == 'maxid_layer': + return 'max_id' + elif inname.endswith('memory') or inname.endswith( + '_seq') or inname.endswith('_sim') or inname == 'hsigmoid': + return inname + elif inname in [ + 'cross_entropy', 'multi_binary_label_cross_entropy', + 'cross_entropy_with_selfnorm' + ]: + return inname + "_cost" + elif inname.endswith('_cost'): + return inname + elif inname.endswith("_layer"): + return inname[:-len("_layer")] + + +def __layer_name_mapping_parent_names__(inname): + all_args = getattr(conf_helps, inname).argspec.args + return filter( + lambda x: x in ['input1', 'input2','label', 'input', 'a', 'b', 'expand_as', + 'weights', 'vectors', 'weight', 'score', 'left', 'right'], + all_args) + + +def __convert_layer__(_new_name_, _old_name_, _parent_names_): + global __all__ + __all__.append(_new_name_) + globals()[new_name] = __convert_to_v2__(_old_name_, _parent_names_) + + +for each_layer_name in dir(conf_helps): + new_name = __layer_name_mapping__(each_layer_name) + if new_name is not None: + parent_names = __layer_name_mapping_parent_names__(each_layer_name) + assert len(parent_names) != 0, each_layer_name + __convert_layer__(new_name, each_layer_name, parent_names) + +del parent_names +del new_name +del each_layer_name # convert projection for prj in __projection_names__: diff --git a/python/paddle/v2/tests/test_layer.py b/python/paddle/v2/tests/test_layer.py index bb0099ea2f..b138ddbbe6 100644 --- a/python/paddle/v2/tests/test_layer.py +++ b/python/paddle/v2/tests/test_layer.py @@ -11,17 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import difflib import unittest -import paddle.trainer_config_helpers as conf_helps import paddle.v2.activation as activation import paddle.v2.attr as attr import paddle.v2.data_type as data_type import paddle.v2.layer as layer import paddle.v2.pooling as pooling -from paddle.trainer_config_helpers.config_parser_utils import \ - parse_network_config as parse_network pixel = layer.data(name='pixel', type=data_type.dense_vector(128)) label = layer.data(name='label', type=data_type.integer_value(10)) @@ -70,7 +66,7 @@ class ImageLayerTest(unittest.TestCase): class AggregateLayerTest(unittest.TestCase): def test_aggregate_layer(self): - pool = layer.pool( + pool = layer.pooling( input=pixel, pooling_type=pooling.Avg(), agg_level=layer.AggregateLevel.EACH_SEQUENCE)