Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into rnn
qiaolongfei
8 years ago
commit
6b1a91f9b8
@ -0,0 +1,82 @@
|
||||
"""
|
||||
CIFAR Dataset.
|
||||
|
||||
URL: https://www.cs.toronto.edu/~kriz/cifar.html
|
||||
|
||||
the default train_creator, test_creator used for CIFAR-10 dataset.
|
||||
"""
|
||||
import cPickle
|
||||
import itertools
|
||||
import tarfile
|
||||
|
||||
import numpy
|
||||
|
||||
from config import download
|
||||
|
||||
__all__ = [
|
||||
'cifar_100_train_creator', 'cifar_100_test_creator', 'train_creator',
|
||||
'test_creator'
|
||||
]
|
||||
|
||||
CIFAR10_URL = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
|
||||
CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a'
|
||||
CIFAR100_URL = 'https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
|
||||
CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85'
|
||||
|
||||
|
||||
def __read_batch__(filename, sub_name):
|
||||
def reader():
|
||||
def __read_one_batch_impl__(batch):
|
||||
data = batch['data']
|
||||
labels = batch.get('labels', batch.get('fine_labels', None))
|
||||
assert labels is not None
|
||||
for sample, label in itertools.izip(data, labels):
|
||||
yield (sample / 255.0).astype(numpy.float32), int(label)
|
||||
|
||||
with tarfile.open(filename, mode='r') as f:
|
||||
names = (each_item.name for each_item in f
|
||||
if sub_name in each_item.name)
|
||||
|
||||
for name in names:
|
||||
batch = cPickle.load(f.extractfile(name))
|
||||
for item in __read_one_batch_impl__(batch):
|
||||
yield item
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
def cifar_100_train_creator():
|
||||
fn = download(url=CIFAR100_URL, md5=CIFAR100_MD5)
|
||||
return __read_batch__(fn, 'train')
|
||||
|
||||
|
||||
def cifar_100_test_creator():
|
||||
fn = download(url=CIFAR100_URL, md5=CIFAR100_MD5)
|
||||
return __read_batch__(fn, 'test')
|
||||
|
||||
|
||||
def train_creator():
|
||||
"""
|
||||
Default train reader creator. Use CIFAR-10 dataset.
|
||||
"""
|
||||
fn = download(url=CIFAR10_URL, md5=CIFAR10_MD5)
|
||||
return __read_batch__(fn, 'data_batch')
|
||||
|
||||
|
||||
def test_creator():
|
||||
"""
|
||||
Default test reader creator. Use CIFAR-10 dataset.
|
||||
"""
|
||||
fn = download(url=CIFAR10_URL, md5=CIFAR10_MD5)
|
||||
return __read_batch__(fn, 'test_batch')
|
||||
|
||||
|
||||
def unittest():
|
||||
for _ in train_creator()():
|
||||
pass
|
||||
for _ in test_creator()():
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest()
|
||||
@ -1,8 +1,36 @@
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import urllib2
|
||||
|
||||
__all__ = ['DATA_HOME']
|
||||
__all__ = ['DATA_HOME', 'download']
|
||||
|
||||
DATA_HOME = os.path.expanduser('~/.cache/paddle_data_set')
|
||||
|
||||
if not os.path.exists(DATA_HOME):
|
||||
os.makedirs(DATA_HOME)
|
||||
|
||||
|
||||
def download(url, md5):
|
||||
filename = os.path.split(url)[-1]
|
||||
assert DATA_HOME is not None
|
||||
filepath = os.path.join(DATA_HOME, md5)
|
||||
if not os.path.exists(filepath):
|
||||
os.makedirs(filepath)
|
||||
__full_file__ = os.path.join(filepath, filename)
|
||||
|
||||
def __file_ok__():
|
||||
if not os.path.exists(__full_file__):
|
||||
return False
|
||||
md5_hash = hashlib.md5()
|
||||
with open(__full_file__, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
md5_hash.update(chunk)
|
||||
|
||||
return md5_hash.hexdigest() == md5
|
||||
|
||||
while not __file_ok__():
|
||||
response = urllib2.urlopen(url)
|
||||
with open(__full_file__, mode='wb') as of:
|
||||
shutil.copyfileobj(fsrc=response, fdst=of)
|
||||
return __full_file__
|
||||
|
||||
@ -0,0 +1,120 @@
|
||||
import zipfile
|
||||
from config import download
|
||||
import re
|
||||
import random
|
||||
import functools
|
||||
|
||||
__all__ = ['train_creator', 'test_creator']
|
||||
|
||||
|
||||
class MovieInfo(object):
|
||||
def __init__(self, index, categories, title):
|
||||
self.index = int(index)
|
||||
self.categories = categories
|
||||
self.title = title
|
||||
|
||||
def value(self):
|
||||
return [
|
||||
self.index, [CATEGORIES_DICT[c] for c in self.categories],
|
||||
[MOVIE_TITLE_DICT[w.lower()] for w in self.title.split()]
|
||||
]
|
||||
|
||||
|
||||
class UserInfo(object):
|
||||
def __init__(self, index, gender, age, job_id):
|
||||
self.index = int(index)
|
||||
self.is_male = gender == 'M'
|
||||
self.age = [1, 18, 25, 35, 45, 50, 56].index(int(age))
|
||||
self.job_id = int(job_id)
|
||||
|
||||
def value(self):
|
||||
return [self.index, 0 if self.is_male else 1, self.age, self.job_id]
|
||||
|
||||
|
||||
MOVIE_INFO = None
|
||||
MOVIE_TITLE_DICT = None
|
||||
CATEGORIES_DICT = None
|
||||
USER_INFO = None
|
||||
|
||||
|
||||
def __initialize_meta_info__():
|
||||
fn = download(
|
||||
url='http://files.grouplens.org/datasets/movielens/ml-1m.zip',
|
||||
md5='c4d9eecfca2ab87c1945afe126590906')
|
||||
global MOVIE_INFO
|
||||
if MOVIE_INFO is None:
|
||||
pattern = re.compile(r'^(.*)\((\d+)\)$')
|
||||
with zipfile.ZipFile(file=fn) as package:
|
||||
for info in package.infolist():
|
||||
assert isinstance(info, zipfile.ZipInfo)
|
||||
MOVIE_INFO = dict()
|
||||
title_word_set = set()
|
||||
categories_set = set()
|
||||
with package.open('ml-1m/movies.dat') as movie_file:
|
||||
for i, line in enumerate(movie_file):
|
||||
movie_id, title, categories = line.strip().split('::')
|
||||
categories = categories.split('|')
|
||||
for c in categories:
|
||||
categories_set.add(c)
|
||||
title = pattern.match(title).group(1)
|
||||
MOVIE_INFO[int(movie_id)] = MovieInfo(
|
||||
index=movie_id, categories=categories, title=title)
|
||||
for w in title.split():
|
||||
title_word_set.add(w.lower())
|
||||
|
||||
global MOVIE_TITLE_DICT
|
||||
MOVIE_TITLE_DICT = dict()
|
||||
for i, w in enumerate(title_word_set):
|
||||
MOVIE_TITLE_DICT[w] = i
|
||||
|
||||
global CATEGORIES_DICT
|
||||
CATEGORIES_DICT = dict()
|
||||
for i, c in enumerate(categories_set):
|
||||
CATEGORIES_DICT[c] = i
|
||||
|
||||
global USER_INFO
|
||||
USER_INFO = dict()
|
||||
with package.open('ml-1m/users.dat') as user_file:
|
||||
for line in user_file:
|
||||
uid, gender, age, job, _ = line.strip().split("::")
|
||||
USER_INFO[int(uid)] = UserInfo(
|
||||
index=uid, gender=gender, age=age, job_id=job)
|
||||
return fn
|
||||
|
||||
|
||||
def __reader__(rand_seed=0, test_ratio=0.1, is_test=False):
|
||||
fn = __initialize_meta_info__()
|
||||
rand = random.Random(x=rand_seed)
|
||||
with zipfile.ZipFile(file=fn) as package:
|
||||
with package.open('ml-1m/ratings.dat') as rating:
|
||||
for line in rating:
|
||||
if (rand.random() < test_ratio) == is_test:
|
||||
uid, mov_id, rating, _ = line.strip().split("::")
|
||||
uid = int(uid)
|
||||
mov_id = int(mov_id)
|
||||
rating = float(rating) * 2 - 5.0
|
||||
|
||||
mov = MOVIE_INFO[mov_id]
|
||||
usr = USER_INFO[uid]
|
||||
yield usr.value() + mov.value() + [[rating]]
|
||||
|
||||
|
||||
def __reader_creator__(**kwargs):
|
||||
return lambda: __reader__(**kwargs)
|
||||
|
||||
|
||||
train_creator = functools.partial(__reader_creator__, is_test=False)
|
||||
test_creator = functools.partial(__reader_creator__, is_test=True)
|
||||
|
||||
|
||||
def unittest():
|
||||
for train_count, _ in enumerate(train_creator()()):
|
||||
pass
|
||||
for test_count, _ in enumerate(test_creator()()):
|
||||
pass
|
||||
|
||||
print train_count, test_count
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest()
|
||||
@ -0,0 +1,83 @@
|
||||
# Copyright PaddlePaddle contributors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import unittest
|
||||
import paddle.v2.layer as layer
|
||||
import paddle.v2.topology as topology
|
||||
import paddle.v2.data_type as data_type
|
||||
import paddle.trainer_config_helpers as conf_helps
|
||||
|
||||
|
||||
class TestTopology(unittest.TestCase):
|
||||
def test_data_type(self):
|
||||
pixel = layer.data(name='pixel', type=data_type.dense_vector(784))
|
||||
label = layer.data(name='label', type=data_type.integer_value(10))
|
||||
hidden = layer.fc(input=pixel,
|
||||
size=100,
|
||||
act=conf_helps.SigmoidActivation())
|
||||
inference = layer.fc(input=hidden,
|
||||
size=10,
|
||||
act=conf_helps.SoftmaxActivation())
|
||||
cost = layer.classification_cost(input=inference, label=label)
|
||||
topo = topology.Topology(cost)
|
||||
data_types = topo.data_type()
|
||||
self.assertEqual(len(data_types), 2)
|
||||
pixel_data_type = filter(lambda type: type[0] == "pixel", data_types)
|
||||
self.assertEqual(len(pixel_data_type), 1)
|
||||
pixel_data_type = pixel_data_type[0]
|
||||
self.assertEqual(pixel_data_type[1].type, data_type.DataType.Dense)
|
||||
self.assertEqual(pixel_data_type[1].dim, 784)
|
||||
|
||||
label_data_type = filter(lambda type: type[0] == "label", data_types)
|
||||
self.assertEqual(len(label_data_type), 1)
|
||||
label_data_type = label_data_type[0]
|
||||
self.assertEqual(label_data_type[1].type, data_type.DataType.Index)
|
||||
self.assertEqual(label_data_type[1].dim, 10)
|
||||
|
||||
def test_get_layer(self):
|
||||
pixel = layer.data(name='pixel', type=data_type.dense_vector(784))
|
||||
label = layer.data(name='label', type=data_type.integer_value(10))
|
||||
hidden = layer.fc(input=pixel,
|
||||
size=100,
|
||||
act=conf_helps.SigmoidActivation())
|
||||
inference = layer.fc(input=hidden,
|
||||
size=10,
|
||||
act=conf_helps.SoftmaxActivation())
|
||||
cost = layer.classification_cost(input=inference, label=label)
|
||||
topo = topology.Topology(cost)
|
||||
pixel_layer = topo.get_layer("pixel")
|
||||
label_layer = topo.get_layer("label")
|
||||
self.assertEqual(pixel_layer, pixel)
|
||||
self.assertEqual(label_layer, label)
|
||||
|
||||
def test_parse(self):
|
||||
pixel = layer.data(name='pixel', type=data_type.dense_vector(784))
|
||||
label = layer.data(name='label', type=data_type.integer_value(10))
|
||||
hidden = layer.fc(input=pixel,
|
||||
size=100,
|
||||
act=conf_helps.SigmoidActivation())
|
||||
inference = layer.fc(input=hidden,
|
||||
size=10,
|
||||
act=conf_helps.SoftmaxActivation())
|
||||
maxid = layer.max_id(input=inference)
|
||||
cost1 = layer.classification_cost(input=inference, label=label)
|
||||
cost2 = layer.cross_entropy_cost(input=inference, label=label)
|
||||
|
||||
topology.Topology(cost2).proto()
|
||||
topology.Topology([cost1]).proto()
|
||||
topology.Topology([cost1, cost2]).proto()
|
||||
topology.Topology([inference, maxid]).proto()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,95 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import collections
|
||||
|
||||
from paddle.proto.ModelConfig_pb2 import ModelConfig
|
||||
|
||||
import layer as v2_layer
|
||||
|
||||
__all__ = ['Topology']
|
||||
|
||||
|
||||
class Topology(object):
|
||||
"""
|
||||
Topology is used to store the information about all layers
|
||||
and network configs.
|
||||
"""
|
||||
|
||||
def __init__(self, layers):
|
||||
if not isinstance(layers, collections.Sequence):
|
||||
__check_layer_type__(layers)
|
||||
layers = [layers]
|
||||
for layer in layers:
|
||||
__check_layer_type__(layer)
|
||||
self.layers = layers
|
||||
self.__model_config__ = v2_layer.parse_network(*layers)
|
||||
assert isinstance(self.__model_config__, ModelConfig)
|
||||
|
||||
def proto(self):
|
||||
return self.__model_config__
|
||||
|
||||
def get_layer(self, name):
|
||||
"""
|
||||
get v2.Layer Class instance by layer name
|
||||
:param name:
|
||||
:return:
|
||||
"""
|
||||
result_layer = []
|
||||
|
||||
def find_layer_by_name(layer, layer_name):
|
||||
if len(result_layer) == 1:
|
||||
return
|
||||
elif layer.name == layer_name:
|
||||
result_layer.append(layer)
|
||||
else:
|
||||
for parent_layer in layer.__parent_layers__.values():
|
||||
find_layer_by_name(parent_layer, layer_name)
|
||||
|
||||
for layer in self.layers:
|
||||
find_layer_by_name(layer, name)
|
||||
|
||||
assert len(result_layer) == 1
|
||||
return result_layer[0]
|
||||
|
||||
def data_layers(self):
|
||||
"""
|
||||
get all data layer
|
||||
:return:
|
||||
"""
|
||||
data_layers = set()
|
||||
|
||||
def find_data_layer(layer):
|
||||
if isinstance(layer, v2_layer.DataLayerV2):
|
||||
data_layers.add(layer)
|
||||
for parent_layer in layer.__parent_layers__.values():
|
||||
find_data_layer(parent_layer)
|
||||
|
||||
for layer in self.layers:
|
||||
find_data_layer(layer)
|
||||
|
||||
return data_layers
|
||||
|
||||
def data_type(self):
|
||||
"""
|
||||
get data_type from proto, such as:
|
||||
[('image', dense_vector(768)), ('label', integer_value(10))]
|
||||
"""
|
||||
return [(data_layer.name, data_layer.type)
|
||||
for data_layer in self.data_layers()]
|
||||
|
||||
|
||||
def __check_layer_type__(layer):
|
||||
if not isinstance(layer, v2_layer.LayerV2):
|
||||
raise ValueError('layer should have type paddle.layer.Layer')
|
||||
Loading…
Reference in new issue