Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into convert
dangqingqing
8 years ago
commit
2c2347dff3
@ -0,0 +1,53 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__all__ = ['np_array', 'text_file']
|
||||
|
||||
|
||||
def np_array(x):
|
||||
"""
|
||||
Creates a reader that yields elements of x, if it is a
|
||||
numpy vector. Or rows of x, if it is a numpy matrix.
|
||||
Or any sub-hyperplane indexed by the highest dimension.
|
||||
|
||||
:param x: the numpy array to create reader from.
|
||||
:returns: data reader created from x.
|
||||
"""
|
||||
|
||||
def reader():
|
||||
if x.ndim < 1:
|
||||
yield x
|
||||
|
||||
for e in x:
|
||||
yield e
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
def text_file(path):
|
||||
"""
|
||||
Creates a data reader that outputs text line by line from given text file.
|
||||
Trailing new line ('\n') of each line will be removed.
|
||||
|
||||
:path: path of the text file.
|
||||
:returns: data reader of text file
|
||||
"""
|
||||
|
||||
def reader():
|
||||
f = open(path, "r")
|
||||
for l in f:
|
||||
yield l.rstrip('\n')
|
||||
f.close()
|
||||
|
||||
return reader
|
||||
@ -0,0 +1,38 @@
|
||||
# Copyright PaddlePaddle contributors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import unittest
|
||||
import paddle.reader.creator
|
||||
import numpy as np
|
||||
import os
|
||||
|
||||
|
||||
class TestNumpyArray(unittest.TestCase):
|
||||
def test_numpy_array(self):
|
||||
l = [[1, 2, 3], [4, 5, 6]]
|
||||
x = np.array(l, np.int32)
|
||||
reader = paddle.reader.creator.np_array(x)
|
||||
for idx, e in enumerate(reader()):
|
||||
self.assertItemsEqual(e, l[idx])
|
||||
|
||||
|
||||
class TestTextFile(unittest.TestCase):
|
||||
def test_text_file(self):
|
||||
path = os.path.join(os.path.dirname(__file__), "test_data_creator.txt")
|
||||
reader = paddle.reader.creator.text_file(path)
|
||||
for idx, e in enumerate(reader()):
|
||||
self.assertEqual(e, str(idx * 2) + " " + str(idx * 2 + 1))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,3 @@
|
||||
0 1
|
||||
2 3
|
||||
4 5
|
||||
@ -0,0 +1,8 @@
|
||||
import os
|
||||
|
||||
__all__ = ['DATA_HOME']
|
||||
|
||||
DATA_HOME = os.path.expanduser('~/.cache/paddle_data_set')
|
||||
|
||||
if not os.path.exists(DATA_HOME):
|
||||
os.makedirs(DATA_HOME)
|
||||
@ -0,0 +1,39 @@
|
||||
import sklearn.datasets.mldata
|
||||
import sklearn.model_selection
|
||||
import numpy
|
||||
from config import DATA_HOME
|
||||
|
||||
__all__ = ['train_creator', 'test_creator']
|
||||
|
||||
|
||||
def __mnist_reader_creator__(data, target):
|
||||
def reader():
|
||||
n_samples = data.shape[0]
|
||||
for i in xrange(n_samples):
|
||||
yield (data[i] / 255.0).astype(numpy.float32), int(target[i])
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
TEST_SIZE = 10000
|
||||
|
||||
data = sklearn.datasets.mldata.fetch_mldata(
|
||||
"MNIST original", data_home=DATA_HOME)
|
||||
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
|
||||
data.data, data.target, test_size=TEST_SIZE, random_state=0)
|
||||
|
||||
|
||||
def train_creator():
|
||||
return __mnist_reader_creator__(X_train, y_train)
|
||||
|
||||
|
||||
def test_creator():
|
||||
return __mnist_reader_creator__(X_test, y_test)
|
||||
|
||||
|
||||
def unittest():
|
||||
assert len(list(test_creator()())) == TEST_SIZE
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest()
|
||||
Loading…
Reference in new issue