move python reader api to fluid.io module, test=develop (#19143)
Zeng Jinle
6 years ago
committed by
GitHub
parent
b86be13c15
commit
0f9b33954a
@ -1,96 +0,0 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Creator package contains some simple reader creator, which could
|
||||
be used in user program.
|
||||
"""
|
||||
|
||||
__all__ = ['np_array', 'text_file', 'recordio']
|
||||
|
||||
|
||||
def np_array(x):
|
||||
"""
|
||||
Creates a reader that yields elements of x, if it is a
|
||||
numpy vector. Or rows of x, if it is a numpy matrix.
|
||||
Or any sub-hyperplane indexed by the highest dimension.
|
||||
|
||||
:param x: the numpy array to create reader from.
|
||||
:returns: data reader created from x.
|
||||
"""
|
||||
|
||||
def reader():
|
||||
if x.ndim < 1:
|
||||
yield x
|
||||
|
||||
for e in x:
|
||||
yield e
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
def text_file(path):
|
||||
"""
|
||||
Creates a data reader that outputs text line by line from given text file.
|
||||
Trailing new line ('\\\\n') of each line will be removed.
|
||||
|
||||
Args:
|
||||
path (str): path of the text file.
|
||||
|
||||
Returns:
|
||||
callable: data reader of text file.
|
||||
"""
|
||||
|
||||
def reader():
|
||||
f = open(path, "r")
|
||||
for l in f:
|
||||
yield l.rstrip('\n')
|
||||
f.close()
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
def recordio(paths, buf_size=100):
|
||||
"""
|
||||
Creates a data reader from given RecordIO file paths separated
|
||||
by ",", glob pattern is supported.
|
||||
|
||||
Args:
|
||||
paths (str|list(str)): path of recordio files.
|
||||
buf_size (int): prefetched buffer size.
|
||||
|
||||
Returns:
|
||||
callable: data reader of recordio files.
|
||||
"""
|
||||
|
||||
import recordio as rec
|
||||
import paddle.reader.decorator as dec
|
||||
import six
|
||||
import six.moves.cPickle as pickle
|
||||
|
||||
def reader():
|
||||
if isinstance(paths, six.string_types):
|
||||
path = paths
|
||||
elif isinstance(paths, six.binary_type):
|
||||
path = paths.decode()
|
||||
else:
|
||||
path = ",".join(paths)
|
||||
f = rec.reader(path)
|
||||
while True:
|
||||
r = f.read()
|
||||
if r is None:
|
||||
break
|
||||
yield pickle.loads(r)
|
||||
f.close()
|
||||
|
||||
return dec.buffered(reader, buf_size)
|
||||
@ -1,2 +1 @@
|
||||
py_test(creator_test SRCS creator_test.py)
|
||||
py_test(decorator_test SRCS decorator_test.py)
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copyright PaddlePaddle contributors. All Rights Reservedd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import unittest
|
||||
import numpy as np
|
||||
import paddle.reader.creator
|
||||
import six
|
||||
|
||||
|
||||
class TestNumpyArray(unittest.TestCase):
|
||||
def test_numpy_array(self):
|
||||
l = [[1, 2, 3], [4, 5, 6]]
|
||||
x = np.array(l, np.int32)
|
||||
reader = paddle.reader.creator.np_array(x)
|
||||
for idx, e in enumerate(reader()):
|
||||
six.assertCountEqual(self, e, l[idx])
|
||||
|
||||
|
||||
class TestTextFile(unittest.TestCase):
|
||||
def test_text_file(self):
|
||||
path = os.path.join(os.path.dirname(__file__), "test_data_creator.txt")
|
||||
reader = paddle.reader.creator.text_file(path)
|
||||
for idx, e in enumerate(reader()):
|
||||
self.assertEqual(e, str(idx * 2) + " " + str(idx * 2 + 1))
|
||||
|
||||
|
||||
class TestRecordIO(unittest.TestCase):
|
||||
def do_test(self, path):
|
||||
reader = paddle.reader.creator.recordio(path)
|
||||
idx = 0
|
||||
for e in reader():
|
||||
if idx == 0:
|
||||
self.assertEqual(e, (1, 2, 3))
|
||||
elif idx == 1:
|
||||
self.assertEqual(e, (4, 5, 6))
|
||||
idx += 1
|
||||
self.assertEqual(idx, 2)
|
||||
|
||||
def test_recordIO(self):
|
||||
self.do_test(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__), "test_reader_recordio.dat"))
|
||||
self.do_test([
|
||||
os.path.join(
|
||||
os.path.dirname(__file__), "test_reader_recordio.dat")
|
||||
])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue