move python reader api to fluid.io module, test=develop (#19143)
Zeng Jinle
6 years ago
committed by
GitHub
parent
b86be13c15
commit
0f9b33954a
@ -1,96 +0,0 @@
|
|||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
"""
|
|
||||||
Creator package contains some simple reader creator, which could
|
|
||||||
be used in user program.
|
|
||||||
"""
|
|
||||||
|
|
||||||
__all__ = ['np_array', 'text_file', 'recordio']
|
|
||||||
|
|
||||||
|
|
||||||
def np_array(x):
|
|
||||||
"""
|
|
||||||
Creates a reader that yields elements of x, if it is a
|
|
||||||
numpy vector. Or rows of x, if it is a numpy matrix.
|
|
||||||
Or any sub-hyperplane indexed by the highest dimension.
|
|
||||||
|
|
||||||
:param x: the numpy array to create reader from.
|
|
||||||
:returns: data reader created from x.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def reader():
|
|
||||||
if x.ndim < 1:
|
|
||||||
yield x
|
|
||||||
|
|
||||||
for e in x:
|
|
||||||
yield e
|
|
||||||
|
|
||||||
return reader
|
|
||||||
|
|
||||||
|
|
||||||
def text_file(path):
|
|
||||||
"""
|
|
||||||
Creates a data reader that outputs text line by line from given text file.
|
|
||||||
Trailing new line ('\\\\n') of each line will be removed.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
path (str): path of the text file.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
callable: data reader of text file.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def reader():
|
|
||||||
f = open(path, "r")
|
|
||||||
for l in f:
|
|
||||||
yield l.rstrip('\n')
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
return reader
|
|
||||||
|
|
||||||
|
|
||||||
def recordio(paths, buf_size=100):
|
|
||||||
"""
|
|
||||||
Creates a data reader from given RecordIO file paths separated
|
|
||||||
by ",", glob pattern is supported.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
paths (str|list(str)): path of recordio files.
|
|
||||||
buf_size (int): prefetched buffer size.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
callable: data reader of recordio files.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import recordio as rec
|
|
||||||
import paddle.reader.decorator as dec
|
|
||||||
import six
|
|
||||||
import six.moves.cPickle as pickle
|
|
||||||
|
|
||||||
def reader():
|
|
||||||
if isinstance(paths, six.string_types):
|
|
||||||
path = paths
|
|
||||||
elif isinstance(paths, six.binary_type):
|
|
||||||
path = paths.decode()
|
|
||||||
else:
|
|
||||||
path = ",".join(paths)
|
|
||||||
f = rec.reader(path)
|
|
||||||
while True:
|
|
||||||
r = f.read()
|
|
||||||
if r is None:
|
|
||||||
break
|
|
||||||
yield pickle.loads(r)
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
return dec.buffered(reader, buf_size)
|
|
||||||
@ -1,2 +1 @@
|
|||||||
py_test(creator_test SRCS creator_test.py)
|
|
||||||
py_test(decorator_test SRCS decorator_test.py)
|
py_test(decorator_test SRCS decorator_test.py)
|
||||||
|
|||||||
@ -1,75 +0,0 @@
|
|||||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
# Copyright PaddlePaddle contributors. All Rights Reservedd
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
import os
|
|
||||||
import unittest
|
|
||||||
import numpy as np
|
|
||||||
import paddle.reader.creator
|
|
||||||
import six
|
|
||||||
|
|
||||||
|
|
||||||
class TestNumpyArray(unittest.TestCase):
|
|
||||||
def test_numpy_array(self):
|
|
||||||
l = [[1, 2, 3], [4, 5, 6]]
|
|
||||||
x = np.array(l, np.int32)
|
|
||||||
reader = paddle.reader.creator.np_array(x)
|
|
||||||
for idx, e in enumerate(reader()):
|
|
||||||
six.assertCountEqual(self, e, l[idx])
|
|
||||||
|
|
||||||
|
|
||||||
class TestTextFile(unittest.TestCase):
|
|
||||||
def test_text_file(self):
|
|
||||||
path = os.path.join(os.path.dirname(__file__), "test_data_creator.txt")
|
|
||||||
reader = paddle.reader.creator.text_file(path)
|
|
||||||
for idx, e in enumerate(reader()):
|
|
||||||
self.assertEqual(e, str(idx * 2) + " " + str(idx * 2 + 1))
|
|
||||||
|
|
||||||
|
|
||||||
class TestRecordIO(unittest.TestCase):
|
|
||||||
def do_test(self, path):
|
|
||||||
reader = paddle.reader.creator.recordio(path)
|
|
||||||
idx = 0
|
|
||||||
for e in reader():
|
|
||||||
if idx == 0:
|
|
||||||
self.assertEqual(e, (1, 2, 3))
|
|
||||||
elif idx == 1:
|
|
||||||
self.assertEqual(e, (4, 5, 6))
|
|
||||||
idx += 1
|
|
||||||
self.assertEqual(idx, 2)
|
|
||||||
|
|
||||||
def test_recordIO(self):
|
|
||||||
self.do_test(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__), "test_reader_recordio.dat"))
|
|
||||||
self.do_test([
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__), "test_reader_recordio.dat")
|
|
||||||
])
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
||||||
Loading…
Reference in new issue