|
|
|
@ -36,6 +36,45 @@ def convert_reader_to_recordio_file(
|
|
|
|
|
compressor=core.RecordIOWriter.Compressor.Snappy,
|
|
|
|
|
max_num_records=1000,
|
|
|
|
|
feed_order=None):
|
|
|
|
|
"""
|
|
|
|
|
Convert a Python Reader to a recordio file.
|
|
|
|
|
|
|
|
|
|
Please see :ref:`api_guide_python_reader` and :ref:`api_guide_reader_op` for
|
|
|
|
|
details.
|
|
|
|
|
|
|
|
|
|
Examples:
|
|
|
|
|
|
|
|
|
|
>>> import paddle.fluid as fluid
|
|
|
|
|
>>> import paddle.dataset.mnist as mnist
|
|
|
|
|
>>> import paddle
|
|
|
|
|
>>>
|
|
|
|
|
>>> tmp_program = fluid.Program()
|
|
|
|
|
>>> with fluid.program_guard(tmp_program):
|
|
|
|
|
>>> img = fluid.layers.data(name='img', shape=[784])
|
|
|
|
|
>>> label = fluid.layers.data(name='label', shape=[1], dtype='int64')
|
|
|
|
|
>>> feeder = fluid.DataFeeder(feed_list=[img, label], place=fluid.CPUPlace())
|
|
|
|
|
>>> # mnist.recordio will be generated in current directory
|
|
|
|
|
>>> fluid.recordio_writer.convert_reader_to_recordio_file(
|
|
|
|
|
>>> filename="mnist.recordio",
|
|
|
|
|
>>> reader_creator=paddle.batch(mnist.train(), batch_size=32),
|
|
|
|
|
>>> feeder=feeder)
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
filename(str): The recordio filename.
|
|
|
|
|
reader_creator(callable): The Python Reader Creator. See
|
|
|
|
|
:ref:`api_guide_python_reader`.
|
|
|
|
|
feeder(DataFeeder): The DataFeeder instance. Used to convert
|
|
|
|
|
:code:`reader_creator` to :code: `lod_tensor`
|
|
|
|
|
compressor: Must in fluid.core.RecordIOWriter.Compressor.Snappy or
|
|
|
|
|
fluid.core.RecordIOWriter.Compressor.NoCompress. Use :code:`Snappy`
|
|
|
|
|
by default.
|
|
|
|
|
max_num_records(int): Maximum number of records in one chuck. Each record
|
|
|
|
|
is each return value from reader function
|
|
|
|
|
feed_order(list): The order of variable names that the reader returns
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
int: the number of record that saved.
|
|
|
|
|
"""
|
|
|
|
|
if feed_order is None:
|
|
|
|
|
feed_order = feeder.feed_names
|
|
|
|
|
counter = 0
|
|
|
|
@ -58,6 +97,13 @@ def convert_reader_to_recordio_files(
|
|
|
|
|
compressor=core.RecordIOWriter.Compressor.Snappy,
|
|
|
|
|
max_num_records=1000,
|
|
|
|
|
feed_order=None):
|
|
|
|
|
"""
|
|
|
|
|
convert a python reader to many recordio files.
|
|
|
|
|
|
|
|
|
|
This API is basically same as :code:`convert_reader_to_recordio_file`,
|
|
|
|
|
instead of it will create many recordio files. Each file contains at
|
|
|
|
|
most :code:`batch_per_file` records.
|
|
|
|
|
"""
|
|
|
|
|
if feed_order is None:
|
|
|
|
|
feed_order = feeder.feed_names
|
|
|
|
|
f_name, f_ext = os.path.splitext(filename)
|
|
|
|
|