m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Port utils to Python3

Browse Source
revert-12469-sum_op_dim_fix
minqiyang 7 years ago
parent 6988aea989
commit 812de6e806
11 changed files with 66 additions and 53 deletions
Show Stats Download Patch File Download Diff File

4
python/paddle/utils/dump_config.py
Unescape View File

@ -37,9 +37,9 @@ if __name__ == '__main__':
assert isinstance(conf, TrainerConfig_pb2.TrainerConfig) assert isinstance(conf, TrainerConfig_pb2.TrainerConfig)
if whole_conf: if whole_conf:
print conf print(conf)
else: else:
if binary: if binary:
sys.stdout.write(conf.model_config.SerializeToString()) sys.stdout.write(conf.model_config.SerializeToString())
else: else:
print conf.model_config print(conf.model_config)

16
python/paddle/utils/image_multiproc.py
Unescape View File

@ -15,7 +15,8 @@
import os, sys import os, sys
import numpy as np import numpy as np
from PIL import Image from PIL import Image
from cStringIO import StringIO import six
from six.moves import cStringIO as StringIO
import multiprocessing import multiprocessing
import functools import functools
import itertools import itertools
@ -187,7 +188,8 @@ class PILTransformer(ImageTransformer):
return self.transform(im) return self.transform(im)
def job(is_img_string, transformer, (data, label)): def job(is_img_string, transformer, data_label_pack):
(data, label) = data_label_pack
if is_img_string: if is_img_string:
return transformer.transform_from_string(data), label return transformer.transform_from_string(data), label
else: else:
@ -208,7 +210,7 @@ class MultiProcessImageTransformer(object):
""" """
Processing image with multi-process. If it is used in PyDataProvider, Processing image with multi-process. If it is used in PyDataProvider,
the simple usage for CNN is as follows: the simple usage for CNN is as follows:
.. code-block:: python .. code-block:: python
def hool(settings, is_train, **kwargs): def hool(settings, is_train, **kwargs):
@ -229,7 +231,7 @@ class MultiProcessImageTransformer(object):
@provider(init_hook=hook, pool_size=20480) @provider(init_hook=hook, pool_size=20480)
def process(settings, file_list): def process(settings, file_list):
with open(file_list, 'r') as fdata: with open(file_list, 'r') as fdata:
for line in fdata: for line in fdata:
data_dic = np.load(line.strip()) # load the data batch pickled by Pickle. data_dic = np.load(line.strip()) # load the data batch pickled by Pickle.
data = data_dic['data'] data = data_dic['data']
labels = data_dic['label'] labels = data_dic['label']
@ -249,10 +251,10 @@ class MultiProcessImageTransformer(object):
:type channel_swap: tuple or list :type channel_swap: tuple or list
:param mean: the mean values of image, per-channel mean or element-wise mean. :param mean: the mean values of image, per-channel mean or element-wise mean.
:type mean: array, The dimension is 1 for per-channel mean. :type mean: array, The dimension is 1 for per-channel mean.
The dimension is 3 for element-wise mean. The dimension is 3 for element-wise mean.
:param is_train: training peroid or testing peroid. :param is_train: training peroid or testing peroid.
:type is_train: bool. :type is_train: bool.
:param is_color: the image is color or gray. :param is_color: the image is color or gray.
:type is_color: bool. :type is_color: bool.
:param is_img_string: The input can be the file name of image or image string. :param is_img_string: The input can be the file name of image or image string.
:type is_img_string: bool. :type is_img_string: bool.
@ -273,4 +275,4 @@ class MultiProcessImageTransformer(object):
def run(self, data, label): def run(self, data, label):
fun = functools.partial(job, self.is_img_string, self.transformer) fun = functools.partial(job, self.is_img_string, self.transformer)
return self.pool.imap_unordered( return self.pool.imap_unordered(
fun, itertools.izip(data, label), chunksize=100 * self.procnum) fun, six.moves.zip(data, label), chunksize=100 * self.procnum)

8
python/paddle/utils/image_util.py
Unescape View File

@ -14,7 +14,7 @@
import numpy as np import numpy as np
from PIL import Image from PIL import Image
from cStringIO import StringIO from six.moves import cStringIO as StringIO
def resize_image(img, target_size): def resize_image(img, target_size):
@ -34,7 +34,7 @@ def flip(im):
""" """
Return the flipped image. Return the flipped image.
Flip an image along the horizontal direction. Flip an image along the horizontal direction.
im: input image, (H x W x K) ndarrays im: input image, (H x W x K) ndarrays
""" """
if len(im.shape) == 3: if len(im.shape) == 3:
return im[:, :, ::-1] return im[:, :, ::-1]
@ -132,7 +132,7 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True):
def load_image(img_path, is_color=True): def load_image(img_path, is_color=True):
""" """
Load image and return. Load image and return.
img_path: image path. img_path: image path.
is_color: is color image or not. is_color: is color image or not.
""" """
@ -205,7 +205,7 @@ class ImageTransformer:
def set_mean(self, mean): def set_mean(self, mean):
if mean is not None: if mean is not None:
# mean value, may be one value per channel # mean value, may be one value per channel
if mean.ndim == 1: if mean.ndim == 1:
mean = mean[:, np.newaxis, np.newaxis] mean = mean[:, np.newaxis, np.newaxis]
else: else:

45
python/paddle/utils/make_model_diagram.py
Unescape View File

@ -15,6 +15,9 @@
# Generate dot diagram file for the given paddle model config # Generate dot diagram file for the given paddle model config
# The generated file can be viewed using Graphviz (http://graphviz.org) # The generated file can be viewed using Graphviz (http://graphviz.org)
from __future__ import print_function
import six
import sys import sys
import traceback import traceback
@ -61,9 +64,9 @@ def make_diagram_from_proto(model_config, dot_file):
name2id[mem.link_name]) name2id[mem.link_name])
return s return s
print >> f, 'digraph graphname {' print('digraph graphname {', file=f)
print >> f, 'node [width=0.375,height=0.25];' print('node [width=0.375,height=0.25];', file=f)
for i in xrange(len(model_config.layers)): for i in six.moves.xrange(len(model_config.layers)):
l = model_config.layers[i] l = model_config.layers[i]
name2id[l.name] = i name2id[l.name] = i
@ -71,12 +74,12 @@ def make_diagram_from_proto(model_config, dot_file):
for sub_model in model_config.sub_models: for sub_model in model_config.sub_models:
if sub_model.name == 'root': if sub_model.name == 'root':
continue continue
print >> f, 'subgraph cluster_%s {' % i print('subgraph cluster_%s {' % i, file=f)
print >> f, 'style=dashed;' print('style=dashed;', file=f)
label = '%s ' % sub_model.name label = '%s ' % sub_model.name
if sub_model.reversed: if sub_model.reversed:
label += '<==' label += '<=='
print >> f, 'label = "%s";' % label print('label = "%s";' % label, file=f)
i += 1 i += 1
submodel_layers.add(sub_model.name) submodel_layers.add(sub_model.name)
for layer_name in sub_model.layer_names: for layer_name in sub_model.layer_names:
@ -84,37 +87,41 @@ def make_diagram_from_proto(model_config, dot_file):
lid = name2id[layer_name] lid = name2id[layer_name]
layer_config = model_config.layers[lid] layer_config = model_config.layers[lid]
label = make_layer_label(layer_config) label = make_layer_label(layer_config)
print >> f, 'l%s [label="%s", shape=box];' % (lid, label) print('l%s [label="%s", shape=box];' % (lid, label), file=f)
print >> f, '}' print('}', file=f)
for i in xrange(len(model_config.layers)): for i in six.moves.xrange(len(model_config.layers)):
l = model_config.layers[i] l = model_config.layers[i]
if l.name not in submodel_layers: if l.name not in submodel_layers:
label = make_layer_label(l) label = make_layer_label(l)
print >> f, 'l%s [label="%s", shape=box];' % (i, label) print('l%s [label="%s", shape=box];' % (i, label), file=f)
for sub_model in model_config.sub_models: for sub_model in model_config.sub_models:
if sub_model.name == 'root': if sub_model.name == 'root':
continue continue
for link in sub_model.in_links: for link in sub_model.in_links:
print >> f, make_link(link) print(make_link(link), file=f)
for link in sub_model.out_links: for link in sub_model.out_links:
print >> f, make_link(link) print(make_link(link), file=f)
for mem in sub_model.memories: for mem in sub_model.memories:
print >> f, make_mem(mem) print(make_mem(mem), file=f)
for i in xrange(len(model_config.layers)): for i in six.moves.xrange(len(model_config.layers)):
for l in model_config.layers[i].inputs: for l in model_config.layers[i].inputs:
print >> f, 'l%s -> l%s [label="%s"];' % ( print(
name2id[l.input_layer_name], i, l.input_parameter_name) 'l%s -> l%s [label="%s"];' % (name2id[l.input_layer_name], i,
l.input_parameter_name),
file=f)
print >> f, '}' print('}', file=f)
f.close() f.close()
def usage(): def usage():
print >> sys.stderr, ("Usage: python show_model_diagram.py" + print(
" CONFIG_FILE DOT_FILE [config_str]") ("Usage: python show_model_diagram.py" +
" CONFIG_FILE DOT_FILE [config_str]"),
file=sys.stderr)
exit(1) exit(1)

2
python/paddle/utils/merge_model.py
Unescape View File

@ -70,4 +70,4 @@ def merge_v2_model(net, param_file, output_file):
for pname in param_names: for pname in param_names:
params.serialize(pname, f) params.serialize(pname, f)
print 'Generate %s success!' % (output_file) print('Generate %s success!' % (output_file))

3
python/paddle/utils/plotcurve.py
Unescape View File

@ -44,6 +44,7 @@ To use this script to generate plot for AvgCost, error:
python plotcurve.py -i paddle.INFO -o figure.png AvgCost error python plotcurve.py -i paddle.INFO -o figure.png AvgCost error
""" """
import six
import sys import sys
import matplotlib import matplotlib
# the following line is added immediately after import matplotlib # the following line is added immediately after import matplotlib
@ -91,7 +92,7 @@ def plot_paddle_curve(keys, inputfile, outputfile, format='png',
sys.stderr.write("No data to plot. Exiting!\n") sys.stderr.write("No data to plot. Exiting!\n")
return return
m = len(keys) + 1 m = len(keys) + 1
for i in xrange(1, m): for i in six.moves.xrange(1, m):
pyplot.plot( pyplot.plot(
x[:, 0], x[:, 0],
x[:, i], x[:, i],

5
python/paddle/utils/predefined_net.py
Unescape View File

@ -13,6 +13,7 @@
# limitations under the License. # limitations under the License.
import numpy as np import numpy as np
import six
import os import os
from paddle.trainer.config_parser import * from paddle.trainer.config_parser import *
from paddle.utils.preprocess_img import \ from paddle.utils.preprocess_img import \
@ -112,7 +113,7 @@ def simple_conv_net(data_conf, is_color=False):
num_classes: num of classes. num_classes: num of classes.
is_color: whether the input images are color. is_color: whether the input images are color.
""" """
for k, v in data_conf.iteritems(): for k, v in six.iteritems(data_conf):
globals()[k] = v globals()[k] = v
data_input, label_input, num_image_channels = \ data_input, label_input, num_image_channels = \
image_data_layers(image_size, num_classes, is_color, is_predict) image_data_layers(image_size, num_classes, is_color, is_predict)
@ -340,7 +341,7 @@ def small_vgg(data_conf, is_predict=False):
num_classes: num of classes. num_classes: num of classes.
is_color: whether the input images are color. is_color: whether the input images are color.
""" """
for k, v in data_conf.iteritems(): for k, v in six.iteritems(data_conf):
globals()[k] = v globals()[k] = v
vgg_conv_net(image_size, num_classes, vgg_conv_net(image_size, num_classes,
num_layers=[2, 2, 3, 3], num_layers=[2, 2, 3, 3],

14
python/paddle/utils/preprocess_img.py
Unescape View File

@ -17,9 +17,9 @@ import os
import random import random
import numpy as np import numpy as np
import PIL.Image as Image import PIL.Image as Image
import StringIO from six.moves import cStringIO as StringIO
import preprocess_util from . import preprocess_util
from image_util import crop_img from .image_util import crop_img
def resize_image(img, target_size): def resize_image(img, target_size):
@ -52,7 +52,7 @@ class DiskImage:
def read_image(self): def read_image(self):
if self.img is None: if self.img is None:
print "reading: " + self.path print("reading: " + self.path)
image = resize_image(Image.open(self.path), self.target_size) image = resize_image(Image.open(self.path), self.target_size)
self.img = image self.img = image
@ -69,7 +69,7 @@ class DiskImage:
convert the image into the paddle batch format. convert the image into the paddle batch format.
""" """
self.read_image() self.read_image()
output = StringIO.StringIO() output = StringIO()
self.img.save(output, "jpeg") self.img.save(output, "jpeg")
contents = output.getvalue() contents = output.getvalue()
return contents return contents
@ -127,7 +127,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater):
image_path = items[0] image_path = items[0]
label_name = items[1] label_name = items[1]
if not label_name in label_set: if not label_name in label_set:
label_set[label_name] = len(label_set.keys()) label_set[label_name] = len(list(label_set.keys()))
img = DiskImage(path=image_path, target_size=self.target_size) img = DiskImage(path=image_path, target_size=self.target_size)
label = preprocess_util.Lablel( label = preprocess_util.Lablel(
label=label_set[label_name], name=label_name) label=label_set[label_name], name=label_name)
@ -144,7 +144,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater):
return create_dataset_from_list(path) return create_dataset_from_list(path)
label_set = preprocess_util.get_label_set_from_dir(path) label_set = preprocess_util.get_label_set_from_dir(path)
data = [] data = []
for l_name in label_set.keys(): for l_name in list(label_set.keys()):
image_paths = preprocess_util.list_images( image_paths = preprocess_util.list_images(
os.path.join(path, l_name)) os.path.join(path, l_name))
for p in image_paths: for p in image_paths:

10
python/paddle/utils/preprocess_util.py
Unescape View File

@ -14,7 +14,7 @@
import os import os
import math import math
import cPickle as pickle import six.moves.cPickle as pickle
import random import random
import collections import collections
@ -169,7 +169,7 @@ class Dataset:
random.shuffle(keyvalue_indices[k]) random.shuffle(keyvalue_indices[k])
num_data_per_key_batch = \ num_data_per_key_batch = \
math.ceil(num_per_batch / float(len(keyvalue_indices.keys()))) math.ceil(num_per_batch / float(len(list(keyvalue_indices.keys()))))
if num_data_per_key_batch < 2: if num_data_per_key_batch < 2:
raise Exception("The number of data in a batch is too small") raise Exception("The number of data in a batch is too small")
@ -182,8 +182,8 @@ class Dataset:
end_idx = int( end_idx = int(
min(begin_idx + num_data_per_key_batch, min(begin_idx + num_data_per_key_batch,
len(keyvalue_indices[k]))) len(keyvalue_indices[k])))
print "begin_idx, end_idx" print("begin_idx, end_idx")
print begin_idx, end_idx print(begin_idx, end_idx)
for idx in range(begin_idx, end_idx): for idx in range(begin_idx, end_idx):
permuted_data.append(self.data[keyvalue_indices[k][idx]]) permuted_data.append(self.data[keyvalue_indices[k][idx]])
keyvalue_readpointer[k] = end_idx keyvalue_readpointer[k] = end_idx
@ -357,6 +357,6 @@ class DatasetCreater(object):
data_batcher.create_batches_and_list( data_batcher.create_batches_and_list(
self.output_path, self.train_list_name, self.test_list_name, self.output_path, self.train_list_name, self.test_list_name,
self.label_set_name) self.label_set_name)
self.num_classes = len(train_label_set.keys()) self.num_classes = len(list(train_label_set.keys()))
self.create_meta_file(train_data) self.create_meta_file(train_data)
return out_path return out_path

8
python/paddle/utils/show_pb.py
Unescape View File

@ -15,6 +15,8 @@
Show the content of proto buffer data file of PADDLE Show the content of proto buffer data file of PADDLE
""" """
from __future__ import print_function
import os import os
import sys import sys
from google.protobuf.internal.decoder import _DecodeVarint from google.protobuf.internal.decoder import _DecodeVarint
@ -39,7 +41,7 @@ def read_proto(file, message):
def usage(): def usage():
print >> sys.stderr, "Usage: python show_pb.py PROTO_DATA_FILE" print("Usage: python show_pb.py PROTO_DATA_FILE", file=sys.stderr)
exit(1) exit(1)
@ -50,8 +52,8 @@ if __name__ == '__main__':
f = open(sys.argv[1]) f = open(sys.argv[1])
header = DataFormat.DataHeader() header = DataFormat.DataHeader()
read_proto(f, header) read_proto(f, header)
print header print(header)
sample = DataFormat.DataSample() sample = DataFormat.DataSample()
while read_proto(f, sample): while read_proto(f, sample):
print sample print(sample)

4
python/paddle/utils/torch2paddle.py
Unescape View File

@ -24,7 +24,7 @@ import sys
import struct import struct
import numpy as np import numpy as np
import torchfile import torchfile
import cPickle as pickle import six.moves.cPickle as pickle
import argparse import argparse
@ -48,7 +48,7 @@ def save_net_parameters(layers, params, output_path):
biases = params[i * 2 + 1] biases = params[i * 2 + 1]
weight_file = os.path.join(output_path, '_%s.w0' % layers[i]) weight_file = os.path.join(output_path, '_%s.w0' % layers[i])
biases_file = os.path.join(output_path, '_%s.wbias' % layers[i]) biases_file = os.path.join(output_path, '_%s.wbias' % layers[i])
print "Saving for layer %s." % layers[i] print("Saving for layer %s." % layers[i])
save_layer_parameters(weight_file, [weight]) save_layer_parameters(weight_file, [weight])
save_layer_parameters(biases_file, biases) save_layer_parameters(biases_file, biases)

Write Preview
Loading…
Cancel
Save