Apply 2to3 to current paddle main python code

revert-12646-feature/jit/xbyak
minqiyang 7 years ago
parent 3ade95d0db
commit 559d36328c

@ -394,8 +394,10 @@ All parameter, weight, gradient are variables in Paddle.
InferenceOptimize(*(origin.Proto()), &pruned_desc);
return new ProgramDesc(pruned_desc);
});
m.def("empty_var_name", []() { return framework::kEmptyVarName; });
m.def("grad_var_suffix", []() { return framework::kGradVarSuffix; });
m.def("empty_var_name",
[]() { return std::string(framework::kEmptyVarName); });
m.def("grad_var_suffix",
[]() { return std::string(framework::kGradVarSuffix); });
m.def_submodule(
"var_names",
"The module will return special predefined variable name in Paddle")

@ -28,7 +28,7 @@ images per class.
"""
import cPickle
import pickle
import itertools
import numpy
import paddle.dataset.common
@ -48,7 +48,7 @@ def reader_creator(filename, sub_name, cycle=False):
data = batch['data']
labels = batch.get('labels', batch.get('fine_labels', None))
assert labels is not None
for sample, label in itertools.izip(data, labels):
for sample, label in zip(data, labels):
yield (sample / 255.0).astype(numpy.float32), int(label)
def reader():
@ -58,7 +58,7 @@ def reader_creator(filename, sub_name, cycle=False):
while True:
for name in names:
batch = cPickle.load(f.extractfile(name))
batch = pickle.load(f.extractfile(name))
for item in read_batch(batch):
yield item
if not cycle:

@ -20,9 +20,9 @@ import shutil
import sys
import importlib
import paddle.dataset
import cPickle
import pickle
import glob
import cPickle as pickle
import pickle as pickle
__all__ = [
'DATA_HOME',
@ -75,13 +75,13 @@ def download(url, module_name, md5sum, save_name=None):
retry_limit = 3
while not (os.path.exists(filename) and md5file(filename) == md5sum):
if os.path.exists(filename):
print "file md5", md5file(filename), md5sum
print(("file md5", md5file(filename), md5sum))
if retry < retry_limit:
retry += 1
else:
raise RuntimeError("Cannot download {0} within retry limit {1}".
format(url, retry_limit))
print "Cache file %s not found, downloading %s" % (filename, url)
print(("Cache file %s not found, downloading %s" % (filename, url)))
r = requests.get(url, stream=True)
total_length = r.headers.get('content-length')
@ -104,8 +104,9 @@ def download(url, module_name, md5sum, save_name=None):
def fetch_all():
for module_name in filter(lambda x: not x.startswith("__"),
dir(paddle.dataset)):
for module_name in [
x for x in dir(paddle.dataset) if not x.startswith("__")
]:
if "fetch" in dir(
importlib.import_module("paddle.dataset.%s" % module_name)):
getattr(
@ -114,8 +115,9 @@ def fetch_all():
def fetch_all_recordio(path):
for module_name in filter(lambda x: not x.startswith("__"),
dir(paddle.dataset)):
for module_name in [
x for x in dir(paddle.dataset) if not x.startswith("__")
]:
if "convert" in dir(
importlib.import_module("paddle.dataset.%s" % module_name)) and \
not module_name == "common":
@ -126,7 +128,7 @@ def fetch_all_recordio(path):
"convert")(ds_path)
def split(reader, line_count, suffix="%05d.pickle", dumper=cPickle.dump):
def split(reader, line_count, suffix="%05d.pickle", dumper=pickle.dump):
"""
you can call the function as:
@ -167,7 +169,7 @@ def split(reader, line_count, suffix="%05d.pickle", dumper=cPickle.dump):
def cluster_files_reader(files_pattern,
trainer_count,
trainer_id,
loader=cPickle.load):
loader=pickle.load):
"""
Create a reader that yield element from the given files, select
a file set according trainer count and trainer_id
@ -188,7 +190,7 @@ def cluster_files_reader(files_pattern,
my_file_list = []
for idx, fn in enumerate(file_list):
if idx % trainer_count == trainer_id:
print "append file: %s" % fn
print(("append file: %s" % fn))
my_file_list.append(fn)
for fn in my_file_list:
with open(fn, "r") as f:
@ -221,7 +223,7 @@ def convert(output_path, reader, line_count, name_prefix):
for l in lines:
# FIXME(Yancey1989):
# dumps with protocol: pickle.HIGHEST_PROTOCOL
writer.write(cPickle.dumps(l))
writer.write(pickle.dumps(l))
writer.close()
lines = []

@ -87,12 +87,12 @@ def corpus_reader(data_path, words_name, props_name):
sentences = []
labels = []
one_seg = []
for word, label in itertools.izip(words_file, props_file):
for word, label in zip(words_file, props_file):
word = word.strip()
label = label.strip().split()
if len(label) == 0: # end of sentence
for i in xrange(len(one_seg[0])):
for i in range(len(one_seg[0])):
a_kind_lable = [x[i] for x in one_seg]
labels.append(a_kind_lable)

@ -28,10 +28,10 @@ Graphics and Image Processing (2008)
http://www.robots.ox.ac.uk/~vgg/publications/papers/nilsback08.{pdf,ps.gz}.
"""
import cPickle
import pickle
import itertools
import functools
from common import download
from .common import download
import tarfile
import scipy.io as scio
from paddle.dataset.image import *
@ -116,10 +116,10 @@ def reader_creator(data_file,
file = file.strip()
batch = None
with open(file, 'r') as f:
batch = cPickle.load(f)
batch = pickle.load(f)
data = batch['data']
labels = batch['label']
for sample, label in itertools.izip(data, batch['label']):
for sample, label in zip(data, batch['label']):
yield sample, int(label) - 1
if not cycle:
break

@ -36,7 +36,7 @@ except ImportError:
cv2 = None
import os
import tarfile
import cPickle
import pickle
__all__ = [
"load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop",
@ -86,10 +86,10 @@ def batch_images_from_tar(data_file,
output = {}
output['label'] = labels
output['data'] = data
cPickle.dump(
pickle.dump(
output,
open('%s/batch_%d' % (out_path, file_id), 'w'),
protocol=cPickle.HIGHEST_PROTOCOL)
protocol=pickle.HIGHEST_PROTOCOL)
file_id += 1
data = []
labels = []
@ -97,10 +97,10 @@ def batch_images_from_tar(data_file,
output = {}
output['label'] = labels
output['data'] = data
cPickle.dump(
pickle.dump(
output,
open('%s/batch_%d' % (out_path, file_id), 'w'),
protocol=cPickle.HIGHEST_PROTOCOL)
protocol=pickle.HIGHEST_PROTOCOL)
with open(meta_file, 'a') as meta:
for file in os.listdir(out_path):

@ -42,13 +42,13 @@ def tokenize(pattern):
# sequential access of member files, other than
# tarfile.extractfile, which does random access and might
# destroy hard disks.
tf = tarf.next()
tf = next(tarf)
while tf != None:
if bool(pattern.match(tf.name)):
# newline and punctuations removal and ad-hoc tokenization.
yield tarf.extractfile(tf).read().rstrip("\n\r").translate(
None, string.punctuation).lower().split()
tf = tarf.next()
tf = next(tarf)
def build_dict(pattern, cutoff):
@ -62,11 +62,11 @@ def build_dict(pattern, cutoff):
word_freq[word] += 1
# Not sure if we should prune less-frequent words here.
word_freq = filter(lambda x: x[1] > cutoff, word_freq.items())
word_freq = [x for x in list(word_freq.items()) if x[1] > cutoff]
dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0]))
words, _ = list(zip(*dictionary))
word_idx = dict(zip(words, xrange(len(words))))
word_idx = dict(list(zip(words, list(range(len(words))))))
word_idx['<unk>'] = len(words)
return word_idx

@ -64,11 +64,11 @@ def build_dict(min_word_freq=50):
# remove <unk> for now, since we will set it as last index
del word_freq['<unk>']
word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items())
word_freq = [x for x in list(word_freq.items()) if x[1] > min_word_freq]
word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0]))
words, _ = list(zip(*word_freq_sorted))
word_idx = dict(zip(words, xrange(len(words))))
word_idx = dict(list(zip(words, list(range(len(words))))))
word_idx['<unk>'] = len(words)
return word_idx

@ -65,7 +65,7 @@ def reader_creator(image_filename, label_filename, buffer_size):
images = images / 255.0 * 2.0 - 1.0
for i in xrange(buffer_size):
for i in range(buffer_size):
yield images[i, :], int(labels[i])
finally:
try:

@ -187,7 +187,7 @@ def max_movie_id():
Get the maximum value of movie id.
"""
__initialize_meta_info__()
return reduce(__max_index_info__, MOVIE_INFO.viewvalues()).index
return reduce(__max_index_info__, list(MOVIE_INFO.values())).index
def max_user_id():
@ -195,7 +195,7 @@ def max_user_id():
Get the maximum value of user id.
"""
__initialize_meta_info__()
return reduce(__max_index_info__, USER_INFO.viewvalues()).index
return reduce(__max_index_info__, list(USER_INFO.values())).index
def __max_job_id_impl__(a, b):
@ -210,7 +210,7 @@ def max_job_id():
Get the maximum value of job id.
"""
__initialize_meta_info__()
return reduce(__max_job_id_impl__, USER_INFO.viewvalues()).job_id
return reduce(__max_job_id_impl__, list(USER_INFO.values())).job_id
def movie_categories():
@ -243,7 +243,7 @@ def unittest():
for test_count, _ in enumerate(test()()):
pass
print train_count, test_count
print((train_count, test_count))
def fetch():

@ -26,7 +26,7 @@ http://research.microsoft.com/en-us/um/beijing/projects/letor/LETOR4.0/Data/MQ20
import os
import functools
import rarfile
from common import download
from .common import download
import numpy as np
# URL = "http://research.microsoft.com/en-us/um/beijing/projects/letor/LETOR4.0/Data/MQ2007.rar"
@ -330,4 +330,4 @@ if __name__ == "__main__":
mytest = functools.partial(
__reader__, filepath="MQ2007/MQ2007/Fold1/sample", format="listwise")
for label, query in mytest():
print label, query
print((label, query))

@ -43,11 +43,11 @@ def download_data_if_not_yet():
nltk.data.path.append(paddle.dataset.common.DATA_HOME)
movie_reviews.categories()
except LookupError:
print "Downloading movie_reviews data set, please wait....."
print("Downloading movie_reviews data set, please wait.....")
nltk.download(
'movie_reviews', download_dir=paddle.dataset.common.DATA_HOME)
print "Download data set success....."
print "Path is " + nltk.data.find('corpora/movie_reviews').path
print("Download data set success.....")
print(("Path is " + nltk.data.find('corpora/movie_reviews').path))
def get_word_dict():
@ -64,7 +64,7 @@ def get_word_dict():
for field in movie_reviews.fileids(category):
for words in movie_reviews.words(field):
word_freq_dict[words] += 1
words_sort_list = word_freq_dict.items()
words_sort_list = list(word_freq_dict.items())
words_sort_list.sort(cmp=lambda a, b: b[1] - a[1])
for index, word in enumerate(words_sort_list):
words_freq_sorted.append((word[0], index))
@ -80,7 +80,8 @@ def sort_files():
files_list = list()
neg_file_list = movie_reviews.fileids('neg')
pos_file_list = movie_reviews.fileids('pos')
files_list = list(chain.from_iterable(zip(neg_file_list, pos_file_list)))
files_list = list(
chain.from_iterable(list(zip(neg_file_list, pos_file_list))))
return files_list

@ -36,7 +36,7 @@ class TestCommon(unittest.TestCase):
def test_split(self):
def test_reader():
def reader():
for x in xrange(10):
for x in range(10):
yield x
return reader
@ -49,7 +49,7 @@ class TestCommon(unittest.TestCase):
def test_cluster_file_reader(self):
_, temp_path = tempfile.mkstemp()
for x in xrange(5):
for x in range(5):
with open(temp_path + '/%05d.test' % x) as f:
f.write('%d\n' % x)
reader = paddle.dataset.common.cluster_files_reader(
@ -63,7 +63,7 @@ class TestCommon(unittest.TestCase):
def test_reader():
def reader():
for x in xrange(record_num):
for x in range(record_num):
yield x
return reader

@ -59,7 +59,7 @@ class TestMikolov(unittest.TestCase):
self.assertEqual(first_line, read_line)
def test_total(self):
_, idx = zip(*WORD_DICT.items())
_, idx = list(zip(*list(WORD_DICT.items())))
self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1)

@ -24,9 +24,8 @@ from nltk.corpus import movie_reviews
class TestSentimentMethods(unittest.TestCase):
def test_get_word_dict(self):
word_dict = st.get_word_dict()[0:10]
test_word_list = [(u',', 0), (u'the', 1), (u'.', 2), (u'a', 3),
(u'and', 4), (u'of', 5), (u'to', 6), (u"'", 7),
(u'is', 8), (u'in', 9)]
test_word_list = [(',', 0), ('the', 1), ('.', 2), ('a', 3), ('and', 4),
('of', 5), ('to', 6), ("'", 7), ('is', 8), ('in', 9)]
for idx, each in enumerate(word_dict):
self.assertEqual(each, test_word_list[idx])
self.assertTrue("/root/.cache/paddle/dataset" in nltk.data.path)

@ -49,9 +49,12 @@ def feature_range(maximums, minimums):
import matplotlib.pyplot as plt
fig, ax = plt.subplots()
feature_num = len(maximums)
ax.bar(range(feature_num), maximums - minimums, color='r', align='center')
ax.bar(list(range(feature_num)),
maximums - minimums,
color='r',
align='center')
ax.set_title('feature scale')
plt.xticks(range(feature_num), feature_names)
plt.xticks(list(range(feature_num)), feature_names)
plt.xlim([-1, feature_num])
fig.set_figheight(6)
fig.set_figwidth(10)
@ -71,7 +74,7 @@ def load_data(filename, feature_num=14, ratio=0.8):
maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum(
axis=0) / data.shape[0]
feature_range(maximums[:-1], minimums[:-1])
for i in xrange(feature_num - 1):
for i in range(feature_num - 1):
data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
offset = int(data.shape[0] * ratio)
UCI_TRAIN_DATA = data[:offset]

@ -154,8 +154,8 @@ def get_dict(dict_size, reverse=True):
tar_file = paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN)
src_dict, trg_dict = __read_to_dict(tar_file, dict_size)
if reverse:
src_dict = {v: k for k, v in src_dict.items()}
trg_dict = {v: k for k, v in trg_dict.items()}
src_dict = {v: k for k, v in list(src_dict.items())}
trg_dict = {v: k for k, v in list(trg_dict.items())}
return src_dict, trg_dict

@ -70,7 +70,9 @@ def __build_dict(tar_file, dict_size, save_path, lang):
fout.write("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK))
for idx, word in enumerate(
sorted(
word_dict.iteritems(), key=lambda x: x[1], reverse=True)):
iter(list(word_dict.items())),
key=lambda x: x[1],
reverse=True)):
if idx + 3 == dict_size: break
fout.write("%s\n" % (word[0]))

@ -14,49 +14,49 @@
from __future__ import print_function
# import all class inside framework into fluid module
import framework
from framework import *
from . import framework
from .framework import *
# import all class inside executor into fluid module
import executor
from executor import *
import trainer
from trainer import Trainer
from trainer import BeginEpochEvent
from trainer import EndEpochEvent
from trainer import BeginStepEvent
from trainer import EndStepEvent
from trainer import CheckpointConfig
import inferencer
from inferencer import Inferencer
import io
import evaluator
import initializer
import layers
import contrib
import nets
import optimizer
import backward
import regularizer
import average
import metrics
import transpiler
from param_attr import ParamAttr, WeightNormParamAttr
from data_feeder import DataFeeder
from core import LoDTensor, LoDTensorArray, CPUPlace, CUDAPlace, CUDAPinnedPlace, Scope
from transpiler import DistributeTranspiler, InferenceTranspiler, \
from . import executor
from .executor import *
from . import trainer
from .trainer import Trainer
from .trainer import BeginEpochEvent
from .trainer import EndEpochEvent
from .trainer import BeginStepEvent
from .trainer import EndStepEvent
from .trainer import CheckpointConfig
from . import inferencer
from .inferencer import Inferencer
from . import io
from . import evaluator
from . import initializer
from . import layers
from . import contrib
from . import nets
from . import optimizer
from . import backward
from . import regularizer
from . import average
from . import metrics
from . import transpiler
from .param_attr import ParamAttr, WeightNormParamAttr
from .data_feeder import DataFeeder
from .core import LoDTensor, LoDTensorArray, CPUPlace, CUDAPlace, CUDAPinnedPlace, Scope
from .transpiler import DistributeTranspiler, InferenceTranspiler, \
memory_optimize, release_memory, DistributeTranspilerConfig
from concurrency import (Go, make_channel, channel_send, channel_recv,
channel_close, Select)
from lod_tensor import create_lod_tensor, create_random_int_lodtensor
import clip
import profiler
import unique_name
import recordio_writer
import parallel_executor
from parallel_executor import *
from .concurrency import (Go, make_channel, channel_send, channel_recv,
channel_close, Select)
from .lod_tensor import create_lod_tensor, create_random_int_lodtensor
from . import clip
from . import profiler
from . import unique_name
from . import recordio_writer
from . import parallel_executor
from .parallel_executor import *
from paddle.fluid.layers.math_op_patch import monkey_patch_variable
Tensor = LoDTensor
@ -99,8 +99,8 @@ def __bootstrap__():
None
"""
import sys
import core
import os
from . import core
in_test = 'unittest' in sys.modules

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import functools
import sys
@ -28,7 +29,7 @@ def deprecated(since, instead, extra_message=""):
@functools.wraps(func)
def wrapper(*args, **kwargs):
print >> sys.stderr, err_msg
print(err_msg, file=sys.stderr)
return func(*args, **kwargs)
wrapper.__doc__ += "\n "

File diff suppressed because it is too large Load Diff

@ -15,8 +15,8 @@
import copy
import functools
import layers
import framework
from . import layers
from . import framework
from . import core
__all__ = [
@ -80,8 +80,7 @@ def error_clip_callback(block, context):
# the context is a grad_to_var map
grad_to_var = context
op_desc = block.desc.op(block.desc.op_size() - 1)
for grad_n in filter(lambda n: grad_to_var.has_key(n),
op_desc.output_arg_names()):
for grad_n in [n for n in op_desc.output_arg_names() if n in grad_to_var]:
fwd_var = block._var_recursive(grad_to_var[grad_n])
error_clip = getattr(fwd_var, "error_clip", None)
if not (error_clip is None or isinstance(error_clip,
@ -247,7 +246,7 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr):
"""
def __init__(self, clip_norm, group_name="default_group"):
if not isinstance(group_name, basestring):
if not isinstance(group_name, str):
raise TypeError("'group_name' must be a basestring.")
self.clip_norm = clip_norm
@ -284,7 +283,7 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr):
x=clip_var,
y=layers.elementwise_max(
x=clip_var, y=group_norm_var))
assert group_scale_var.shape == (1L, )
assert group_scale_var.shape == (1, )
self.context[group_scale_name] = group_scale_var
new_grad = layers.elementwise_mul(
@ -313,7 +312,7 @@ def set_gradient_clip(clip, param_list=None, program=None):
program = framework.default_main_program()
if param_list is None:
param_list = program.block(0).all_parameters()
if all(isinstance(elem, basestring) for elem in param_list):
if all(isinstance(elem, str) for elem in param_list):
param_list = [program.block(0).var(elem) for elem in param_list]
if not all(isinstance(elem, framework.Parameter) for elem in param_list):
raise TypeError(

@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from layers.control_flow import BlockGuard, equal
from .layers.control_flow import BlockGuard, equal
from .framework import Operator
from layer_helper import LayerHelper, unique_name
from layers import fill_constant
import core
from .layer_helper import LayerHelper, unique_name
from .layers import fill_constant
from . import core
__all__ = [
'Go', 'make_channel', 'channel_send', 'channel_recv', 'channel_close',

@ -12,14 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import core
from . import core
import numpy
import os
import six.moves as six
import multiprocessing
from framework import Variable, default_main_program
from .framework import Variable, default_main_program
__all__ = ['DataFeeder']
@ -142,7 +141,7 @@ class DataFeeder(object):
if program is None:
program = default_main_program()
for each_var in feed_list:
if isinstance(each_var, basestring):
if isinstance(each_var, str):
each_var = program.block(0).var(each_var)
if not isinstance(each_var, Variable):
raise TypeError("Feed list should contain a list of variable")

@ -14,8 +14,8 @@
import sys
import re
from graphviz import GraphPreviewGenerator
import proto.framework_pb2 as framework_pb2
from .graphviz import GraphPreviewGenerator
from .proto import framework_pb2
from google.protobuf import text_format
_vartype2str_ = [

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save