Merge branch 'develop' of https://github.com/PaddlePaddle/paddle into enhance-lookup_table_op-padidx
guosheng
7 years ago
commit
b03c8b90ae
@ -0,0 +1,66 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import paddle.v2.dataset.wmt16
|
||||
import unittest
|
||||
|
||||
|
||||
class TestWMT16(unittest.TestCase):
|
||||
def checkout_one_sample(self, sample):
|
||||
# train data has 3 field: source language word indices,
|
||||
# target language word indices, and target next word indices.
|
||||
self.assertEqual(len(sample), 3)
|
||||
|
||||
# test start mark and end mark in source word indices.
|
||||
self.assertEqual(sample[0][0], 0)
|
||||
self.assertEqual(sample[0][-1], 1)
|
||||
|
||||
# test start mask in target word indices
|
||||
self.assertEqual(sample[1][0], 0)
|
||||
|
||||
# test en mask in target next word indices
|
||||
self.assertEqual(sample[2][-1], 1)
|
||||
|
||||
def test_train(self):
|
||||
for idx, sample in enumerate(
|
||||
paddle.v2.dataset.wmt16.train(
|
||||
src_dict_size=100000, trg_dict_size=100000)()):
|
||||
if idx >= 10: break
|
||||
self.checkout_one_sample(sample)
|
||||
|
||||
def test_test(self):
|
||||
for idx, sample in enumerate(
|
||||
paddle.v2.dataset.wmt16.test(
|
||||
src_dict_size=1000, trg_dict_size=1000)()):
|
||||
if idx >= 10: break
|
||||
self.checkout_one_sample(sample)
|
||||
|
||||
def test_val(self):
|
||||
for idx, sample in enumerate(
|
||||
paddle.v2.dataset.wmt16.validation(
|
||||
src_dict_size=1000, trg_dict_size=1000)()):
|
||||
if idx >= 10: break
|
||||
self.checkout_one_sample(sample)
|
||||
|
||||
def test_get_dict(self):
|
||||
dict_size = 1000
|
||||
word_dict = paddle.v2.dataset.wmt16.get_dict("en", dict_size, True)
|
||||
self.assertEqual(len(word_dict), dict_size)
|
||||
self.assertEqual(word_dict[0], "<s>")
|
||||
self.assertEqual(word_dict[1], "<e>")
|
||||
self.assertEqual(word_dict[2], "<unk>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,152 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from ..framework import Variable, unique_name
|
||||
from ..registry import OpProtoHolder
|
||||
|
||||
__all__ = ['monkey_patch_variable']
|
||||
|
||||
|
||||
def monkey_patch_variable():
|
||||
def unique_tmp_name():
|
||||
return unique_name("tmp")
|
||||
|
||||
def safe_get_dtype(var):
|
||||
try:
|
||||
dtype = var.dtype
|
||||
except:
|
||||
raise ValueError("Cannot get data type from %s", var.name)
|
||||
return dtype
|
||||
|
||||
def create_tensor(block, value, dtype, shape):
|
||||
value = float(value)
|
||||
tmp_name = unique_tmp_name()
|
||||
var = block.create_var(name=tmp_name, shape=shape, dtype=dtype)
|
||||
block.append_op(
|
||||
type="fill_constant",
|
||||
outputs={'Out': [var]},
|
||||
attrs={'dtype': var.dtype,
|
||||
'shape': shape,
|
||||
'value': value})
|
||||
return var
|
||||
|
||||
def create_scalar(block, value, dtype):
|
||||
return create_tensor(block, value, dtype, shape=[1])
|
||||
|
||||
def create_tensor_with_batchsize(ref_var, value, dtype):
|
||||
assert isinstance(ref_var, Variable)
|
||||
value = float(value)
|
||||
tmp_name = unique_tmp_name()
|
||||
var = ref_var.block.create_var(name=tmp_name, dtype=dtype)
|
||||
ref_var.block.append_op(
|
||||
type='fill_constant_batch_size_like',
|
||||
outputs={'Out': [var]},
|
||||
inputs={'Input': [ref_var]},
|
||||
attrs={'shape': ref_var.shape,
|
||||
'value': value})
|
||||
return var
|
||||
|
||||
def astype(self, dtype):
|
||||
"""
|
||||
Cast a variable to a specified data type.
|
||||
NOTE: The variable must be a Tensor
|
||||
Args:
|
||||
self(Variable): The source variable
|
||||
dtype: The target dtype
|
||||
|
||||
Returns:
|
||||
Variable with new dtype
|
||||
"""
|
||||
tmp_name = unique_tmp_name()
|
||||
out = self.block.create_var(name=tmp_name, dtype=dtype)
|
||||
self.block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": [self]},
|
||||
outputs={"Out": [out]},
|
||||
attrs={"in_dtype": self.dtype,
|
||||
"out_dtype": out.dtype})
|
||||
return out
|
||||
|
||||
def _elemwise_method_creator_(method_name, op_type, reverse=False):
|
||||
def __impl__(self, other_var):
|
||||
lhs_dtype = safe_get_dtype(self)
|
||||
|
||||
if not isinstance(other_var, Variable):
|
||||
if reverse:
|
||||
has_batch_size = False
|
||||
for elem in self.shape:
|
||||
if elem < 0:
|
||||
has_batch_size = True
|
||||
break
|
||||
if not has_batch_size:
|
||||
other_var = create_tensor(
|
||||
self.block,
|
||||
other_var,
|
||||
dtype=lhs_dtype,
|
||||
shape=self.shape)
|
||||
else:
|
||||
other_var = create_tensor_with_batchsize(
|
||||
self, other_var, lhs_dtype)
|
||||
else:
|
||||
# add fill_op to self.block
|
||||
other_var = create_scalar(
|
||||
self.block, value=other_var, dtype=lhs_dtype)
|
||||
|
||||
rhs_dtype = safe_get_dtype(other_var)
|
||||
if lhs_dtype != rhs_dtype:
|
||||
other_var = astype(other_var, lhs_dtype)
|
||||
if reverse:
|
||||
tmp = self
|
||||
self = other_var
|
||||
other_var = tmp
|
||||
|
||||
tmp_name = unique_tmp_name()
|
||||
out = self.block.create_var(name=tmp_name, dtype=lhs_dtype)
|
||||
self.block.append_op(
|
||||
type=op_type,
|
||||
inputs={'X': [self],
|
||||
'Y': [other_var]},
|
||||
outputs={'Out': out})
|
||||
return out
|
||||
|
||||
comment = OpProtoHolder.instance().get_op_proto(op_type).comment
|
||||
|
||||
__impl__.__doc__ = """
|
||||
{0}
|
||||
Args:
|
||||
self(Variable): left hand variable
|
||||
other_var(Variable|float|int): right hand variable
|
||||
|
||||
Returns:
|
||||
Variable
|
||||
""".format(comment)
|
||||
__impl__.__name__ = method_name
|
||||
return __impl__
|
||||
|
||||
# inject methods
|
||||
for method_name, op_type, reverse in (
|
||||
("__add__", "elementwise_add", False),
|
||||
# a+b == b+a. Do not need to reverse explicitly
|
||||
("__radd__", "elementwise_add", False),
|
||||
("__sub__", "elementwise_sub", False),
|
||||
("__rsub__", "elementwise_sub", True),
|
||||
("__mul__", "elementwise_mul", False),
|
||||
# a*b == b*a. Do not need to reverse explicitly
|
||||
("__rmul__", "elementwise_mul", False),
|
||||
("__div__", "elementwise_div", False),
|
||||
("__rdiv__", "elementwise_div", True)):
|
||||
setattr(Variable, method_name,
|
||||
_elemwise_method_creator_(method_name, op_type, reverse))
|
||||
|
||||
Variable.astype = astype
|
||||
@ -0,0 +1,11 @@
|
||||
file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
|
||||
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
|
||||
|
||||
list(REMOVE_ITEM TEST_OPS test_memopt_image_classification_train)
|
||||
py_test(test_memopt_image_classification_train_resnet SRCS test_memopt_image_classification_train.py ARGS resnet)
|
||||
py_test(test_memopt_image_classification_train_vgg SRCS test_memopt_image_classification_train.py ARGS vgg)
|
||||
|
||||
# default test
|
||||
foreach(src ${TEST_OPS})
|
||||
py_test(${src} SRCS ${src}.py)
|
||||
endforeach()
|
||||
@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy as np
|
||||
import paddle.v2 as paddle
|
||||
import paddle.v2.fluid as fluid
|
||||
|
||||
x = fluid.layers.data(name='x', shape=[13], dtype='float32')
|
||||
|
||||
y_predict = fluid.layers.fc(input=x, size=1, act=None)
|
||||
|
||||
y = fluid.layers.data(name='y', shape=[1], dtype='float32')
|
||||
|
||||
cost = fluid.layers.square_error_cost(input=y_predict, label=y)
|
||||
avg_cost = fluid.layers.mean(x=cost)
|
||||
|
||||
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.1)
|
||||
sgd_optimizer.minimize(avg_cost)
|
||||
|
||||
# memopt_program = fluid.default_main_program()
|
||||
memopt_program = fluid.memory_optimize(fluid.default_main_program())
|
||||
|
||||
BATCH_SIZE = 200
|
||||
|
||||
train_reader = paddle.batch(
|
||||
paddle.reader.shuffle(
|
||||
paddle.dataset.uci_housing.train(), buf_size=500),
|
||||
batch_size=BATCH_SIZE)
|
||||
|
||||
place = fluid.CPUPlace()
|
||||
feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
|
||||
exe = fluid.Executor(place)
|
||||
|
||||
exe.run(fluid.default_startup_program())
|
||||
|
||||
PASS_NUM = 100
|
||||
for pass_id in range(PASS_NUM):
|
||||
fluid.io.save_persistables(exe, "./fit_a_line.model/")
|
||||
fluid.io.load_persistables(exe, "./fit_a_line.model/")
|
||||
for data in train_reader():
|
||||
avg_loss_value, = exe.run(memopt_program,
|
||||
feed=feeder.feed(data),
|
||||
fetch_list=[avg_cost])
|
||||
|
||||
if avg_loss_value[0] < 10.0:
|
||||
exit(0) # if avg cost less than 10.0, we think our code is good.
|
||||
exit(1)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue