Fix clip.py (#14718)
* expose square test=develop * fix activation test=develop * Add square API test=develop * add necessary op * code refine * fix API.spec test=develop * fix unit test test=develop * add unit test sparse_grad_clip test=develop * fix API.spec test=develop * remove mac test for test_gradient_clip test=develop * remove selectedrows_mul_tensor test=developrevert-14398-imperative
chengduo
7 years ago
committed by
GitHub
parent
052cc5f538
commit
04539d4c5d
@ -0,0 +1,117 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/fluid/framework/op_registry.h"
|
||||||
|
#include "paddle/fluid/framework/tensor_util.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
|
||||||
|
void InferShape(framework::InferShapeContext *ctx) const override {
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("X"),
|
||||||
|
"GetTensorFromSelectedRowsOp must has input X.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("Out"),
|
||||||
|
"GetTensorFromSelectedRowsOp must has output Out.");
|
||||||
|
PADDLE_ENFORCE(
|
||||||
|
ctx->GetInputsVarType("X").front() ==
|
||||||
|
framework::proto::VarType::SELECTED_ROWS,
|
||||||
|
"The input X's type should be SelectedRows, but the received is %s",
|
||||||
|
ctx->Inputs("X").front(), ctx->GetInputsVarType("X").front());
|
||||||
|
PADDLE_ENFORCE(
|
||||||
|
ctx->GetOutputsVarType("Out").front() ==
|
||||||
|
framework::proto::VarType::LOD_TENSOR,
|
||||||
|
"The output Out's type should be LoDTensor, but the received is %s",
|
||||||
|
ctx->Outputs("Out").front(), ctx->GetOutputsVarType("Out").front());
|
||||||
|
|
||||||
|
ctx->SetOutputDim("Out", ctx->GetInputDim("X"));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected:
|
||||||
|
framework::OpKernelType GetExpectedKernelType(
|
||||||
|
const framework::ExecutionContext &ctx) const override {
|
||||||
|
return framework::OpKernelType(
|
||||||
|
framework::GetDataTypeOfVar(ctx.InputVar("X")), ctx.device_context());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsKernel {
|
||||||
|
public:
|
||||||
|
void operator()(const framework::ExecutionContext &ctx) const {
|
||||||
|
auto *x = ctx.Input<framework::SelectedRows>("X");
|
||||||
|
auto *out = ctx.Output<framework::LoDTensor>("Out");
|
||||||
|
|
||||||
|
out->Resize(x->value().dims());
|
||||||
|
out->mutable_data(ctx.GetPlace(), x->value().type());
|
||||||
|
framework::TensorCopy(x->value(), ctx.GetPlace(), ctx.device_context(),
|
||||||
|
out);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsOpProtoMaker
|
||||||
|
: public framework::OpProtoAndCheckerMaker {
|
||||||
|
public:
|
||||||
|
void Make() override {
|
||||||
|
AddInput("X", "The input type is SelectedRows.");
|
||||||
|
AddOutput("Out", "The output type is LoDTensor.");
|
||||||
|
AddComment(
|
||||||
|
R"DOC(
|
||||||
|
GetTensorFromSelectedRows Operator
|
||||||
|
|
||||||
|
GetTensorFromSelectedRows is used to get the tensor from SelectedRows.
|
||||||
|
|
||||||
|
)DOC");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsOpVarTypeInference
|
||||||
|
: public framework::VarTypeInference {
|
||||||
|
public:
|
||||||
|
void operator()(const framework::OpDesc &op_desc,
|
||||||
|
framework::BlockDesc *block) const final {
|
||||||
|
auto out_var_name = op_desc.Output("Out").front();
|
||||||
|
auto in_var_name = op_desc.Input("X").front();
|
||||||
|
|
||||||
|
auto out_var = block->FindRecursiveOrCreateVar(out_var_name);
|
||||||
|
auto in_var = block->FindRecursiveOrCreateVar(in_var_name);
|
||||||
|
out_var.SetType(framework::proto::VarType::LOD_TENSOR);
|
||||||
|
out_var.SetDataType(in_var.GetDataType());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
REGISTER_OPERATOR(get_tensor_from_selected_rows,
|
||||||
|
ops::GetTensorFromSelectedRowsOp,
|
||||||
|
ops::GetTensorFromSelectedRowsOpProtoMaker,
|
||||||
|
ops::GetTensorFromSelectedRowsOpVarTypeInference);
|
||||||
|
|
||||||
|
REGISTER_OP_CPU_KERNEL_FUNCTOR(get_tensor_from_selected_rows, float,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, double,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int64_t,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel);
|
||||||
|
|
||||||
|
#ifdef PADDLE_WITH_CUDA
|
||||||
|
REGISTER_OP_CUDA_KERNEL_FUNCTOR(get_tensor_from_selected_rows, float,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, double,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int64_t,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel);
|
||||||
|
#endif
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/fluid/operators/merge_selected_rows_op.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
class MergeSelectedRowsOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
|
||||||
|
void InferShape(framework::InferShapeContext* ctx) const override {
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("X"),
|
||||||
|
"Input(X) of MergeSelectedRowsOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("Out"),
|
||||||
|
"Output(Out) of MergeSelectedRowsOp should not be null.");
|
||||||
|
ctx->ShareDim("X", /*->*/ "Out");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class MergeSelectedRowsOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||||
|
public:
|
||||||
|
void Make() override {
|
||||||
|
AddInput("X",
|
||||||
|
"The input type is SelectedRows, and the selected rows may be "
|
||||||
|
"duplicated.");
|
||||||
|
AddOutput("Out",
|
||||||
|
"The output type is SelectedRows, and the selected rows are not "
|
||||||
|
"duplicated.");
|
||||||
|
AddComment(
|
||||||
|
R"DOC(
|
||||||
|
MergeSelectedRows Operator.
|
||||||
|
|
||||||
|
MergeSelectedRows is used to merge the duplicated rows of the input.
|
||||||
|
)DOC");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class MergeSelectedRowsOpInferVarType
|
||||||
|
: public framework::PassInDtypeAndVarTypeToOutput {
|
||||||
|
protected:
|
||||||
|
std::unordered_map<std::string, std::string> GetInputOutputWithSameType()
|
||||||
|
const override {
|
||||||
|
return std::unordered_map<std::string, std::string>{{"X", /*->*/ "Out"}};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
namespace plat = paddle::platform;
|
||||||
|
REGISTER_OPERATOR(merge_selected_rows, ops::MergeSelectedRowsOp,
|
||||||
|
ops::MergeSelectedRowsOpMaker,
|
||||||
|
ops::MergeSelectedRowsOpInferVarType);
|
||||||
|
|
||||||
|
REGISTER_OP_CPU_KERNEL(
|
||||||
|
merge_selected_rows,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CPUDeviceContext, float>,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CPUDeviceContext, double>);
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/fluid/operators/merge_selected_rows_op.h"
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
namespace plat = paddle::platform;
|
||||||
|
|
||||||
|
REGISTER_OP_CUDA_KERNEL(
|
||||||
|
merge_selected_rows,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CUDADeviceContext, float>,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CUDADeviceContext, double>);
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include <string>
|
||||||
|
#include "paddle/fluid/framework/op_registry.h"
|
||||||
|
#include "paddle/fluid/operators/math/selected_rows_functor.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
template <typename DeviceContext, typename T>
|
||||||
|
class MergeSelectedRowsKernel : public framework::OpKernel<T> {
|
||||||
|
public:
|
||||||
|
void Compute(const framework::ExecutionContext& context) const override {
|
||||||
|
auto* x = context.Input<framework::SelectedRows>("X");
|
||||||
|
auto* out = context.Output<framework::SelectedRows>("Out");
|
||||||
|
|
||||||
|
math::scatter::MergeAdd<DeviceContext, T> merge_func;
|
||||||
|
merge_func(context.template device_context<DeviceContext>(), *x, out);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
@ -1,84 +0,0 @@
|
|||||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
from __future__ import print_function
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import paddle
|
|
||||||
import paddle.fluid as fluid
|
|
||||||
|
|
||||||
BATCH_SIZE = 128
|
|
||||||
CLIP = 1
|
|
||||||
|
|
||||||
prog = fluid.framework.Program()
|
|
||||||
with fluid.program_guard(main_program=prog):
|
|
||||||
image = fluid.layers.data(name='x', shape=[784], dtype='float32')
|
|
||||||
|
|
||||||
hidden1 = fluid.layers.fc(input=image, size=128, act='relu')
|
|
||||||
hidden2 = fluid.layers.fc(input=hidden1, size=64, act='relu')
|
|
||||||
predict = fluid.layers.fc(input=hidden2, size=10, act='softmax')
|
|
||||||
|
|
||||||
label = fluid.layers.data(name='y', shape=[1], dtype='int64')
|
|
||||||
|
|
||||||
cost = fluid.layers.cross_entropy(input=predict, label=label)
|
|
||||||
avg_cost = fluid.layers.mean(cost)
|
|
||||||
|
|
||||||
prog_clip = prog.clone()
|
|
||||||
|
|
||||||
avg_cost_clip = prog_clip.block(0).var(avg_cost.name)
|
|
||||||
|
|
||||||
p_g = fluid.backward.append_backward(loss=avg_cost)
|
|
||||||
p_g_clip = fluid.backward.append_backward(loss=avg_cost_clip)
|
|
||||||
|
|
||||||
with fluid.program_guard(main_program=prog_clip):
|
|
||||||
fluid.clip.set_gradient_clip(
|
|
||||||
fluid.clip.GradientClipByGlobalNorm(clip_norm=CLIP))
|
|
||||||
p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip)
|
|
||||||
|
|
||||||
grad_list = [elem[1] for elem in p_g]
|
|
||||||
grad_clip_list = [elem[1] for elem in p_g_clip]
|
|
||||||
|
|
||||||
train_reader = paddle.batch(
|
|
||||||
paddle.reader.shuffle(
|
|
||||||
paddle.dataset.mnist.train(), buf_size=8192),
|
|
||||||
batch_size=BATCH_SIZE)
|
|
||||||
|
|
||||||
place = fluid.CPUPlace()
|
|
||||||
exe = fluid.Executor(place)
|
|
||||||
feeder = fluid.DataFeeder(feed_list=[image, label], place=place)
|
|
||||||
exe.run(fluid.default_startup_program())
|
|
||||||
|
|
||||||
count = 0
|
|
||||||
for data in train_reader():
|
|
||||||
count += 1
|
|
||||||
if count > 5:
|
|
||||||
break
|
|
||||||
out = exe.run(prog, feed=feeder.feed(data), fetch_list=grad_list)
|
|
||||||
out_clip = exe.run(prog_clip,
|
|
||||||
feed=feeder.feed(data),
|
|
||||||
fetch_list=grad_clip_list)
|
|
||||||
global_norm = 0
|
|
||||||
for v in out[1:]:
|
|
||||||
global_norm += np.sum(np.power(v, 2))
|
|
||||||
global_norm = np.sqrt(global_norm)
|
|
||||||
|
|
||||||
global_norm_clip = 0
|
|
||||||
for v in out_clip[1:]:
|
|
||||||
global_norm_clip += np.sum(np.power(v, 2))
|
|
||||||
global_norm_clip = np.sqrt(global_norm_clip)
|
|
||||||
|
|
||||||
if not np.isclose(
|
|
||||||
a=global_norm_clip, b=np.minimum(global_norm, CLIP), rtol=5e-3):
|
|
||||||
exit(1)
|
|
||||||
exit(0)
|
|
||||||
@ -0,0 +1,65 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
import paddle.fluid.core as core
|
||||||
|
import numpy as np
|
||||||
|
from paddle.fluid.op import Operator
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetTensorFromSelectedRows(unittest.TestCase):
|
||||||
|
def get_places(self):
|
||||||
|
places = [core.CPUPlace()]
|
||||||
|
if core.is_compiled_with_cuda():
|
||||||
|
places.append(core.CUDAPlace(0))
|
||||||
|
return places
|
||||||
|
|
||||||
|
def check_with_place(self, place):
|
||||||
|
scope = core.Scope()
|
||||||
|
x_rows = [0, 5, 5, 4, 20]
|
||||||
|
height = 20
|
||||||
|
row_numel = 2
|
||||||
|
|
||||||
|
np_array = np.ones((len(x_rows), row_numel)).astype("float32")
|
||||||
|
np_array[1, :] = 2.0
|
||||||
|
np_array[2, :] = 3.0
|
||||||
|
np_array[3, :] = 4.0
|
||||||
|
|
||||||
|
# initialize input variable X
|
||||||
|
x = scope.var('X').get_selected_rows()
|
||||||
|
x.set_rows(x_rows)
|
||||||
|
x.set_height(height)
|
||||||
|
x_tensor = x.get_tensor()
|
||||||
|
x_tensor.set(np_array, place)
|
||||||
|
|
||||||
|
# initialize input variable Out
|
||||||
|
out = scope.var("Out").get_tensor()
|
||||||
|
|
||||||
|
op = Operator("get_tensor_from_selected_rows", X="X", Out="Out")
|
||||||
|
|
||||||
|
op.run(scope, place)
|
||||||
|
|
||||||
|
out_array = np.array(out)
|
||||||
|
self.assertEqual((5, 2), out_array.shape)
|
||||||
|
assert (out_array == np_array).all()
|
||||||
|
|
||||||
|
def test_check_output(self):
|
||||||
|
for place in self.get_places():
|
||||||
|
self.check_with_place(place)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@ -0,0 +1,162 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
import numpy as np
|
||||||
|
import paddle
|
||||||
|
import paddle.fluid.core as core
|
||||||
|
import paddle.fluid as fluid
|
||||||
|
|
||||||
|
BATCH_SIZE = 128
|
||||||
|
CLIP = 1
|
||||||
|
|
||||||
|
|
||||||
|
def bow_net(data,
|
||||||
|
label,
|
||||||
|
dict_dim,
|
||||||
|
emb_dim=128,
|
||||||
|
hid_dim=128,
|
||||||
|
hid_dim2=96,
|
||||||
|
class_dim=2):
|
||||||
|
"""
|
||||||
|
BOW net
|
||||||
|
This model is from https://github.com/PaddlePaddle/models:
|
||||||
|
fluid/PaddleNLP/text_classification/nets.py
|
||||||
|
"""
|
||||||
|
emb = fluid.layers.embedding(
|
||||||
|
input=data, is_sparse=True, size=[dict_dim, emb_dim])
|
||||||
|
bow = fluid.layers.sequence_pool(input=emb, pool_type='sum')
|
||||||
|
bow_tanh = fluid.layers.tanh(bow)
|
||||||
|
fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh")
|
||||||
|
fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh")
|
||||||
|
prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax")
|
||||||
|
cost = fluid.layers.cross_entropy(input=prediction, label=label)
|
||||||
|
avg_cost = fluid.layers.mean(x=cost)
|
||||||
|
|
||||||
|
return avg_cost
|
||||||
|
|
||||||
|
|
||||||
|
class TestGradientClip(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.word_dict = paddle.dataset.imdb.word_dict()
|
||||||
|
self.BATCH_SIZE = 2
|
||||||
|
self.train_data = paddle.batch(
|
||||||
|
paddle.dataset.imdb.train(self.word_dict),
|
||||||
|
batch_size=self.BATCH_SIZE)
|
||||||
|
|
||||||
|
def get_places(self):
|
||||||
|
places = [core.CPUPlace()]
|
||||||
|
if core.is_compiled_with_cuda():
|
||||||
|
places.append(core.CUDAPlace(0))
|
||||||
|
return places
|
||||||
|
|
||||||
|
def check_operators(self, place):
|
||||||
|
prog = fluid.framework.Program()
|
||||||
|
startup_program = fluid.framework.Program()
|
||||||
|
with fluid.program_guard(
|
||||||
|
main_program=prog, startup_program=startup_program):
|
||||||
|
image = fluid.layers.data(name='x', shape=[784], dtype='float32')
|
||||||
|
label = fluid.layers.data(name='y', shape=[1], dtype='int64')
|
||||||
|
|
||||||
|
hidden1 = fluid.layers.fc(input=image, size=128, act='relu')
|
||||||
|
hidden2 = fluid.layers.fc(input=hidden1, size=64, act='relu')
|
||||||
|
predict = fluid.layers.fc(input=hidden2, size=10, act='softmax')
|
||||||
|
|
||||||
|
cost = fluid.layers.cross_entropy(input=predict, label=label)
|
||||||
|
avg_cost = fluid.layers.mean(cost)
|
||||||
|
|
||||||
|
prog_clip = prog.clone()
|
||||||
|
|
||||||
|
avg_cost_clip = prog_clip.block(0).var(avg_cost.name)
|
||||||
|
|
||||||
|
p_g = fluid.backward.append_backward(loss=avg_cost)
|
||||||
|
p_g_clip = fluid.backward.append_backward(loss=avg_cost_clip)
|
||||||
|
|
||||||
|
with fluid.program_guard(main_program=prog_clip):
|
||||||
|
fluid.clip.set_gradient_clip(
|
||||||
|
fluid.clip.GradientClipByGlobalNorm(clip_norm=CLIP))
|
||||||
|
p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip)
|
||||||
|
|
||||||
|
grad_list = [elem[1] for elem in p_g]
|
||||||
|
grad_clip_list = [elem[1] for elem in p_g_clip]
|
||||||
|
|
||||||
|
train_reader = paddle.batch(
|
||||||
|
paddle.reader.shuffle(
|
||||||
|
paddle.dataset.mnist.train(), buf_size=8192),
|
||||||
|
batch_size=BATCH_SIZE)
|
||||||
|
|
||||||
|
exe = fluid.Executor(place)
|
||||||
|
feeder = fluid.DataFeeder(feed_list=[image, label], place=place)
|
||||||
|
exe.run(startup_program)
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
for data in train_reader():
|
||||||
|
count += 1
|
||||||
|
if count > 5:
|
||||||
|
break
|
||||||
|
out = exe.run(prog, feed=feeder.feed(data), fetch_list=grad_list)
|
||||||
|
out_clip = exe.run(prog_clip,
|
||||||
|
feed=feeder.feed(data),
|
||||||
|
fetch_list=grad_clip_list)
|
||||||
|
global_norm = 0
|
||||||
|
for v in out[1:]:
|
||||||
|
global_norm += np.sum(np.power(v, 2))
|
||||||
|
global_norm = np.sqrt(global_norm)
|
||||||
|
|
||||||
|
global_norm_clip = 0
|
||||||
|
for v in out_clip[1:]:
|
||||||
|
global_norm_clip += np.sum(np.power(v, 2))
|
||||||
|
global_norm_clip = np.sqrt(global_norm_clip)
|
||||||
|
|
||||||
|
assert np.isclose(
|
||||||
|
a=global_norm_clip, b=np.minimum(global_norm, CLIP), rtol=5e-3)
|
||||||
|
|
||||||
|
def check_sparse_gradient_clip(self, place):
|
||||||
|
prog = fluid.framework.Program()
|
||||||
|
startup_program = fluid.framework.Program()
|
||||||
|
with fluid.program_guard(
|
||||||
|
main_program=prog, startup_program=startup_program):
|
||||||
|
data = fluid.layers.data(
|
||||||
|
name="words", shape=[1], dtype="int64", lod_level=1)
|
||||||
|
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
|
||||||
|
cost = bow_net(data, label, len(self.word_dict))
|
||||||
|
|
||||||
|
fluid.clip.set_gradient_clip(
|
||||||
|
clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0))
|
||||||
|
|
||||||
|
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.01)
|
||||||
|
sgd_optimizer.minimize(cost)
|
||||||
|
|
||||||
|
exe = fluid.Executor(place)
|
||||||
|
feeder = fluid.DataFeeder(feed_list=[data, label], place=place)
|
||||||
|
exe.run(startup_program)
|
||||||
|
|
||||||
|
data = next(self.train_data())
|
||||||
|
val = exe.run(prog, feed=feeder.feed(data), fetch_list=[cost])[0]
|
||||||
|
self.assertEqual((1, ), val.shape)
|
||||||
|
print(val)
|
||||||
|
self.assertFalse(np.isnan(val))
|
||||||
|
|
||||||
|
def test_operators(self):
|
||||||
|
self.check_operators(core.CPUPlace())
|
||||||
|
|
||||||
|
def test_sparse_gradient_clip(self):
|
||||||
|
for place in self.get_places():
|
||||||
|
self.check_sparse_gradient_clip(place)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@ -0,0 +1,73 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
import paddle.fluid.core as core
|
||||||
|
import numpy as np
|
||||||
|
from paddle.fluid.op import Operator
|
||||||
|
|
||||||
|
|
||||||
|
class TestMergeSelectedRows(unittest.TestCase):
|
||||||
|
def get_places(self):
|
||||||
|
places = [core.CPUPlace()]
|
||||||
|
if core.is_compiled_with_cuda():
|
||||||
|
places.append(core.CUDAPlace(0))
|
||||||
|
return places
|
||||||
|
|
||||||
|
def check_with_place(self, place):
|
||||||
|
scope = core.Scope()
|
||||||
|
x_rows = [0, 5, 5, 4, 20]
|
||||||
|
out_rows = [0, 4, 5, 20]
|
||||||
|
height = 20
|
||||||
|
row_numel = 2
|
||||||
|
|
||||||
|
np_array = np.ones((len(x_rows), row_numel)).astype("float32")
|
||||||
|
np_array[1, :] = 2.0
|
||||||
|
np_array[2, :] = 3.0
|
||||||
|
np_array[3, :] = 4.0
|
||||||
|
|
||||||
|
# initialize input variable X
|
||||||
|
x = scope.var('X').get_selected_rows()
|
||||||
|
x.set_rows(x_rows)
|
||||||
|
x.set_height(height)
|
||||||
|
x_tensor = x.get_tensor()
|
||||||
|
x_tensor.set(np_array, place)
|
||||||
|
|
||||||
|
# initialize input variable Out
|
||||||
|
out = scope.var("Out").get_selected_rows()
|
||||||
|
|
||||||
|
op = Operator("merge_selected_rows", X="X", Out="Out")
|
||||||
|
|
||||||
|
op.run(scope, place)
|
||||||
|
|
||||||
|
self.assertEqual(out.rows(), out_rows)
|
||||||
|
self.assertEqual(out.height(), height)
|
||||||
|
|
||||||
|
out_array = np.array(out.get_tensor())
|
||||||
|
self.assertEqual((4, 2), out_array.shape)
|
||||||
|
|
||||||
|
assert (out_array[0, :] == 1.0).all()
|
||||||
|
assert (out_array[1, :] == 4.0).all()
|
||||||
|
assert (out_array[2, :] == 5.0).all()
|
||||||
|
assert (out_array[3, :] == 1.0).all()
|
||||||
|
|
||||||
|
def test_check_output(self):
|
||||||
|
for place in self.get_places():
|
||||||
|
self.check_with_place(place)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in new issue