Merge commit 'refs/pull/4537/head' of https://github.com/PaddlePaddle/Paddle into 4537
gangliao
8 years ago
commit
683ef60d7c
@ -0,0 +1,93 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/operators/adagrad_op.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
class AdagradOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
void InferShape(framework::InferShapeContextBase *ctx) const override {
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("Param"),
|
||||||
|
"Input(Param) of AdagradOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("Grad"),
|
||||||
|
"Input(Grad) of AdagradOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("Moment"),
|
||||||
|
"Input(Moment) of AdagradOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("LearningRate"),
|
||||||
|
"Input(LearningRate) of AdagradOp should not be null.");
|
||||||
|
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
|
||||||
|
"Output(ParamOut) of AdagradOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("MomentOut"),
|
||||||
|
"Output(MomentOut) of AdagradOp should not be null.");
|
||||||
|
|
||||||
|
auto lr_dims = ctx->GetInputDim("LearningRate");
|
||||||
|
PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1,
|
||||||
|
"LearningRate should have one element");
|
||||||
|
auto param_dims = ctx->GetInputDim("Param");
|
||||||
|
PADDLE_ENFORCE_EQ(
|
||||||
|
param_dims, ctx->GetInputDim("Grad"),
|
||||||
|
"Param and Grad input of AdagradOp should have the same dimension.");
|
||||||
|
PADDLE_ENFORCE_EQ(
|
||||||
|
param_dims, ctx->GetInputDim("Moment"),
|
||||||
|
"Param and Moment input of AdagradOp should have the same dimension.");
|
||||||
|
|
||||||
|
ctx->SetOutputDim("ParamOut", param_dims);
|
||||||
|
ctx->SetOutputDim("MomentOut", param_dims);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class AdagradOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||||
|
public:
|
||||||
|
AdagradOpMaker(framework::OpProto *proto,
|
||||||
|
framework::OpAttrChecker *op_checker)
|
||||||
|
: OpProtoAndCheckerMaker(proto, op_checker) {
|
||||||
|
AddInput("Param", "(Tensor) Input parameter");
|
||||||
|
AddInput("Grad", "(Tensor) Input gradient");
|
||||||
|
AddInput("Moment", "(Tensor) Second moment");
|
||||||
|
AddInput("LearningRate", "(Tensor) Learning rate");
|
||||||
|
|
||||||
|
AddOutput("ParamOut", "(Tensor) Output parameter");
|
||||||
|
AddOutput("MomentOut", "(Tensor) Output second moment");
|
||||||
|
|
||||||
|
AddAttr<float>("epsilon",
|
||||||
|
"(float, default 1.0e-6) "
|
||||||
|
"Constant for numerical stability")
|
||||||
|
.SetDefault(1.0e-6f);
|
||||||
|
AddComment(R"DOC(
|
||||||
|
|
||||||
|
Adaptive Gradient Algorithm (Adagrad).
|
||||||
|
|
||||||
|
moment_out = moment + grad * grad
|
||||||
|
param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon)
|
||||||
|
|
||||||
|
The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
|
||||||
|
does not have the epsilon attribute. It is added here for numerical stability
|
||||||
|
by avoiding division by zero.
|
||||||
|
|
||||||
|
)DOC");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
REGISTER_OP_WITHOUT_GRADIENT(adagrad, ops::AdagradOp, ops::AdagradOpMaker);
|
||||||
|
REGISTER_OP_CPU_KERNEL(adagrad,
|
||||||
|
ops::AdagradOpKernel<paddle::platform::CPUPlace, float>);
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#define EIGEN_USE_GPU
|
||||||
|
#include "paddle/operators/adagrad_op.h"
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
REGISTER_OP_GPU_KERNEL(adagrad,
|
||||||
|
ops::AdagradOpKernel<paddle::platform::GPUPlace, float>);
|
||||||
@ -0,0 +1,55 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include "paddle/framework/eigen.h"
|
||||||
|
#include "paddle/framework/op_registry.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
template <typename Place, typename T>
|
||||||
|
class AdagradOpKernel : public framework::OpKernel<T> {
|
||||||
|
public:
|
||||||
|
void Compute(const framework::ExecutionContext& ctx) const override {
|
||||||
|
auto param_out_tensor = ctx.Output<framework::Tensor>("ParamOut");
|
||||||
|
auto moment_out_tensor = ctx.Output<framework::Tensor>("MomentOut");
|
||||||
|
|
||||||
|
param_out_tensor->mutable_data<T>(ctx.GetPlace());
|
||||||
|
moment_out_tensor->mutable_data<T>(ctx.GetPlace());
|
||||||
|
|
||||||
|
float epsilon = ctx.Attr<float>("epsilon");
|
||||||
|
|
||||||
|
auto param = framework::EigenVector<T>::Flatten(
|
||||||
|
*ctx.Input<framework::Tensor>("Param"));
|
||||||
|
auto grad = framework::EigenVector<T>::Flatten(
|
||||||
|
*ctx.Input<framework::Tensor>("Grad"));
|
||||||
|
auto moment = framework::EigenVector<T>::Flatten(
|
||||||
|
*ctx.Input<framework::Tensor>("Moment"));
|
||||||
|
auto lr = framework::EigenVector<T>::Flatten(
|
||||||
|
*ctx.Input<framework::Tensor>("LearningRate"));
|
||||||
|
|
||||||
|
auto param_out = framework::EigenVector<T>::Flatten(*param_out_tensor);
|
||||||
|
auto moment_out = framework::EigenVector<T>::Flatten(*moment_out_tensor);
|
||||||
|
auto place = ctx.GetEigenDevice<Place>();
|
||||||
|
|
||||||
|
moment_out.device(place) = moment + grad * grad;
|
||||||
|
Eigen::DSizes<int, 1> m_dsize(moment_out_tensor->numel());
|
||||||
|
param_out.device(place) =
|
||||||
|
param - lr.broadcast(m_dsize) * grad / (moment_out.sqrt() + epsilon);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,120 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/operators/rmsprop_op.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
class RmspropOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
void InferShape(framework::InferShapeContextBase *ctx) const override {
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("Param"),
|
||||||
|
"Input(Param) of RmspropOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("MeanSquare"),
|
||||||
|
"Input(MeanSquare) of RmspropOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("LearningRate"),
|
||||||
|
"Input(LearningRate) of RmspropOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("Grad"),
|
||||||
|
"Input(Grad) of RmspropOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("Moment"),
|
||||||
|
"Input(Moment) of RmspropOp should not be null.");
|
||||||
|
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
|
||||||
|
"Output(param_out) of RmspropOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("MomentOut"),
|
||||||
|
"Output(Momentum_out) of RmspropOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("MeanSquareOut"),
|
||||||
|
"Output(MeanSquareOut) of RmspropOp should not be null.");
|
||||||
|
|
||||||
|
auto param_dim = ctx->GetInputDim("Param");
|
||||||
|
PADDLE_ENFORCE_EQ(
|
||||||
|
param_dim, ctx->GetInputDim("Grad"),
|
||||||
|
"Param and grad input of RmspropOp should have the same dimension.");
|
||||||
|
PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("Moment"),
|
||||||
|
"Param and Momentum input of RmspropOp "
|
||||||
|
"should have the same dimension.");
|
||||||
|
PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("MeanSquare"),
|
||||||
|
"Param and Momentum input of RmspropOp "
|
||||||
|
"should have the same dimension.");
|
||||||
|
|
||||||
|
auto lr_dim = ctx->GetInputDim("LearningRate");
|
||||||
|
PADDLE_ENFORCE_EQ(framework::product(lr_dim), 1,
|
||||||
|
"Learning Rate should be a scalar.");
|
||||||
|
|
||||||
|
ctx->SetOutputDim("ParamOut", param_dim);
|
||||||
|
ctx->SetOutputDim("MomentOut", param_dim);
|
||||||
|
ctx->SetOutputDim("MeanSquareOut", param_dim);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class RmspropOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||||
|
public:
|
||||||
|
RmspropOpMaker(framework::OpProto *proto,
|
||||||
|
framework::OpAttrChecker *op_checker)
|
||||||
|
: OpProtoAndCheckerMaker(proto, op_checker) {
|
||||||
|
AddInput("Param",
|
||||||
|
"(Tensor, default Tensor<float>) "
|
||||||
|
"Input parameter value that has to be updated");
|
||||||
|
AddInput("MeanSquare",
|
||||||
|
"(Tensor, default Tensor<float>)"
|
||||||
|
" The mean square value that gets updated");
|
||||||
|
AddInput("LearningRate",
|
||||||
|
"(Tensor, default Tensor<float>) "
|
||||||
|
"The learning rate should be a tensor of size 1");
|
||||||
|
AddInput("Grad",
|
||||||
|
"(Tensor, default Tensor<float>) "
|
||||||
|
"Input gradient of the parameter");
|
||||||
|
AddInput("Moment",
|
||||||
|
"(Tensor, default Tensor<float>) The moment that gets updated");
|
||||||
|
|
||||||
|
AddOutput("ParamOut", "(Tensor) Output updated parameter value");
|
||||||
|
AddOutput("MomentOut", "(Tensor) Output updated moment");
|
||||||
|
AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value");
|
||||||
|
|
||||||
|
AddAttr<float>("epsilon",
|
||||||
|
"(float, default 1e-10) Constant "
|
||||||
|
"for numerical stability.")
|
||||||
|
.SetDefault(1.0e-10f);
|
||||||
|
AddAttr<float>("decay",
|
||||||
|
"(float, default 0.9) "
|
||||||
|
"Discounting factor for coming gradient.")
|
||||||
|
.SetDefault(0.9f);
|
||||||
|
AddAttr<float>("momentum", "(float, default 0.0) Constant value")
|
||||||
|
.SetDefault(0.0f);
|
||||||
|
AddComment(R"DOC(
|
||||||
|
|
||||||
|
RMSprop
|
||||||
|
|
||||||
|
MeanSquareOut = decay * MeanSquare + (1 - decay) * Grad * Grad
|
||||||
|
MomentOut = momentum * Moment +
|
||||||
|
LearningRate * Grad / sqrt(MeanSquareOut + epsilon)
|
||||||
|
ParamOut = Param - MomentOut
|
||||||
|
|
||||||
|
The original slides that proposed RMSprop: Slide 29 of
|
||||||
|
http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf)
|
||||||
|
|
||||||
|
)DOC");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
REGISTER_OP_WITHOUT_GRADIENT(rmsprop, ops::RmspropOp, ops::RmspropOpMaker);
|
||||||
|
REGISTER_OP_CPU_KERNEL(rmsprop,
|
||||||
|
ops::RmspropOpKernel<paddle::platform::CPUPlace, float>);
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#define EIGEN_USE_GPU
|
||||||
|
#include "paddle/operators/rmsprop_op.h"
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
REGISTER_OP_GPU_KERNEL(rmsprop,
|
||||||
|
ops::RmspropOpKernel<paddle::platform::GPUPlace, float>);
|
||||||
@ -0,0 +1,67 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include "paddle/framework/eigen.h"
|
||||||
|
#include "paddle/framework/op_registry.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
using Tensor = framework::Tensor;
|
||||||
|
template <typename T, int MajorType = Eigen::RowMajor,
|
||||||
|
typename IndexType = Eigen::DenseIndex>
|
||||||
|
using EigenVector = framework::EigenVector<T, MajorType, IndexType>;
|
||||||
|
|
||||||
|
template <typename Place, typename T>
|
||||||
|
class RmspropOpKernel : public framework::OpKernel<T> {
|
||||||
|
public:
|
||||||
|
void Compute(const framework::ExecutionContext& ctx) const override {
|
||||||
|
auto* param_out = ctx.Output<Tensor>("ParamOut");
|
||||||
|
auto* moment_out = ctx.Output<Tensor>("MomentOut");
|
||||||
|
auto* mean_square_out = ctx.Output<Tensor>("MeanSquareOut");
|
||||||
|
|
||||||
|
auto grad = ctx.Input<Tensor>("Grad");
|
||||||
|
|
||||||
|
param_out->mutable_data<T>(ctx.GetPlace());
|
||||||
|
moment_out->mutable_data<T>(ctx.GetPlace());
|
||||||
|
mean_square_out->mutable_data<T>(ctx.GetPlace());
|
||||||
|
|
||||||
|
float epsilon = ctx.Attr<float>("epsilon");
|
||||||
|
float rho = ctx.Attr<float>("decay");
|
||||||
|
float momentum = ctx.Attr<float>("momentum");
|
||||||
|
|
||||||
|
auto p = EigenVector<T>::Flatten(*ctx.Input<Tensor>("Param"));
|
||||||
|
auto ms = EigenVector<T>::Flatten(*ctx.Input<Tensor>("MeanSquare"));
|
||||||
|
auto lr = EigenVector<T>::Flatten(*ctx.Input<Tensor>("LearningRate"));
|
||||||
|
auto g = EigenVector<T>::Flatten(*grad);
|
||||||
|
auto mom = EigenVector<T>::Flatten(*ctx.Input<Tensor>("Moment"));
|
||||||
|
|
||||||
|
auto p_out = EigenVector<T>::Flatten(*param_out);
|
||||||
|
auto mom_out = EigenVector<T>::Flatten(*moment_out);
|
||||||
|
auto ms_out = EigenVector<T>::Flatten(*mean_square_out);
|
||||||
|
auto place = ctx.GetEigenDevice<Place>();
|
||||||
|
|
||||||
|
Eigen::DSizes<int, 1> grad_dsize(grad->numel());
|
||||||
|
|
||||||
|
ms_out.device(place) = rho * ms + (1 - rho) * g * g;
|
||||||
|
mom_out.device(place) =
|
||||||
|
momentum * mom +
|
||||||
|
lr.broadcast(grad_dsize) * g / (ms_out + epsilon).sqrt();
|
||||||
|
p_out.device(place) = p - mom_out;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,69 @@
|
|||||||
|
import unittest
|
||||||
|
import numpy as np
|
||||||
|
from op_test import OpTest
|
||||||
|
|
||||||
|
|
||||||
|
class TestAdagradOp1(OpTest):
|
||||||
|
''' Test Adagrad operator with explicit attributes
|
||||||
|
'''
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.op_type = "adagrad"
|
||||||
|
|
||||||
|
param = np.random.random((123, 321)).astype("float32")
|
||||||
|
grad = np.random.random((123, 321)).astype("float32")
|
||||||
|
moment = np.zeros((123, 321)).astype("float32")
|
||||||
|
lr = 0.01
|
||||||
|
epsilon = 1e-8
|
||||||
|
|
||||||
|
self.inputs = {
|
||||||
|
'Param': param,
|
||||||
|
'Grad': grad,
|
||||||
|
'Moment': moment,
|
||||||
|
'LearningRate': np.array([lr]).astype("float32")
|
||||||
|
}
|
||||||
|
|
||||||
|
self.attrs = {'epsilon': epsilon}
|
||||||
|
|
||||||
|
moment_out = moment + grad * grad
|
||||||
|
param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon)
|
||||||
|
|
||||||
|
self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out}
|
||||||
|
|
||||||
|
def test_check_output(self):
|
||||||
|
self.check_output()
|
||||||
|
|
||||||
|
|
||||||
|
class TestAdagradOp2(OpTest):
|
||||||
|
''' Test Adagrad operator with default attributes
|
||||||
|
'''
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.op_type = "adagrad"
|
||||||
|
|
||||||
|
param = np.random.random((123, 321)).astype("float32")
|
||||||
|
grad = np.random.random((123, 321)).astype("float32")
|
||||||
|
moment = np.zeros((123, 321)).astype("float32")
|
||||||
|
lr = 0.01
|
||||||
|
epsilon = 1e-6
|
||||||
|
|
||||||
|
self.inputs = {
|
||||||
|
'Param': param,
|
||||||
|
'Grad': grad,
|
||||||
|
'Moment': moment,
|
||||||
|
'LearningRate': np.array([lr]).astype("float32")
|
||||||
|
}
|
||||||
|
|
||||||
|
self.attrs = {'epsilon': epsilon}
|
||||||
|
|
||||||
|
moment_out = moment + grad * grad
|
||||||
|
param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon)
|
||||||
|
|
||||||
|
self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out}
|
||||||
|
|
||||||
|
def test_check_output(self):
|
||||||
|
self.check_output()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@ -0,0 +1,63 @@
|
|||||||
|
import unittest
|
||||||
|
import paddle.v2.framework.core as core
|
||||||
|
from paddle.v2.framework.op import Operator
|
||||||
|
|
||||||
|
|
||||||
|
class TestInferShape(unittest.TestCase):
|
||||||
|
def test_sum_op(self):
|
||||||
|
prog = core.ProgramDesc.__create_program_desc__()
|
||||||
|
self.assertIsNotNone(prog)
|
||||||
|
block = prog.block(0)
|
||||||
|
self.assertIsNotNone(block)
|
||||||
|
|
||||||
|
shape = [10, 20]
|
||||||
|
|
||||||
|
# prepare input/output
|
||||||
|
x1 = block.new_var("x1")
|
||||||
|
x1.set_shape(shape)
|
||||||
|
x2 = block.new_var("x2")
|
||||||
|
x2.set_shape(shape)
|
||||||
|
|
||||||
|
out = block.new_var("out")
|
||||||
|
|
||||||
|
# prepare the operator
|
||||||
|
sum_op_desc = block.append_op()
|
||||||
|
sum_op_desc.set_type("sum")
|
||||||
|
sum_op_desc.set_input("X", ["x1", "x2"])
|
||||||
|
sum_op_desc.set_output("Out", ["out"])
|
||||||
|
|
||||||
|
core.Operator.infer_shape(sum_op_desc, block)
|
||||||
|
self.assertEqual(out.shape(), shape)
|
||||||
|
|
||||||
|
def test_mul_op(self):
|
||||||
|
prog = core.ProgramDesc.__create_program_desc__()
|
||||||
|
self.assertIsNotNone(prog)
|
||||||
|
block = prog.block(0)
|
||||||
|
self.assertIsNotNone(block)
|
||||||
|
|
||||||
|
x_shape = [10, 20]
|
||||||
|
y_shape = [20, 30]
|
||||||
|
|
||||||
|
# prepare input/output
|
||||||
|
x1 = block.new_var("x")
|
||||||
|
x1.set_shape(x_shape)
|
||||||
|
x2 = block.new_var("y")
|
||||||
|
x2.set_shape(y_shape)
|
||||||
|
|
||||||
|
out = block.new_var("out")
|
||||||
|
|
||||||
|
# prepare the operator
|
||||||
|
mul_op_desc = block.append_op()
|
||||||
|
mul_op_desc.set_type("mul")
|
||||||
|
mul_op_desc.set_input("X", ["x"])
|
||||||
|
mul_op_desc.set_input("Y", ["y"])
|
||||||
|
mul_op_desc.set_output("Out", ["out"])
|
||||||
|
mul_op_desc.set_attr("x_num_col_dims", 1)
|
||||||
|
mul_op_desc.set_attr("y_num_col_dims", 1)
|
||||||
|
|
||||||
|
core.Operator.infer_shape(mul_op_desc, block)
|
||||||
|
self.assertEqual(out.shape(), [x_shape[0], y_shape[1]])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@ -0,0 +1,89 @@
|
|||||||
|
import unittest
|
||||||
|
import numpy as np
|
||||||
|
from op_test import OpTest
|
||||||
|
|
||||||
|
|
||||||
|
class TestRmspropOp1(OpTest):
|
||||||
|
''' Test RMSProp with explicit inputs
|
||||||
|
'''
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.op_type = "rmsprop"
|
||||||
|
|
||||||
|
param = np.random.random((123, 321)).astype("float32")
|
||||||
|
mean_square = np.random.random((123, 321)).astype("float32")
|
||||||
|
learning_rate = np.array([0.01]).astype("float32")
|
||||||
|
grad = np.random.random((123, 321)).astype("float32")
|
||||||
|
moment = np.zeros((123, 321)).astype("float32")
|
||||||
|
|
||||||
|
epsilon = 1e-6
|
||||||
|
decay = 0.9
|
||||||
|
momentum = 0.0
|
||||||
|
|
||||||
|
self.inputs = {
|
||||||
|
'Param': param,
|
||||||
|
'MeanSquare': mean_square,
|
||||||
|
'LearningRate': learning_rate,
|
||||||
|
'Grad': grad,
|
||||||
|
'Moment': moment,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.attrs = {'epsilon': epsilon, 'decay': decay, 'momentum': momentum}
|
||||||
|
|
||||||
|
ms_out = decay * mean_square + (1 - decay) * grad * grad
|
||||||
|
moment_out = momentum * moment + \
|
||||||
|
learning_rate * grad / np.sqrt(ms_out + epsilon)
|
||||||
|
param_out = param - moment_out
|
||||||
|
|
||||||
|
self.outputs = {
|
||||||
|
'ParamOut': param_out,
|
||||||
|
'MomentOut': moment_out,
|
||||||
|
'MeanSquareOut': ms_out
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_check_output(self):
|
||||||
|
self.check_output()
|
||||||
|
|
||||||
|
|
||||||
|
class TestRmspropOp2(OpTest):
|
||||||
|
'''Test RMSProp with defaukt values for attributes
|
||||||
|
'''
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.op_type = "rmsprop"
|
||||||
|
|
||||||
|
param = np.random.random((123, 321)).astype("float32")
|
||||||
|
mean_square = np.random.random((123, 321)).astype("float32")
|
||||||
|
learning_rate = np.array([0.01]).astype("float32")
|
||||||
|
grad = np.random.random((123, 321)).astype("float32")
|
||||||
|
moment = np.zeros((123, 321)).astype("float32")
|
||||||
|
|
||||||
|
epsilon = 1.0e-10
|
||||||
|
decay = 0.9
|
||||||
|
momentum = 0.0
|
||||||
|
|
||||||
|
self.inputs = {
|
||||||
|
'Param': param,
|
||||||
|
'MeanSquare': mean_square,
|
||||||
|
'LearningRate': learning_rate,
|
||||||
|
'Grad': grad,
|
||||||
|
'Moment': moment,
|
||||||
|
}
|
||||||
|
|
||||||
|
ms_out = decay * mean_square + (1 - decay) * grad * grad
|
||||||
|
moment_out = momentum * moment + \
|
||||||
|
learning_rate * grad / np.sqrt(ms_out + epsilon)
|
||||||
|
param_out = param - moment_out
|
||||||
|
|
||||||
|
self.outputs = {
|
||||||
|
'ParamOut': param_out,
|
||||||
|
'MomentOut': moment_out,
|
||||||
|
'MeanSquareOut': ms_out
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_check_output(self):
|
||||||
|
self.check_output()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in new issue