qijun
8 years ago
commit
f087533cc3
@ -0,0 +1,93 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/operators/adagrad_op.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class AdagradOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
|
||||
protected:
|
||||
void InferShape(framework::InferShapeContextBase *ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasInput("Param"),
|
||||
"Input(Param) of AdagradOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("Grad"),
|
||||
"Input(Grad) of AdagradOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("Moment"),
|
||||
"Input(Moment) of AdagradOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("LearningRate"),
|
||||
"Input(LearningRate) of AdagradOp should not be null.");
|
||||
|
||||
PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
|
||||
"Output(ParamOut) of AdagradOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput("MomentOut"),
|
||||
"Output(MomentOut) of AdagradOp should not be null.");
|
||||
|
||||
auto lr_dims = ctx->GetInputDim("LearningRate");
|
||||
PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1,
|
||||
"LearningRate should have one element");
|
||||
auto param_dims = ctx->GetInputDim("Param");
|
||||
PADDLE_ENFORCE_EQ(
|
||||
param_dims, ctx->GetInputDim("Grad"),
|
||||
"Param and Grad input of AdagradOp should have the same dimension.");
|
||||
PADDLE_ENFORCE_EQ(
|
||||
param_dims, ctx->GetInputDim("Moment"),
|
||||
"Param and Moment input of AdagradOp should have the same dimension.");
|
||||
|
||||
ctx->SetOutputDim("ParamOut", param_dims);
|
||||
ctx->SetOutputDim("MomentOut", param_dims);
|
||||
}
|
||||
};
|
||||
|
||||
class AdagradOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
AdagradOpMaker(framework::OpProto *proto,
|
||||
framework::OpAttrChecker *op_checker)
|
||||
: OpProtoAndCheckerMaker(proto, op_checker) {
|
||||
AddInput("Param", "(Tensor) Input parameter");
|
||||
AddInput("Grad", "(Tensor) Input gradient");
|
||||
AddInput("Moment", "(Tensor) Second moment");
|
||||
AddInput("LearningRate", "(Tensor) Learning rate");
|
||||
|
||||
AddOutput("ParamOut", "(Tensor) Output parameter");
|
||||
AddOutput("MomentOut", "(Tensor) Output second moment");
|
||||
|
||||
AddAttr<float>("epsilon",
|
||||
"(float, default 1.0e-6) "
|
||||
"Constant for numerical stability")
|
||||
.SetDefault(1.0e-6f);
|
||||
AddComment(R"DOC(
|
||||
|
||||
Adaptive Gradient Algorithm (Adagrad).
|
||||
|
||||
moment_out = moment + grad * grad
|
||||
param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon)
|
||||
|
||||
The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
|
||||
does not have the epsilon attribute. It is added here for numerical stability
|
||||
by avoiding division by zero.
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
REGISTER_OP_WITHOUT_GRADIENT(adagrad, ops::AdagradOp, ops::AdagradOpMaker);
|
||||
REGISTER_OP_CPU_KERNEL(adagrad,
|
||||
ops::AdagradOpKernel<paddle::platform::CPUPlace, float>);
|
||||
@ -0,0 +1,20 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
#include "paddle/operators/adagrad_op.h"
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
REGISTER_OP_GPU_KERNEL(adagrad,
|
||||
ops::AdagradOpKernel<paddle::platform::GPUPlace, float>);
|
||||
@ -0,0 +1,55 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include "paddle/framework/eigen.h"
|
||||
#include "paddle/framework/op_registry.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
template <typename Place, typename T>
|
||||
class AdagradOpKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& ctx) const override {
|
||||
auto param_out_tensor = ctx.Output<framework::Tensor>("ParamOut");
|
||||
auto moment_out_tensor = ctx.Output<framework::Tensor>("MomentOut");
|
||||
|
||||
param_out_tensor->mutable_data<T>(ctx.GetPlace());
|
||||
moment_out_tensor->mutable_data<T>(ctx.GetPlace());
|
||||
|
||||
float epsilon = ctx.Attr<float>("epsilon");
|
||||
|
||||
auto param = framework::EigenVector<T>::Flatten(
|
||||
*ctx.Input<framework::Tensor>("Param"));
|
||||
auto grad = framework::EigenVector<T>::Flatten(
|
||||
*ctx.Input<framework::Tensor>("Grad"));
|
||||
auto moment = framework::EigenVector<T>::Flatten(
|
||||
*ctx.Input<framework::Tensor>("Moment"));
|
||||
auto lr = framework::EigenVector<T>::Flatten(
|
||||
*ctx.Input<framework::Tensor>("LearningRate"));
|
||||
|
||||
auto param_out = framework::EigenVector<T>::Flatten(*param_out_tensor);
|
||||
auto moment_out = framework::EigenVector<T>::Flatten(*moment_out_tensor);
|
||||
auto place = ctx.GetEigenDevice<Place>();
|
||||
|
||||
moment_out.device(place) = moment + grad * grad;
|
||||
Eigen::DSizes<int, 1> m_dsize(moment_out_tensor->numel());
|
||||
param_out.device(place) =
|
||||
param - lr.broadcast(m_dsize) * grad / (moment_out.sqrt() + epsilon);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,120 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/operators/rmsprop_op.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class RmspropOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
|
||||
protected:
|
||||
void InferShape(framework::InferShapeContextBase *ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasInput("Param"),
|
||||
"Input(Param) of RmspropOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("MeanSquare"),
|
||||
"Input(MeanSquare) of RmspropOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("LearningRate"),
|
||||
"Input(LearningRate) of RmspropOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("Grad"),
|
||||
"Input(Grad) of RmspropOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("Moment"),
|
||||
"Input(Moment) of RmspropOp should not be null.");
|
||||
|
||||
PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
|
||||
"Output(param_out) of RmspropOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput("MomentOut"),
|
||||
"Output(Momentum_out) of RmspropOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput("MeanSquareOut"),
|
||||
"Output(MeanSquareOut) of RmspropOp should not be null.");
|
||||
|
||||
auto param_dim = ctx->GetInputDim("Param");
|
||||
PADDLE_ENFORCE_EQ(
|
||||
param_dim, ctx->GetInputDim("Grad"),
|
||||
"Param and grad input of RmspropOp should have the same dimension.");
|
||||
PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("Moment"),
|
||||
"Param and Momentum input of RmspropOp "
|
||||
"should have the same dimension.");
|
||||
PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("MeanSquare"),
|
||||
"Param and Momentum input of RmspropOp "
|
||||
"should have the same dimension.");
|
||||
|
||||
auto lr_dim = ctx->GetInputDim("LearningRate");
|
||||
PADDLE_ENFORCE_EQ(framework::product(lr_dim), 1,
|
||||
"Learning Rate should be a scalar.");
|
||||
|
||||
ctx->SetOutputDim("ParamOut", param_dim);
|
||||
ctx->SetOutputDim("MomentOut", param_dim);
|
||||
ctx->SetOutputDim("MeanSquareOut", param_dim);
|
||||
}
|
||||
};
|
||||
|
||||
class RmspropOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
RmspropOpMaker(framework::OpProto *proto,
|
||||
framework::OpAttrChecker *op_checker)
|
||||
: OpProtoAndCheckerMaker(proto, op_checker) {
|
||||
AddInput("Param",
|
||||
"(Tensor, default Tensor<float>) "
|
||||
"Input parameter value that has to be updated");
|
||||
AddInput("MeanSquare",
|
||||
"(Tensor, default Tensor<float>)"
|
||||
" The mean square value that gets updated");
|
||||
AddInput("LearningRate",
|
||||
"(Tensor, default Tensor<float>) "
|
||||
"The learning rate should be a tensor of size 1");
|
||||
AddInput("Grad",
|
||||
"(Tensor, default Tensor<float>) "
|
||||
"Input gradient of the parameter");
|
||||
AddInput("Moment",
|
||||
"(Tensor, default Tensor<float>) The moment that gets updated");
|
||||
|
||||
AddOutput("ParamOut", "(Tensor) Output updated parameter value");
|
||||
AddOutput("MomentOut", "(Tensor) Output updated moment");
|
||||
AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value");
|
||||
|
||||
AddAttr<float>("epsilon",
|
||||
"(float, default 1e-10) Constant "
|
||||
"for numerical stability.")
|
||||
.SetDefault(1.0e-10f);
|
||||
AddAttr<float>("decay",
|
||||
"(float, default 0.9) "
|
||||
"Discounting factor for coming gradient.")
|
||||
.SetDefault(0.9f);
|
||||
AddAttr<float>("momentum", "(float, default 0.0) Constant value")
|
||||
.SetDefault(0.0f);
|
||||
AddComment(R"DOC(
|
||||
|
||||
RMSprop
|
||||
|
||||
MeanSquareOut = decay * MeanSquare + (1 - decay) * Grad * Grad
|
||||
MomentOut = momentum * Moment +
|
||||
LearningRate * Grad / sqrt(MeanSquareOut + epsilon)
|
||||
ParamOut = Param - MomentOut
|
||||
|
||||
The original slides that proposed RMSprop: Slide 29 of
|
||||
http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf)
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
REGISTER_OP_WITHOUT_GRADIENT(rmsprop, ops::RmspropOp, ops::RmspropOpMaker);
|
||||
REGISTER_OP_CPU_KERNEL(rmsprop,
|
||||
ops::RmspropOpKernel<paddle::platform::CPUPlace, float>);
|
||||
@ -0,0 +1,20 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
#include "paddle/operators/rmsprop_op.h"
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
REGISTER_OP_GPU_KERNEL(rmsprop,
|
||||
ops::RmspropOpKernel<paddle::platform::GPUPlace, float>);
|
||||
@ -0,0 +1,67 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include "paddle/framework/eigen.h"
|
||||
#include "paddle/framework/op_registry.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
using Tensor = framework::Tensor;
|
||||
template <typename T, int MajorType = Eigen::RowMajor,
|
||||
typename IndexType = Eigen::DenseIndex>
|
||||
using EigenVector = framework::EigenVector<T, MajorType, IndexType>;
|
||||
|
||||
template <typename Place, typename T>
|
||||
class RmspropOpKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& ctx) const override {
|
||||
auto* param_out = ctx.Output<Tensor>("ParamOut");
|
||||
auto* moment_out = ctx.Output<Tensor>("MomentOut");
|
||||
auto* mean_square_out = ctx.Output<Tensor>("MeanSquareOut");
|
||||
|
||||
auto grad = ctx.Input<Tensor>("Grad");
|
||||
|
||||
param_out->mutable_data<T>(ctx.GetPlace());
|
||||
moment_out->mutable_data<T>(ctx.GetPlace());
|
||||
mean_square_out->mutable_data<T>(ctx.GetPlace());
|
||||
|
||||
float epsilon = ctx.Attr<float>("epsilon");
|
||||
float rho = ctx.Attr<float>("decay");
|
||||
float momentum = ctx.Attr<float>("momentum");
|
||||
|
||||
auto p = EigenVector<T>::Flatten(*ctx.Input<Tensor>("Param"));
|
||||
auto ms = EigenVector<T>::Flatten(*ctx.Input<Tensor>("MeanSquare"));
|
||||
auto lr = EigenVector<T>::Flatten(*ctx.Input<Tensor>("LearningRate"));
|
||||
auto g = EigenVector<T>::Flatten(*grad);
|
||||
auto mom = EigenVector<T>::Flatten(*ctx.Input<Tensor>("Moment"));
|
||||
|
||||
auto p_out = EigenVector<T>::Flatten(*param_out);
|
||||
auto mom_out = EigenVector<T>::Flatten(*moment_out);
|
||||
auto ms_out = EigenVector<T>::Flatten(*mean_square_out);
|
||||
auto place = ctx.GetEigenDevice<Place>();
|
||||
|
||||
Eigen::DSizes<int, 1> grad_dsize(grad->numel());
|
||||
|
||||
ms_out.device(place) = rho * ms + (1 - rho) * g * g;
|
||||
mom_out.device(place) =
|
||||
momentum * mom +
|
||||
lr.broadcast(grad_dsize) * g / (ms_out + epsilon).sqrt();
|
||||
p_out.device(place) = p - mom_out;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,69 @@
|
||||
import unittest
|
||||
import numpy as np
|
||||
from op_test import OpTest
|
||||
|
||||
|
||||
class TestAdagradOp1(OpTest):
|
||||
''' Test Adagrad operator with explicit attributes
|
||||
'''
|
||||
|
||||
def setUp(self):
|
||||
self.op_type = "adagrad"
|
||||
|
||||
param = np.random.random((123, 321)).astype("float32")
|
||||
grad = np.random.random((123, 321)).astype("float32")
|
||||
moment = np.zeros((123, 321)).astype("float32")
|
||||
lr = 0.01
|
||||
epsilon = 1e-8
|
||||
|
||||
self.inputs = {
|
||||
'Param': param,
|
||||
'Grad': grad,
|
||||
'Moment': moment,
|
||||
'LearningRate': np.array([lr]).astype("float32")
|
||||
}
|
||||
|
||||
self.attrs = {'epsilon': epsilon}
|
||||
|
||||
moment_out = moment + grad * grad
|
||||
param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon)
|
||||
|
||||
self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out}
|
||||
|
||||
def test_check_output(self):
|
||||
self.check_output()
|
||||
|
||||
|
||||
class TestAdagradOp2(OpTest):
|
||||
''' Test Adagrad operator with default attributes
|
||||
'''
|
||||
|
||||
def setUp(self):
|
||||
self.op_type = "adagrad"
|
||||
|
||||
param = np.random.random((123, 321)).astype("float32")
|
||||
grad = np.random.random((123, 321)).astype("float32")
|
||||
moment = np.zeros((123, 321)).astype("float32")
|
||||
lr = 0.01
|
||||
epsilon = 1e-6
|
||||
|
||||
self.inputs = {
|
||||
'Param': param,
|
||||
'Grad': grad,
|
||||
'Moment': moment,
|
||||
'LearningRate': np.array([lr]).astype("float32")
|
||||
}
|
||||
|
||||
self.attrs = {'epsilon': epsilon}
|
||||
|
||||
moment_out = moment + grad * grad
|
||||
param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon)
|
||||
|
||||
self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out}
|
||||
|
||||
def test_check_output(self):
|
||||
self.check_output()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@ -0,0 +1,63 @@
|
||||
import unittest
|
||||
import paddle.v2.framework.core as core
|
||||
from paddle.v2.framework.op import Operator
|
||||
|
||||
|
||||
class TestInferShape(unittest.TestCase):
|
||||
def test_sum_op(self):
|
||||
prog = core.ProgramDesc.__create_program_desc__()
|
||||
self.assertIsNotNone(prog)
|
||||
block = prog.block(0)
|
||||
self.assertIsNotNone(block)
|
||||
|
||||
shape = [10, 20]
|
||||
|
||||
# prepare input/output
|
||||
x1 = block.new_var("x1")
|
||||
x1.set_shape(shape)
|
||||
x2 = block.new_var("x2")
|
||||
x2.set_shape(shape)
|
||||
|
||||
out = block.new_var("out")
|
||||
|
||||
# prepare the operator
|
||||
sum_op_desc = block.append_op()
|
||||
sum_op_desc.set_type("sum")
|
||||
sum_op_desc.set_input("X", ["x1", "x2"])
|
||||
sum_op_desc.set_output("Out", ["out"])
|
||||
|
||||
core.Operator.infer_shape(sum_op_desc, block)
|
||||
self.assertEqual(out.shape(), shape)
|
||||
|
||||
def test_mul_op(self):
|
||||
prog = core.ProgramDesc.__create_program_desc__()
|
||||
self.assertIsNotNone(prog)
|
||||
block = prog.block(0)
|
||||
self.assertIsNotNone(block)
|
||||
|
||||
x_shape = [10, 20]
|
||||
y_shape = [20, 30]
|
||||
|
||||
# prepare input/output
|
||||
x1 = block.new_var("x")
|
||||
x1.set_shape(x_shape)
|
||||
x2 = block.new_var("y")
|
||||
x2.set_shape(y_shape)
|
||||
|
||||
out = block.new_var("out")
|
||||
|
||||
# prepare the operator
|
||||
mul_op_desc = block.append_op()
|
||||
mul_op_desc.set_type("mul")
|
||||
mul_op_desc.set_input("X", ["x"])
|
||||
mul_op_desc.set_input("Y", ["y"])
|
||||
mul_op_desc.set_output("Out", ["out"])
|
||||
mul_op_desc.set_attr("x_num_col_dims", 1)
|
||||
mul_op_desc.set_attr("y_num_col_dims", 1)
|
||||
|
||||
core.Operator.infer_shape(mul_op_desc, block)
|
||||
self.assertEqual(out.shape(), [x_shape[0], y_shape[1]])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,89 @@
|
||||
import unittest
|
||||
import numpy as np
|
||||
from op_test import OpTest
|
||||
|
||||
|
||||
class TestRmspropOp1(OpTest):
|
||||
''' Test RMSProp with explicit inputs
|
||||
'''
|
||||
|
||||
def setUp(self):
|
||||
self.op_type = "rmsprop"
|
||||
|
||||
param = np.random.random((123, 321)).astype("float32")
|
||||
mean_square = np.random.random((123, 321)).astype("float32")
|
||||
learning_rate = np.array([0.01]).astype("float32")
|
||||
grad = np.random.random((123, 321)).astype("float32")
|
||||
moment = np.zeros((123, 321)).astype("float32")
|
||||
|
||||
epsilon = 1e-6
|
||||
decay = 0.9
|
||||
momentum = 0.0
|
||||
|
||||
self.inputs = {
|
||||
'Param': param,
|
||||
'MeanSquare': mean_square,
|
||||
'LearningRate': learning_rate,
|
||||
'Grad': grad,
|
||||
'Moment': moment,
|
||||
}
|
||||
|
||||
self.attrs = {'epsilon': epsilon, 'decay': decay, 'momentum': momentum}
|
||||
|
||||
ms_out = decay * mean_square + (1 - decay) * grad * grad
|
||||
moment_out = momentum * moment + \
|
||||
learning_rate * grad / np.sqrt(ms_out + epsilon)
|
||||
param_out = param - moment_out
|
||||
|
||||
self.outputs = {
|
||||
'ParamOut': param_out,
|
||||
'MomentOut': moment_out,
|
||||
'MeanSquareOut': ms_out
|
||||
}
|
||||
|
||||
def test_check_output(self):
|
||||
self.check_output()
|
||||
|
||||
|
||||
class TestRmspropOp2(OpTest):
|
||||
'''Test RMSProp with defaukt values for attributes
|
||||
'''
|
||||
|
||||
def setUp(self):
|
||||
self.op_type = "rmsprop"
|
||||
|
||||
param = np.random.random((123, 321)).astype("float32")
|
||||
mean_square = np.random.random((123, 321)).astype("float32")
|
||||
learning_rate = np.array([0.01]).astype("float32")
|
||||
grad = np.random.random((123, 321)).astype("float32")
|
||||
moment = np.zeros((123, 321)).astype("float32")
|
||||
|
||||
epsilon = 1.0e-10
|
||||
decay = 0.9
|
||||
momentum = 0.0
|
||||
|
||||
self.inputs = {
|
||||
'Param': param,
|
||||
'MeanSquare': mean_square,
|
||||
'LearningRate': learning_rate,
|
||||
'Grad': grad,
|
||||
'Moment': moment,
|
||||
}
|
||||
|
||||
ms_out = decay * mean_square + (1 - decay) * grad * grad
|
||||
moment_out = momentum * moment + \
|
||||
learning_rate * grad / np.sqrt(ms_out + epsilon)
|
||||
param_out = param - moment_out
|
||||
|
||||
self.outputs = {
|
||||
'ParamOut': param_out,
|
||||
'MomentOut': moment_out,
|
||||
'MeanSquareOut': ms_out
|
||||
}
|
||||
|
||||
def test_check_output(self):
|
||||
self.check_output()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue