Merge branch 'fix-optimizer-accumulator' of ssh://github.com/jacquesqiao/Paddle into distribute-transpiler-handle-adam-accumulator
qiaolongfei
7 years ago
commit
39d88ebc02
@ -0,0 +1,167 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/hierarchical_sigmoid_op.h"
|
||||
#include <vector>
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
/**
|
||||
* Organize the classes into a binary tree. At each node, a sigmoid function
|
||||
* is used to calculate the probability of belonging to the right branch.
|
||||
* This idea is from "F. Morin, Y. Bengio (AISTATS 05):
|
||||
* Hierarchical Probabilistic Neural Network Language Model."
|
||||
*
|
||||
* Here we uses a simple way of making the binary tree.
|
||||
* Assuming the number of classes C = 6,
|
||||
* The classes are organized as a binary tree in the following way:
|
||||
*
|
||||
* @code{.py}
|
||||
* *-*-*- 2
|
||||
* | | |- 3
|
||||
* | |
|
||||
* | |-*- 4
|
||||
* | |- 5
|
||||
* |
|
||||
* |-*- 0
|
||||
* |- 1
|
||||
* @endcode
|
||||
*
|
||||
* where * indicates an internal node, and each leaf node represents a class.
|
||||
* - Node 0 ... C-2 are internal nodes.
|
||||
* - Node C-1 ... 2C-2 are leaf nodes.
|
||||
* - Class c is represented by leaf node \f$c+C-1\f$.
|
||||
*
|
||||
* We assign an id for each node:
|
||||
* - the id of root be 0.
|
||||
* - the left child of a node i is 2*i+1.
|
||||
* - the right child of a node i is 2*i+2.
|
||||
*
|
||||
* It's easy to see that:
|
||||
* - the parent of node i is \f$\left\lfloor(i-1)/2\right\rfloor\f$.
|
||||
* - the j-th level ancestor of node i is
|
||||
* \f$\left\lfloor(i+1)/2^{j+1}\right\rfloor - 1\f$.
|
||||
* - A node i is a left child of its parent if \f$(i-1)\%2==0\f$.
|
||||
*
|
||||
*/
|
||||
|
||||
class HierarchicalSigmoidOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
void InferShape(framework::InferShapeContext* ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("W"), "Input(W) should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput("PreOut"),
|
||||
"Output(PreOut) should not be null.");
|
||||
const int64_t batch_size = ctx->GetInputDim("X")[0];
|
||||
std::vector<int64_t> output_shape({batch_size, 1});
|
||||
ctx->SetOutputDim("Out", framework::make_ddim(output_shape));
|
||||
}
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext& ctx) const override {
|
||||
return framework::OpKernelType(
|
||||
framework::ToDataType(ctx.Input<framework::Tensor>("X")->type()),
|
||||
ctx.GetPlace());
|
||||
}
|
||||
};
|
||||
|
||||
template <typename AttrType>
|
||||
class HierarchicalSigmoidOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddInput("X",
|
||||
"(Tensor, required) The input tensor with shape [N, D], "
|
||||
"where N is the size of mini-batch, and D is the feature size.");
|
||||
AddInput("W",
|
||||
"(Tensor, required), The parameters of hierarchical "
|
||||
"sigmoid operator, each of them is a 2-D tensor, the shape is"
|
||||
"[num_classes - 1, D].");
|
||||
AddInput("Label",
|
||||
"(Tensor, required), The labels of training data. It's a"
|
||||
"tensor with shape [N, 1].");
|
||||
AddInput("Bias",
|
||||
"(Tensor, optional), The bias is a tensor with shape"
|
||||
"[1, num_classes - 1].");
|
||||
AddOutput("Out",
|
||||
"(Tensor, required) The output of hierarchical sigmoid operator."
|
||||
"The shape is [N, 1].");
|
||||
AddOutput("PreOut",
|
||||
"(Tensor, required) A intermedia 2-D tensor with shape "
|
||||
"[batch_size, code_length], where code_length represents the "
|
||||
"maximum path length from root to leaf nodes.")
|
||||
.AsIntermediate();
|
||||
AddAttr<AttrType>("num_classes", "(int, required), The number of classes")
|
||||
.SetDefault(2);
|
||||
AddComment(R"DOC(
|
||||
The hierarchical sigmoid operator organize the classes into a binary tree.
|
||||
At each node, a sigmoid function is used to calculate the probability of
|
||||
belonging to the right branch. This idea is from
|
||||
"F. Morin, Y. Bengio (AISTATS 05):
|
||||
Hierarchical Probabilistic Neural Network Language Model."
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
|
||||
class HierarchicalSigmoidGradOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
void InferShape(framework::InferShapeContext* ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasInput("W"), "Input(W) should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("PreOut"),
|
||||
"Input(Preout) should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("W")),
|
||||
"Output(W@Grad should not be null.)");
|
||||
PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")));
|
||||
if (ctx->HasOutput(framework::GradVarName("Bias"))) {
|
||||
ctx->SetOutputDim(framework::GradVarName("Bias"),
|
||||
ctx->GetInputDim("Bias"));
|
||||
}
|
||||
ctx->SetOutputDim(framework::GradVarName("W"), ctx->GetInputDim("W"));
|
||||
ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
|
||||
}
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext& ctx) const override {
|
||||
return framework::OpKernelType(
|
||||
framework::ToDataType(ctx.Input<framework::Tensor>("X")->type()),
|
||||
ctx.GetPlace());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
REGISTER_OPERATOR(hierarchical_sigmoid, ops::HierarchicalSigmoidOp,
|
||||
ops::HierarchicalSigmoidOpMaker<int>,
|
||||
paddle::framework::DefaultGradOpDescMaker<true>);
|
||||
REGISTER_OPERATOR(hierarchical_sigmoid_grad, ops::HierarchicalSigmoidGradOp);
|
||||
REGISTER_OP_CPU_KERNEL(
|
||||
hierarchical_sigmoid,
|
||||
ops::HierarchicalSigmoidOpKernel<paddle::platform::CPUDeviceContext, float>,
|
||||
ops::HierarchicalSigmoidOpKernel<paddle::platform::CPUDeviceContext,
|
||||
double>);
|
||||
REGISTER_OP_CPU_KERNEL(
|
||||
hierarchical_sigmoid_grad,
|
||||
ops::HierarchicalSigmoidGradOpKernel<paddle::platform::CPUDeviceContext,
|
||||
float>,
|
||||
ops::HierarchicalSigmoidGradOpKernel<paddle::platform::CPUDeviceContext,
|
||||
double>);
|
||||
@ -0,0 +1,135 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/operators/clip_op.h"
|
||||
#include "paddle/fluid/operators/math/math_function.h"
|
||||
#include "paddle/fluid/operators/math/matrix_bit_code.h"
|
||||
#include "paddle/fluid/platform/transform.h"
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
template <typename T, int MajorType = Eigen::RowMajor,
|
||||
typename IndexType = Eigen::DenseIndex>
|
||||
using EigenMatrix = framework::EigenMatrix<T, MajorType, IndexType>;
|
||||
using platform::Transform;
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class HierarchicalSigmoidOpKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& ctx) const override {
|
||||
auto* in = ctx.Input<framework::Tensor>("X");
|
||||
auto* w = ctx.Input<framework::Tensor>("W");
|
||||
auto* label = ctx.Input<framework::Tensor>("Label");
|
||||
auto* bias = ctx.Input<framework::Tensor>("Bias");
|
||||
auto* out = ctx.Output<framework::Tensor>("Out");
|
||||
auto* pre_out = ctx.Output<framework::Tensor>("PreOut");
|
||||
size_t num_classes = static_cast<size_t>(ctx.Attr<int>("num_classes"));
|
||||
int64_t code_length = math::FindLastSet(num_classes - 1);
|
||||
int64_t batch_size = in->dims()[0];
|
||||
framework::Tensor sum;
|
||||
auto& dev_ctx = ctx.template device_context<DeviceContext>();
|
||||
auto* pre_out_data = pre_out->mutable_data<T>(
|
||||
framework::make_ddim({batch_size, code_length}), ctx.GetPlace());
|
||||
auto pre_out_mat = EigenMatrix<T>::From(*pre_out);
|
||||
// Not all class(leaf) nodes' path lengths equal code_length, thus init as
|
||||
// 0s can avoid out of path's loss.
|
||||
math::SetConstant<DeviceContext, T> zero;
|
||||
zero(dev_ctx, pre_out, static_cast<T>(0.0));
|
||||
auto& place = *ctx.template device_context<DeviceContext>().eigen_device();
|
||||
math::RowwiseSum<DeviceContext, T> row_sum;
|
||||
math::MatrixBitCodeFunctor<T> bit_code(num_classes, label->data<int64_t>());
|
||||
|
||||
std::vector<int64_t> sum_dims({batch_size, 1UL});
|
||||
sum.mutable_data<T>(framework::make_ddim(sum_dims), ctx.GetPlace());
|
||||
auto sum_mat = EigenMatrix<T>::From(sum);
|
||||
out->mutable_data<T>(ctx.GetPlace());
|
||||
auto out_mat = framework::EigenVector<T>::Flatten(*out);
|
||||
if (bias) {
|
||||
bit_code.Add(pre_out, *bias);
|
||||
}
|
||||
bit_code.Mul(pre_out, *w, *in);
|
||||
// clip to [-40, 40]
|
||||
Transform<DeviceContext> trans;
|
||||
trans(ctx.template device_context<DeviceContext>(), pre_out_data,
|
||||
pre_out_data + pre_out->numel(), pre_out_data,
|
||||
ClipFunctor<T>(static_cast<T>(-40.0), static_cast<T>(40.0)));
|
||||
bit_code.Sum(*pre_out, out, static_cast<T>(-1));
|
||||
// use softrelu to calculate cross entropy
|
||||
pre_out_mat.device(place) = (static_cast<T>(1.0) + pre_out_mat.exp()).log();
|
||||
row_sum(dev_ctx, *pre_out, &sum);
|
||||
// TODO(guosheng): Subtract the out of path's loss, since not all
|
||||
// class(leaf) nodes' path lengths equal code_length. But it won't break the
|
||||
// gradient check since both have the out of path's loss and will cancel out
|
||||
// each other.
|
||||
out_mat.device(place) = sum_mat + out_mat;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class HierarchicalSigmoidGradOpKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& ctx) const override {
|
||||
auto* in = ctx.Input<framework::Tensor>("X");
|
||||
auto* w = ctx.Input<framework::Tensor>("W");
|
||||
auto* in_grad = ctx.Output<framework::Tensor>(framework::GradVarName("X"));
|
||||
auto* w_grad = ctx.Output<framework::Tensor>(framework::GradVarName("W"));
|
||||
auto* bias_grad =
|
||||
ctx.Output<framework::Tensor>(framework::GradVarName("Bias"));
|
||||
auto* label = ctx.Input<framework::Tensor>("Label");
|
||||
auto* pre_out = ctx.Input<framework::Tensor>("PreOut");
|
||||
auto* out_grad =
|
||||
ctx.Input<framework::Tensor>(framework::GradVarName("Out"));
|
||||
framework::Tensor pre_out_grad;
|
||||
|
||||
pre_out_grad.mutable_data<T>(pre_out->dims(), ctx.GetPlace());
|
||||
in_grad->mutable_data<T>(ctx.GetPlace());
|
||||
w_grad->mutable_data<T>(ctx.GetPlace());
|
||||
auto& dev_ctx = ctx.template device_context<DeviceContext>();
|
||||
math::SetConstant<DeviceContext, T> zero;
|
||||
zero(dev_ctx, in_grad, static_cast<T>(0.0));
|
||||
zero(dev_ctx, w_grad, static_cast<T>(0.0));
|
||||
|
||||
size_t num_classes = static_cast<size_t>(ctx.Attr<int>("num_classes"));
|
||||
math::MatrixBitCodeFunctor<T> bit_code(num_classes, label->data<int64_t>());
|
||||
|
||||
auto& place = *ctx.template device_context<DeviceContext>().eigen_device();
|
||||
auto pre_out_mat = EigenMatrix<T>::From(*pre_out);
|
||||
auto pre_out_grad_mat = EigenMatrix<T>::From(pre_out_grad);
|
||||
auto out_grad_mat = EigenMatrix<T>::From(*out_grad);
|
||||
Eigen::array<int, 2> bcast({{1, static_cast<int>(pre_out_grad.dims()[1])}});
|
||||
|
||||
// softrelu derivative
|
||||
pre_out_grad_mat.device(place) =
|
||||
static_cast<T>(1.0) - static_cast<T>(1.0) / pre_out_mat.exp();
|
||||
bit_code.Sub(&pre_out_grad); // the gradient of clip(w * x + b)
|
||||
pre_out_grad_mat.device(place) =
|
||||
pre_out_grad_mat * out_grad_mat.broadcast(bcast);
|
||||
// TODO(guosheng): multiply pre_out_grad with subgradient of clipping to
|
||||
// be consistent with the clipping in forward.
|
||||
if (bias_grad) {
|
||||
bias_grad->mutable_data<T>(ctx.GetPlace());
|
||||
zero(dev_ctx, bias_grad, static_cast<T>(0.0));
|
||||
bit_code.AddGrad(pre_out_grad, bias_grad);
|
||||
}
|
||||
bit_code.MulGradWeight(pre_out_grad, w_grad, *in);
|
||||
bit_code.MulGradError(pre_out_grad, *w, in_grad);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,176 @@
|
||||
/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/math/matrix_bit_code.h"
|
||||
#include <iostream>
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace math {
|
||||
|
||||
template <typename T>
|
||||
void MatrixBitCodeFunctor<T>::Add(framework::Tensor* tmat,
|
||||
const framework::Tensor& vec) {
|
||||
SimpleCodeTable code_table(num_classes_);
|
||||
size_t batch_size = tmat->dims()[0];
|
||||
size_t width = tmat->dims()[1];
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
auto code = code_table(static_cast<size_t>(ids_[i]));
|
||||
int code_length = code.get_length();
|
||||
for (int j = 0; j < code_length; ++j) {
|
||||
size_t index = code.calc_index(j);
|
||||
tmat->data<T>()[i * width + j] += vec.data<T>()[index];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void MatrixBitCodeFunctor<T>::AddGrad(const framework::Tensor& tmat,
|
||||
framework::Tensor* vec) {
|
||||
SimpleCodeTable code_table(num_classes_);
|
||||
size_t batch_size = tmat.dims()[0];
|
||||
size_t width = tmat.dims()[1];
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
auto code = code_table(static_cast<size_t>(ids_[i]));
|
||||
int code_length = code.get_length();
|
||||
for (int j = 0; j < code_length; ++j) {
|
||||
size_t index = code.calc_index(j);
|
||||
vec->data<T>()[index] += tmat.data<T>()[i * width + j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void MatrixBitCodeFunctor<T>::Sum(const framework::Tensor& tmat,
|
||||
framework::Tensor* sum, T scale_sum) {
|
||||
SimpleCodeTable code_table(num_classes_);
|
||||
size_t num_samples = tmat.dims()[0];
|
||||
size_t o_width = tmat.dims()[1];
|
||||
for (size_t i = 0; i < num_samples; ++i) {
|
||||
T sm = static_cast<T>(0.0);
|
||||
auto code = code_table(static_cast<size_t>(ids_[i]));
|
||||
int code_length = code.get_length();
|
||||
for (int j = 0; j < code_length; ++j) {
|
||||
if (code.calc_bit(j)) {
|
||||
// calc_bit starts from right most bit, while data in tmat[i] is in the
|
||||
// reverse order.
|
||||
sm += tmat.data<T>()[i * o_width + j];
|
||||
}
|
||||
}
|
||||
sum->data<T>()[i] = scale_sum * sm;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void MatrixBitCodeFunctor<T>::Mul(framework::Tensor* tmat,
|
||||
const framework::Tensor& weight,
|
||||
const framework::Tensor& input) {
|
||||
SimpleCodeTable code_table(num_classes_);
|
||||
size_t num_samples = tmat->dims()[0];
|
||||
size_t tmat_width = tmat->dims()[1];
|
||||
size_t input_width = input.dims()[1];
|
||||
size_t weight_width = weight.dims()[1];
|
||||
auto tmat_value = tmat->data<T>();
|
||||
auto weight_value = weight.data<T>();
|
||||
auto input_value = input.data<T>();
|
||||
for (size_t i = 0; i < num_samples; ++i) {
|
||||
auto code = code_table(static_cast<size_t>(ids_[i]));
|
||||
int code_length = code.get_length();
|
||||
for (int j = 0; j < code_length; ++j) {
|
||||
size_t index = code.calc_index(j);
|
||||
T sum = static_cast<T>(0.0);
|
||||
for (size_t k = 0; k < input_width; ++k) {
|
||||
sum += weight_value[weight_width * index + k] *
|
||||
input_value[input_width * i + k];
|
||||
}
|
||||
tmat_value[i * tmat_width + j] += sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void MatrixBitCodeFunctor<T>::MulGradWeight(const framework::Tensor& tmat,
|
||||
framework::Tensor* weight,
|
||||
const framework::Tensor& input) {
|
||||
SimpleCodeTable code_table(num_classes_);
|
||||
size_t num_samples = tmat.dims()[0];
|
||||
size_t input_width = input.dims()[1];
|
||||
size_t tmat_width = tmat.dims()[1];
|
||||
size_t weight_width = weight->dims()[1];
|
||||
auto tmat_value = tmat.data<T>();
|
||||
auto weight_value = weight->data<T>();
|
||||
auto input_value = input.data<T>();
|
||||
for (size_t i = 0; i < num_samples; ++i) {
|
||||
auto code = code_table(static_cast<size_t>(ids_[i]));
|
||||
int code_length = code.get_length();
|
||||
for (int j = 0; j < code_length; ++j) {
|
||||
size_t index = code.calc_index(j);
|
||||
|
||||
for (size_t k = 0; k < input_width; ++k) {
|
||||
weight_value[weight_width * index + k] +=
|
||||
tmat_value[i * tmat_width + j] * input_value[input_width * i + k];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void MatrixBitCodeFunctor<T>::MulGradError(const framework::Tensor& tmat,
|
||||
const framework::Tensor& weight,
|
||||
framework::Tensor* input) {
|
||||
SimpleCodeTable code_table(num_classes_);
|
||||
size_t num_samples = tmat.dims()[0];
|
||||
size_t tmat_width = tmat.dims()[1];
|
||||
size_t input_width = input->dims()[1];
|
||||
size_t weight_width = weight.dims()[1];
|
||||
auto tmat_value = tmat.data<T>();
|
||||
auto weight_value = weight.data<T>();
|
||||
auto input_value = input->data<T>();
|
||||
|
||||
for (size_t i = 0; i < num_samples; ++i) {
|
||||
auto code = code_table(static_cast<size_t>(ids_[i]));
|
||||
int code_length = code.get_length();
|
||||
for (int j = 0; j < code_length; ++j) {
|
||||
size_t index = code.calc_index(j);
|
||||
|
||||
for (size_t k = 0; k < input_width; ++k) {
|
||||
input_value[input_width * i + k] +=
|
||||
tmat_value[i * tmat_width + j] *
|
||||
weight_value[weight_width * index + k];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void MatrixBitCodeFunctor<T>::Sub(framework::Tensor* tmat) {
|
||||
SimpleCodeTable code_table(num_classes_);
|
||||
size_t num_samples = tmat->dims()[0];
|
||||
size_t o_width = tmat->dims()[1];
|
||||
for (size_t i = 0; i < num_samples; ++i) {
|
||||
auto code = code_table(static_cast<size_t>(ids_[i]));
|
||||
int code_length = code.get_length();
|
||||
for (int j = 0; j < code_length; ++j) {
|
||||
if (code.calc_bit(j)) {
|
||||
tmat->data<T>()[i * o_width + j] -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template class MatrixBitCodeFunctor<float>;
|
||||
template class MatrixBitCodeFunctor<double>;
|
||||
|
||||
} // namespace math
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,143 @@
|
||||
/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include "paddle/fluid/framework/eigen.h"
|
||||
#include "paddle/fluid/framework/tensor.h"
|
||||
#include "paddle/fluid/platform/device_context.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace math {
|
||||
/**
|
||||
* SimpleCodeTable class should support 3 functions:
|
||||
*
|
||||
* size_t size()
|
||||
* return the number of ids
|
||||
*
|
||||
* int get_max_code_length()
|
||||
* return the maximal code length
|
||||
*
|
||||
* SimpleCode operator()(size_t i)
|
||||
* return the i-th code. Code class is descriebed below.
|
||||
*
|
||||
* SimpleCode class should support 3 functions:
|
||||
*
|
||||
* int get_length()
|
||||
* return the length of the code
|
||||
*
|
||||
* size_t cal_index(int bit)
|
||||
* bit ranges from 0 to get_length() - 1
|
||||
* return the index for the (1+bit) level parent
|
||||
*
|
||||
* bool calc_bit(int bit)
|
||||
* return true if the bit level parent is the right child of (1+bit) level
|
||||
* parent
|
||||
*
|
||||
*/
|
||||
|
||||
/**
|
||||
* return the 1-based index of the highest bit set
|
||||
*
|
||||
* for x > 0:
|
||||
* \f[
|
||||
* FindLastSet(x) = 1 + \floor*{\log_{2}x}
|
||||
* \f]
|
||||
*/
|
||||
inline constexpr size_t FindLastSet(size_t x) {
|
||||
return std::is_same<size_t, unsigned int>::value
|
||||
? (x ? 8 * sizeof(x) - __builtin_clz(x) : 0)
|
||||
: (std::is_same<size_t, unsigned long>::value // NOLINT
|
||||
? (x ? 8 * sizeof(x) - __builtin_clzl(x) : 0)
|
||||
: (x ? 8 * sizeof(x) - __builtin_clzll(x) : 0));
|
||||
}
|
||||
|
||||
struct SimpleCode {
|
||||
SimpleCode(size_t code, size_t num_classes) : c_(code + num_classes) {}
|
||||
/**
|
||||
* Here the id of root shoud be 1 rather than 0, thus the encoding of class c
|
||||
* is `c + num_classes` and all siblings can get the same weight indice using
|
||||
* prefixes.
|
||||
* Weight index is the prefixes of encoding, thus leave out the right most
|
||||
* bit in calc_index.
|
||||
* Binary classification path is the suffixes of encoding, thus leave out the
|
||||
* left most bit in calc_bit.
|
||||
*/
|
||||
inline size_t calc_index(int bit) const { return (c_ >> (bit + 1)) - 1; }
|
||||
inline bool calc_bit(int bit) const { return c_ & (1 << bit); }
|
||||
inline int get_length() const { return FindLastSet(c_) - 1; }
|
||||
|
||||
private:
|
||||
size_t c_;
|
||||
};
|
||||
|
||||
struct SimpleCodeTable {
|
||||
explicit SimpleCodeTable(size_t num_classes) : num_classes_(num_classes) {}
|
||||
SimpleCode operator()(size_t code) const {
|
||||
return SimpleCode(code, num_classes_);
|
||||
}
|
||||
size_t size() const { return num_classes_; }
|
||||
int get_max_code_length() const { return FindLastSet(num_classes_ - 1); }
|
||||
|
||||
private:
|
||||
size_t num_classes_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class MatrixBitCodeFunctor {
|
||||
public:
|
||||
explicit MatrixBitCodeFunctor(size_t num_classes, const int64_t* ids)
|
||||
: num_classes_(num_classes), ids_(ids) {}
|
||||
/* For j < code_length
|
||||
tmat(i, j) += vec(0, index(i, j))
|
||||
*/
|
||||
void Add(framework::Tensor* tmat, const framework::Tensor& vec);
|
||||
|
||||
/* For j < code_length
|
||||
vec(0, index(i, j)) += tmat(i, j)
|
||||
*/
|
||||
void AddGrad(const framework::Tensor& tmat, framework::Tensor* vec);
|
||||
|
||||
/* For j < code_length
|
||||
sum(i, 0) = \sum_j bit(i, j) * tmat(i, j)
|
||||
*/
|
||||
void Sum(const framework::Tensor& tmat, framework::Tensor* sum, T scale_sum);
|
||||
|
||||
/* For j < code_length
|
||||
tmat(i, j) -= bit(i, j)
|
||||
*/
|
||||
void Sub(framework::Tensor* tmat);
|
||||
/* For j < code_length
|
||||
input.row(i) += tmat(i, j) * weight.row(index(i, j))
|
||||
*/
|
||||
void Mul(framework::Tensor* tmat, const framework::Tensor& weight,
|
||||
const framework::Tensor& input);
|
||||
|
||||
/* For index(i, j) >= 0:
|
||||
weight.row(index(i, j)) += tmat(i, j) * input.row(i)
|
||||
*/
|
||||
void MulGradWeight(const framework::Tensor& tmat, framework::Tensor* weight,
|
||||
const framework::Tensor& input);
|
||||
/* For j < code_length
|
||||
input.row(i) += tmat(i, j) * weight.row(index(i, j))
|
||||
*/
|
||||
void MulGradError(const framework::Tensor& tmat,
|
||||
const framework::Tensor& weight, framework::Tensor* input);
|
||||
|
||||
size_t num_classes_;
|
||||
const int64_t* ids_;
|
||||
};
|
||||
} // namespace math
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,75 +0,0 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import paddle.fluid as fluid
|
||||
import unittest
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestCheckpoint(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dirname = tempfile.mktemp()
|
||||
self.max_num_checkpoints = 3
|
||||
self.epoch_interval = 1
|
||||
self.step_interval = 1
|
||||
self.trainer_id = 0
|
||||
self.chief = self.trainer_id == 0
|
||||
self.place = fluid.CPUPlace()
|
||||
self.epoch_id = 100
|
||||
self.step_id = 20
|
||||
|
||||
def test_checkpoint(self):
|
||||
self.save_checkpoint()
|
||||
serial = fluid.io.get_latest_checkpoint_serial(self.dirname)
|
||||
self.assertTrue(serial >= 0)
|
||||
trainer_args = ["epoch_id", "step_id"]
|
||||
epoch_id, step_id = fluid.io.load_trainer_args(
|
||||
self.dirname, serial, self.trainer_id, trainer_args)
|
||||
self.assertEqual(self.step_id, int(step_id))
|
||||
self.assertEqual(self.epoch_id, int(epoch_id))
|
||||
|
||||
program = fluid.Program()
|
||||
with fluid.program_guard(program):
|
||||
exe = fluid.Executor(self.place)
|
||||
fluid.io.load_checkpoint(exe, self.dirname, serial, program)
|
||||
|
||||
fluid.io.clean_checkpoint(self.dirname, delete_dir=True)
|
||||
self.assertFalse(os.path.isdir(self.dirname))
|
||||
|
||||
def save_checkpoint(self):
|
||||
config = fluid.CheckpointConfig(self.dirname, self.max_num_checkpoints,
|
||||
self.epoch_interval, self.step_interval)
|
||||
|
||||
trainer_args = {}
|
||||
trainer_args["epoch_id"] = self.epoch_id
|
||||
trainer_args["step_id"] = self.step_id
|
||||
|
||||
program = fluid.Program()
|
||||
with fluid.program_guard(program):
|
||||
program.global_block().create_var(
|
||||
name="scale_0",
|
||||
psersistable=True,
|
||||
dtype="float32",
|
||||
shape=[32, 32])
|
||||
|
||||
exe = fluid.Executor(self.place)
|
||||
for i in xrange(10):
|
||||
fluid.io.save_checkpoint(exe, config.checkpoint_dir,
|
||||
self.trainer_id, trainer_args, program,
|
||||
config.max_num_checkpoints)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,99 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
import numpy as np
|
||||
import math
|
||||
from op_test import OpTest
|
||||
|
||||
|
||||
def find_latest_set(num):
|
||||
return 1 + int(math.floor(math.log(num, 2)))
|
||||
|
||||
|
||||
class CodeTable(object):
|
||||
def __init__(self, num_classes, code):
|
||||
self.c = num_classes + code
|
||||
|
||||
def cal_index(self, bit):
|
||||
return (self.c >> (bit + 1)) - 1
|
||||
|
||||
def get_length(self):
|
||||
return find_latest_set(self.c) - 1
|
||||
|
||||
def cal_bit(self, bit):
|
||||
return self.c & (1 << bit)
|
||||
|
||||
|
||||
def hsigmoid(x, w, label, bias, num_classes):
|
||||
batch_size = x.shape[0]
|
||||
code_length = find_latest_set(num_classes - 1)
|
||||
code_table = [0 for _ in range(code_length)]
|
||||
pre_output = np.zeros((batch_size, code_length))
|
||||
pre_sum = np.zeros((batch_size, 1))
|
||||
out = np.zeros((batch_size, 1)).astype("float32")
|
||||
for i in range(batch_size):
|
||||
code_table = CodeTable(num_classes, label[i])
|
||||
length = code_table.get_length()
|
||||
for j in range(length):
|
||||
idx = code_table.cal_index(j)
|
||||
pre_output[i][j] += bias[0][idx]
|
||||
for i in range(batch_size):
|
||||
code_table = CodeTable(num_classes, label[i])
|
||||
length = code_table.get_length()
|
||||
for j in range(length):
|
||||
idx = code_table.cal_index(j)
|
||||
pre_output[i][j] += np.dot(w[idx], x[i])
|
||||
# clip[-40.0, 40.0]
|
||||
pre_output = np.clip(pre_output, -40.0, 40.0)
|
||||
# out(i, 0) = \sum_j bit(i, j) * preout(i, j)
|
||||
for i in range(batch_size):
|
||||
code_table = CodeTable(num_classes, label[i])
|
||||
length = code_table.get_length()
|
||||
sum = 0.0
|
||||
for j in range(length):
|
||||
if code_table.cal_bit(j):
|
||||
sum += pre_output[i][j]
|
||||
out[i] = -1.0 * sum
|
||||
# soft relu
|
||||
pre_output = np.log(1 + np.exp(pre_output))
|
||||
pre_sum = pre_output.sum(1).reshape((batch_size, 1))
|
||||
out += pre_sum
|
||||
return pre_output, out
|
||||
|
||||
|
||||
class TestHSigmoidOp(OpTest):
|
||||
def setUp(self):
|
||||
self.op_type = "hierarchical_sigmoid"
|
||||
num_classes = 6
|
||||
feature_size = 8
|
||||
batch_size = 4
|
||||
x = np.random.random((batch_size, feature_size)).astype("float32")
|
||||
w = np.random.random((num_classes - 1, feature_size)).astype("float32")
|
||||
label = np.random.randint(0, num_classes, (batch_size, 1))
|
||||
bias = np.random.random((1, num_classes - 1)).astype("float32")
|
||||
self.attrs = {'num_classes': num_classes}
|
||||
self.inputs = {'X': x, 'W': w, 'Label': label, 'Bias': bias}
|
||||
pre_output, out = hsigmoid(x, w, label, bias, num_classes)
|
||||
self.outputs = {'PreOut': pre_output, 'Out': out}
|
||||
|
||||
def test_check_output(self):
|
||||
self.check_output()
|
||||
|
||||
def test_check_grad(self):
|
||||
self.check_grad(['Bias', 'X', 'W'], ['Out'], no_grad_set=set('Label'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue