|
|
|
|
@ -44,8 +44,20 @@ class SGDOpKernel<platform::CPUDeviceContext, T>
|
|
|
|
|
if (grad_var->IsType<framework::LoDTensor>()) {
|
|
|
|
|
const auto *grad = ctx.Input<framework::Tensor>("Grad");
|
|
|
|
|
auto sz = param_out->numel();
|
|
|
|
|
PADDLE_ENFORCE_EQ(param->numel(), sz);
|
|
|
|
|
PADDLE_ENFORCE_EQ(grad->numel(), sz);
|
|
|
|
|
PADDLE_ENFORCE_EQ(param->numel(), sz,
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"The input tensor Param's numel of SgdOp "
|
|
|
|
|
"should be equal with ParamOut's numel. "
|
|
|
|
|
"But received Param's "
|
|
|
|
|
"numel = [%s], ParamOut's numel = [%s]",
|
|
|
|
|
param->numel(), sz));
|
|
|
|
|
PADDLE_ENFORCE_EQ(grad->numel(), sz,
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"The input tensor Grad's numel of SgdOp "
|
|
|
|
|
"should be equal with ParamOut's numel. "
|
|
|
|
|
"But received Grad's "
|
|
|
|
|
"numel = [%s], ParamOut's numel = [%s]",
|
|
|
|
|
grad->numel(), sz));
|
|
|
|
|
|
|
|
|
|
jit::sgd_attr_t attr(1, sz, 1, sz, 1);
|
|
|
|
|
const T *lr = learning_rate->data<T>();
|
|
|
|
|
@ -62,7 +74,11 @@ class SGDOpKernel<platform::CPUDeviceContext, T>
|
|
|
|
|
// TODO(qijun): In Sparse SGD operator, in-place update is enforced.
|
|
|
|
|
// This manual optimization brings difficulty to track data dependency.
|
|
|
|
|
// It's better to find a more elegant solution.
|
|
|
|
|
PADDLE_ENFORCE_EQ(param, param_out);
|
|
|
|
|
PADDLE_ENFORCE_EQ(param, param_out,
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"The input tensor Param of SgdOp "
|
|
|
|
|
"should be equal with ParamOut if variable's "
|
|
|
|
|
"type is SelectedRows. "));
|
|
|
|
|
const auto *grad = ctx.Input<framework::SelectedRows>("Grad");
|
|
|
|
|
auto &grad_rows = grad->rows();
|
|
|
|
|
|
|
|
|
|
@ -73,7 +89,13 @@ class SGDOpKernel<platform::CPUDeviceContext, T>
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
auto out_dims = param_out->dims();
|
|
|
|
|
PADDLE_ENFORCE_EQ(grad->height(), out_dims[0]);
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
grad->height(), out_dims[0],
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"The input tensor Grad's height of SgdOp "
|
|
|
|
|
"should be equal with ParamOut's dims. But received Grad's "
|
|
|
|
|
"height [%s] and ParamOut's dims [%s]",
|
|
|
|
|
grad->height(), out_dims[0]));
|
|
|
|
|
auto &grad_value = grad->value();
|
|
|
|
|
const T *param_data = param->data<T>();
|
|
|
|
|
const T *grad_data = grad_value.data<T>();
|
|
|
|
|
@ -87,19 +109,31 @@ class SGDOpKernel<platform::CPUDeviceContext, T>
|
|
|
|
|
attr.grad_height = grad_rows.size(); // note: it is not grad->height()
|
|
|
|
|
attr.grad_width = grad_value.numel() / attr.grad_height;
|
|
|
|
|
attr.selected_rows_size = grad_rows.size();
|
|
|
|
|
PADDLE_ENFORCE_EQ(attr.grad_width, attr.param_width);
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
attr.grad_width, attr.param_width,
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"The grad_value's numel of SgdOp "
|
|
|
|
|
"should be equal with param_out's numel. But received "
|
|
|
|
|
"grad_value's numel [%s] and param_out's numel [%s]",
|
|
|
|
|
attr.grad_width, attr.param_width));
|
|
|
|
|
|
|
|
|
|
auto sgd =
|
|
|
|
|
jit::KernelFuncs<jit::SgdTuple<T>, platform::CPUPlace>::Cache().At(
|
|
|
|
|
attr);
|
|
|
|
|
sgd(lr, param_data, grad_data, rows_data, out_data, &attr);
|
|
|
|
|
} else {
|
|
|
|
|
PADDLE_THROW("Unsupported Variable Type of Grad");
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
false, true,
|
|
|
|
|
platform::errors::PermissionDenied(
|
|
|
|
|
"Unsupported Variable Type of Grad in SgdOp. Excepted "
|
|
|
|
|
"LodTensor or SelectedRows, But received [%s]",
|
|
|
|
|
paddle::framework::ToTypeName(grad_var->Type())));
|
|
|
|
|
}
|
|
|
|
|
} else if (param_var->IsType<framework::SelectedRows>()) {
|
|
|
|
|
PADDLE_ENFORCE(grad_var->IsType<framework::SelectedRows>(),
|
|
|
|
|
"when param "
|
|
|
|
|
"is SelectedRows, gradient should also be SelectedRows");
|
|
|
|
|
PADDLE_ENFORCE_EQ(grad_var->IsType<framework::SelectedRows>(), true,
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"when param is SelectedRows, "
|
|
|
|
|
"gradient should also be SelectedRows"));
|
|
|
|
|
const auto ¶m = param_var->Get<framework::SelectedRows>();
|
|
|
|
|
auto *param_out = ctx.Output<framework::SelectedRows>("ParamOut");
|
|
|
|
|
const auto &grad = grad_var->Get<framework::SelectedRows>();
|
|
|
|
|
@ -112,27 +146,36 @@ class SGDOpKernel<platform::CPUDeviceContext, T>
|
|
|
|
|
|
|
|
|
|
auto param_row_width = param.value().dims()[1];
|
|
|
|
|
auto grad_row_width = grad.value().dims()[1];
|
|
|
|
|
VLOG(4) << " param rows: " << param.rows().size()
|
|
|
|
|
<< " param memory rows: " << param.value().dims()[0]
|
|
|
|
|
<< " grad rows: " << grad.rows().size()
|
|
|
|
|
<< " grad memory rows: " << grad.value().dims()[0];
|
|
|
|
|
PADDLE_ENFORCE_EQ(param_row_width, grad_row_width,
|
|
|
|
|
"param_row should have the same size with grad_row");
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
param_row_width, grad_row_width,
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"The param_row in SgdOP should have the same size with grad_row. "
|
|
|
|
|
"But received param_row's width is [%s], and grad_row's width is "
|
|
|
|
|
"[%s]",
|
|
|
|
|
param_row_width, grad_row_width));
|
|
|
|
|
|
|
|
|
|
const auto *lr = learning_rate->data<T>();
|
|
|
|
|
const auto *grad_data = grad.value().data<T>();
|
|
|
|
|
auto *out_data = param_out->mutable_value()->data<T>();
|
|
|
|
|
for (size_t i = 0; i < grad.rows().size(); i++) {
|
|
|
|
|
int64_t id_index = param_out->AutoGrownIndex(grad.rows()[i], false);
|
|
|
|
|
PADDLE_ENFORCE_GE(id_index, static_cast<int64_t>(0),
|
|
|
|
|
"id should be in the table");
|
|
|
|
|
PADDLE_ENFORCE_GE(
|
|
|
|
|
id_index, static_cast<int64_t>(0),
|
|
|
|
|
platform::errors::InvalidArgument(
|
|
|
|
|
"The id in SgdOp should be >= 0. But recevied id_index is [%s]",
|
|
|
|
|
id_index));
|
|
|
|
|
for (int64_t j = 0; j < grad_row_width; j++) {
|
|
|
|
|
out_data[id_index * grad_row_width + j] -=
|
|
|
|
|
lr[0] * grad_data[i * grad_row_width + j];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
PADDLE_THROW("Unsupported Variable Type of Parameter");
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
false, true,
|
|
|
|
|
platform::errors::PermissionDenied(
|
|
|
|
|
"Unsupported Variable Type of Parameter in SgdOp. Excepted "
|
|
|
|
|
"LodTensor or SelectedRows, But received [%s]",
|
|
|
|
|
paddle::framework::ToTypeName(param_var->Type())));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
Chengmo
4 years ago
committed by
GitHub