Update amp_check_finite_and_scale_op and add an updating_loss_scaling op for static graph amp training. (#26240)
* update amp_check_finite_and_scale_op for static_amp. * use amp_check_finite_and_scale in static graph amp. * update grads to zero when grads own infinite values(as for amp_checkout_finite_and_scale op). * add update_loss_scaling op in cpp. * add update_loss_scaling_op unit test. * update the doc of the check_finite_and_unscale op * Update the process of gradients updating skipping if the gradients have infinite values. * update the way to zero grads. * update test_update_loss_scaling_op.py * add log info when find infinite grads. * add the unit test for UpdateLossScaling Layer.disable_ut_1
Zhen Wang
4 years ago
committed by
GitHub
parent
2b6a5793fe
commit
d708b21074
@ -1,104 +0,0 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/amp/amp_check_finite_and_scale_op.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class AmpCheckFiniteAndScaleOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
AmpCheckFiniteAndScaleOp(const std::string &type,
|
||||
const framework::VariableNameMap &inputs,
|
||||
const framework::VariableNameMap &outputs,
|
||||
const framework::AttributeMap &attrs)
|
||||
: OperatorWithKernel(type, inputs, outputs, attrs) {}
|
||||
|
||||
void InferShape(framework::InferShapeContext *ctx) const override {
|
||||
OP_INOUT_CHECK(ctx->HasInputs("X"), "Input", "X",
|
||||
"amp_check_finite_and_unscale");
|
||||
OP_INOUT_CHECK(ctx->HasOutputs("Out"), "Output", "Out",
|
||||
"amp_check_finite_and_unscale");
|
||||
PADDLE_ENFORCE_EQ(
|
||||
ctx->Inputs("X").size(), ctx->Outputs("Out").size(),
|
||||
platform::errors::InvalidArgument(
|
||||
"The input(X) and output(Out) should have same size in "
|
||||
"Operator(amp_check_finite_and_unscale), size of input(X) is %d "
|
||||
"and size of output(Out) is %d.",
|
||||
ctx->Inputs("X").size(), ctx->Outputs("Out").size()));
|
||||
auto x_dims = ctx->GetInputsDim("X");
|
||||
ctx->SetOutputsDim("Out", x_dims);
|
||||
ctx->SetOutputDim("FoundInfinite", {1});
|
||||
}
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext &ctx) const override {
|
||||
return framework::OpKernelType(
|
||||
OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace());
|
||||
}
|
||||
};
|
||||
|
||||
class AmpCheckFiniteAndScaleOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddInput(
|
||||
"X",
|
||||
"(Tensors) The input tensors of amp_check_finite_and_scale operator.")
|
||||
.AsDuplicable();
|
||||
AddInput("Scale",
|
||||
"(Tensor) 1-dim tensor, the scale of amp_check_finite_and_scale "
|
||||
"operator.");
|
||||
AddOutput("Out",
|
||||
"(Tensors) The scaled output tensor of "
|
||||
"amp_check_finite_and_unscale operator.")
|
||||
.AsDuplicable();
|
||||
AddOutput("FoundInfinite",
|
||||
"(Tensor) 1-dim tensor, contains a bool scalar, which indicates "
|
||||
"if there there is infinite or nan item in input X.");
|
||||
AddComment(R"DOC(
|
||||
amp_check_finite_and_scale operator.
|
||||
Check if input X contains all finite data, if yes, scale it by input Scale.
|
||||
|
||||
$$Out = X * scale$$
|
||||
|
||||
If any tensor in X contains Inf or Nan, the Out will generate a indicator.
|
||||
FoundInfinite will be 1 (True), and Out will not be scaled. In this case, the data of
|
||||
Out should not be used, and its data may not be deterministic.
|
||||
Otherwise, FoundInfinite will be 0 (False).
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
|
||||
REGISTER_OPERATOR(
|
||||
amp_check_finite_and_scale, ops::AmpCheckFiniteAndScaleOp,
|
||||
ops::AmpCheckFiniteAndScaleOpMaker,
|
||||
paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
|
||||
paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
|
||||
|
||||
REGISTER_OP_CPU_KERNEL(
|
||||
amp_check_finite_and_scale,
|
||||
ops::AmpCheckFiniteAndScaleKernel<paddle::platform::CPUDeviceContext,
|
||||
float>,
|
||||
ops::AmpCheckFiniteAndScaleKernel<paddle::platform::CPUDeviceContext,
|
||||
double>);
|
||||
@ -1,66 +0,0 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "paddle/fluid/framework/tensor_util.h"
|
||||
#include "paddle/fluid/operators/elementwise/elementwise_op_function.h"
|
||||
#include "paddle/fluid/operators/isfinite_op.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class AmpCheckFiniteAndScaleKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& ctx) const {
|
||||
auto& dev_ctx = ctx.template device_context<DeviceContext>();
|
||||
const auto xs = ctx.MultiInput<framework::Tensor>("X");
|
||||
const auto* scale = ctx.Input<framework::Tensor>("Scale");
|
||||
auto outs = ctx.MultiOutput<framework::Tensor>("Out");
|
||||
auto* found_inf = ctx.Output<framework::Tensor>("FoundInfinite");
|
||||
|
||||
const T* scale_data = scale->data<T>();
|
||||
bool* found_inf_data = found_inf->mutable_data<bool>(dev_ctx.GetPlace());
|
||||
|
||||
*found_inf_data = false;
|
||||
framework::Tensor is_finite =
|
||||
ctx.AllocateTmpTensor<bool, DeviceContext>({1}, dev_ctx);
|
||||
bool* is_finite_data = is_finite.template data<bool>();
|
||||
|
||||
auto& dev = *ctx.template device_context<DeviceContext>().eigen_device();
|
||||
for (size_t i = 0; i < xs.size(); ++i) {
|
||||
const auto* x = xs[i];
|
||||
auto* out = outs[i];
|
||||
out->mutable_data<T>(dev_ctx.GetPlace());
|
||||
if (!(*found_inf_data)) {
|
||||
framework::TensorIsfinite(*x, &is_finite);
|
||||
if (*is_finite_data) {
|
||||
auto eigen_out = framework::EigenVector<T>::Flatten(*out);
|
||||
auto eigen_in = framework::EigenVector<T>::Flatten(*x);
|
||||
eigen_out.device(dev) = (*scale_data) * eigen_in;
|
||||
} else {
|
||||
*found_inf_data = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,141 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/amp/check_finite_and_unscale_op.h"
|
||||
#include "paddle/fluid/framework/tensor_util.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class CheckFiniteAndUnscaleOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
CheckFiniteAndUnscaleOp(const std::string& type,
|
||||
const framework::VariableNameMap& inputs,
|
||||
const framework::VariableNameMap& outputs,
|
||||
const framework::AttributeMap& attrs)
|
||||
: OperatorWithKernel(type, inputs, outputs, attrs) {}
|
||||
|
||||
void InferShape(framework::InferShapeContext* ctx) const override {
|
||||
OP_INOUT_CHECK(ctx->HasInputs("X"), "Input", "X",
|
||||
"check_finite_and_unscale");
|
||||
OP_INOUT_CHECK(ctx->HasOutputs("Out"), "Output", "Out",
|
||||
"check_finite_and_unscale");
|
||||
PADDLE_ENFORCE_EQ(
|
||||
ctx->Inputs("X").size(), ctx->Outputs("Out").size(),
|
||||
platform::errors::InvalidArgument(
|
||||
"The input(X) and output(Out) should have same size in "
|
||||
"Operator(check_finite_and_unscale), size of input(X) is %d "
|
||||
"and size of output(Out) is %d.",
|
||||
ctx->Inputs("X").size(), ctx->Outputs("Out").size()));
|
||||
auto x_dims = ctx->GetInputsDim("X");
|
||||
ctx->SetOutputsDim("Out", x_dims);
|
||||
ctx->SetOutputDim("FoundInfinite", {1});
|
||||
}
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext& ctx) const override {
|
||||
return framework::OpKernelType(
|
||||
OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace());
|
||||
}
|
||||
};
|
||||
|
||||
class CheckFiniteAndUnscaleOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddInput(
|
||||
"X",
|
||||
"(Tensors) The input tensors of check_finite_and_unscale operator.")
|
||||
.AsDuplicable();
|
||||
AddInput("Scale",
|
||||
"(Tensor) 1-dim tensor, the scale of check_finite_and_unscale "
|
||||
"operator.");
|
||||
AddOutput("Out",
|
||||
"(Tensors) The scaled output tensor of "
|
||||
"check_finite_and_unscale operator.")
|
||||
.AsDuplicable();
|
||||
AddOutput("FoundInfinite",
|
||||
"(Tensor) 1-dim tensor, contains a bool scalar, which indicates "
|
||||
"if there there is infinite or nan item in input X.");
|
||||
AddComment(R"DOC(
|
||||
check_finite_and_unscale operator.
|
||||
Check if input X contains all finite data, if yes, scale it by input Scale.
|
||||
|
||||
$$Out = X / scale$$
|
||||
|
||||
If any tensor in X contains Inf or Nan, the Out will generate a indicator.
|
||||
FoundInfinite will be 1 (True), and Out will not be scaled. In this case, the data of
|
||||
Out should not be used, and its data may not be deterministic.
|
||||
Otherwise, FoundInfinite will be 0 (False).
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class CheckFiniteAndUnscaleCpuKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& ctx) const {
|
||||
auto& dev_ctx = ctx.template device_context<platform::CPUDeviceContext>();
|
||||
const auto xs = ctx.MultiInput<framework::Tensor>("X");
|
||||
const auto* scale = ctx.Input<framework::Tensor>("Scale");
|
||||
auto outs = ctx.MultiOutput<framework::Tensor>("Out");
|
||||
auto* found_inf = ctx.Output<framework::Tensor>("FoundInfinite");
|
||||
|
||||
const T* scale_data = scale->data<T>();
|
||||
bool* found_inf_data = found_inf->mutable_data<bool>(dev_ctx.GetPlace());
|
||||
|
||||
*found_inf_data = false;
|
||||
framework::Tensor is_finite =
|
||||
ctx.AllocateTmpTensor<bool, platform::CPUDeviceContext>({1}, dev_ctx);
|
||||
bool* is_finite_data = is_finite.template data<bool>();
|
||||
|
||||
auto& dev = *ctx.template device_context<platform::CPUDeviceContext>()
|
||||
.eigen_device();
|
||||
|
||||
T inverse_scale = Inverse<T>(*scale_data);
|
||||
for (size_t i = 0; i < xs.size(); ++i) {
|
||||
const auto* x = xs[i];
|
||||
auto* out = outs[i];
|
||||
out->mutable_data<T>(dev_ctx.GetPlace());
|
||||
if (!(*found_inf_data)) {
|
||||
framework::TensorIsfinite(*x, &is_finite);
|
||||
*found_inf_data = !(*is_finite_data);
|
||||
}
|
||||
auto eigen_out = framework::EigenVector<T>::Flatten(*out);
|
||||
auto eigen_in = framework::EigenVector<T>::Flatten(*x);
|
||||
if (!(*found_inf_data)) {
|
||||
eigen_out.device(dev) = eigen_in * inverse_scale;
|
||||
} else {
|
||||
eigen_out.device(dev) = eigen_in * static_cast<T>(0);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
|
||||
REGISTER_OPERATOR(
|
||||
check_finite_and_unscale, ops::CheckFiniteAndUnscaleOp,
|
||||
ops::CheckFiniteAndUnscaleOpMaker,
|
||||
paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
|
||||
paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
|
||||
|
||||
REGISTER_OP_CPU_KERNEL(check_finite_and_unscale,
|
||||
ops::CheckFiniteAndUnscaleCpuKernel<float>,
|
||||
ops::CheckFiniteAndUnscaleCpuKernel<double>);
|
||||
@ -0,0 +1,31 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "paddle/fluid/operators/isfinite_op.h"
|
||||
#include "paddle/fluid/platform/hostdevice.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
template <typename T>
|
||||
inline HOSTDEVICE T Inverse(T s) {
|
||||
return 1.0 / s;
|
||||
}
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,170 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/amp/update_loss_scaling_op.h"
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class UpdateLossScalingOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
|
||||
void InferShape(framework::InferShapeContext* ctx) const override {
|
||||
OP_INOUT_CHECK(ctx->HasInputs("X"), "Input", "X", "update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasInput("FoundInfinite"), "Input", "FoundInfinite",
|
||||
"update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasInput("PrevLossScaling"), "Input", "PrevLossScaling",
|
||||
"update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasInput("InGoodSteps"), "Input", "InGoodSteps",
|
||||
"update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasInput("InBadSteps"), "Input", "InBadSteps",
|
||||
"update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasOutputs("Out"), "Output", "Out",
|
||||
"update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasOutput("LossScaling"), "Output", "LossScaling",
|
||||
"update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasOutput("OutGoodSteps"), "Output", "OutGoodSteps",
|
||||
"update_loss_scaling");
|
||||
OP_INOUT_CHECK(ctx->HasOutput("OutBadSteps"), "Output", "OutBadSteps",
|
||||
"update_loss_scaling");
|
||||
auto x_dims = ctx->GetInputsDim("X");
|
||||
ctx->SetOutputsDim("Out", x_dims);
|
||||
ctx->SetOutputDim("LossScaling", {1});
|
||||
ctx->SetOutputDim("OutGoodSteps", {1});
|
||||
ctx->SetOutputDim("OutBadSteps", {1});
|
||||
}
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext& ctx) const override {
|
||||
return framework::OpKernelType(
|
||||
OperatorWithKernel::IndicateVarDataType(ctx, "PrevLossScaling"),
|
||||
ctx.device_context());
|
||||
}
|
||||
};
|
||||
|
||||
class UpdateLossScalingOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddInput("X",
|
||||
"(Tensors) The input tensors of update_loss_scaling operator.")
|
||||
.AsDuplicable();
|
||||
AddInput("FoundInfinite",
|
||||
"(Tensor) 1-dim tensor, contains a bool scalar, which indicates "
|
||||
"whether there is any infinite gradient.");
|
||||
AddInput("PrevLossScaling",
|
||||
"(Tensor) 1-dim tensor, previous loss scaling.");
|
||||
AddInput("InGoodSteps",
|
||||
"(Tensor) 1-dim tensor, accumulates good steps in which all "
|
||||
"gradients are finite.");
|
||||
AddInput("InBadSteps",
|
||||
"(Tensor) 1-dim tensor, accumulates bad steps in which some "
|
||||
"gradients are infinite.");
|
||||
AddOutput("Out",
|
||||
"(Tensors) The output tensor of update_loss_scaling operator.")
|
||||
.AsDuplicable();
|
||||
AddOutput("LossScaling", "(Tensor) 1-dim tensor, updated loss scaling.");
|
||||
AddOutput("OutGoodSteps", "(Tensor) 1-dim tensor, pdated good steps.");
|
||||
AddOutput("OutBadSteps", "(Tensor) 1-dim tensor, updated bad steps.");
|
||||
AddAttr<int>("incr_every_n_steps",
|
||||
"A value represents increasing loss scaling every n "
|
||||
"consecutive steps with finite gradients.");
|
||||
AddAttr<int>("decr_every_n_nan_or_inf",
|
||||
"A value represents decreasing loss scaling every n "
|
||||
"accumulated steps with nan or inf gradients.");
|
||||
AddAttr<float>("incr_ratio",
|
||||
"The multiplier to use when increasing the loss scaling.")
|
||||
.AddCustomChecker([](float incr_ratio) {
|
||||
PADDLE_ENFORCE_EQ(incr_ratio > 1.0f, true,
|
||||
platform::errors::InvalidArgument(
|
||||
"'incr_ratio' should be greater than 1, but "
|
||||
"the received is %f",
|
||||
incr_ratio));
|
||||
});
|
||||
AddAttr<float>(
|
||||
"decr_ratio",
|
||||
"The less-than-one-multiplier to use when decreasing loss scaling.")
|
||||
.AddCustomChecker([](float decr_ratio) {
|
||||
PADDLE_ENFORCE_EQ(decr_ratio > 0.0f && decr_ratio < 1.0f, true,
|
||||
platform::errors::InvalidArgument(
|
||||
"'incr_ratio' should be between 0 and 1, but "
|
||||
"the received is %f",
|
||||
decr_ratio));
|
||||
});
|
||||
AddComment(R"DOC(
|
||||
Update loss scaling according to overall gradients. If all gradients is
|
||||
finite after incr_every_n_steps, loss scaling will increase by incr_ratio.
|
||||
Otherwise, loss scaling will decrease by decr_ratio after
|
||||
decr_every_n_nan_or_inf steps and each step some gradients are infinite.
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class UpdateLossScalingFunctor<platform::CPUDeviceContext, T> {
|
||||
public:
|
||||
void operator()(const platform::CPUDeviceContext& ctx,
|
||||
const bool* found_inf_data, const T* pre_loss_scaling_data,
|
||||
const int* good_in_data, const int* bad_in_data,
|
||||
const int incr_every_n_steps,
|
||||
const int decr_every_n_nan_or_inf, const float incr_ratio,
|
||||
const float decr_ratio, T* updated_loss_scaling_data,
|
||||
int* good_out_data, int* bad_out_data) const {
|
||||
Update<T>(found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data,
|
||||
incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio,
|
||||
decr_ratio, updated_loss_scaling_data, good_out_data,
|
||||
bad_out_data);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class LazyZeroInputs<platform::CPUDeviceContext, T> {
|
||||
public:
|
||||
void operator()(const platform::CPUDeviceContext& dev_ctx,
|
||||
const bool* found_inf_data,
|
||||
const std::vector<const framework::Tensor*>& xs,
|
||||
const std::vector<framework::Tensor*>& outs) const {
|
||||
if (*found_inf_data) {
|
||||
VLOG(1) << "-- UpdateLossScaling: Infinite values are found in grads. --";
|
||||
for (size_t i = 0; i < xs.size(); ++i) {
|
||||
auto* out = outs[i];
|
||||
T* out_data = out->mutable_data<T>(dev_ctx.GetPlace());
|
||||
int num = out->numel();
|
||||
std::memset(out_data, 0, num * sizeof(T));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
using CPU = paddle::platform::CPUDeviceContext;
|
||||
|
||||
REGISTER_OPERATOR(
|
||||
update_loss_scaling, ops::UpdateLossScalingOp,
|
||||
ops::UpdateLossScalingOpMaker,
|
||||
paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
|
||||
paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
|
||||
|
||||
REGISTER_OP_CPU_KERNEL(update_loss_scaling,
|
||||
ops::UpdateLossScalingKernel<CPU, float>,
|
||||
ops::UpdateLossScalingKernel<CPU, double>);
|
||||
@ -0,0 +1,84 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <vector>
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/operators/amp/update_loss_scaling_op.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
template <typename T>
|
||||
__global__ void GpuUpdateLossScaling(
|
||||
const bool* found_inf_data, const T* pre_loss_scaling_data,
|
||||
const int* good_in_data, const int* bad_in_data,
|
||||
const int incr_every_n_steps, const int decr_every_n_nan_or_inf,
|
||||
const float incr_ratio, const float decr_ratio,
|
||||
T* updated_loss_scaling_data, int* good_out_data, int* bad_out_data) {
|
||||
Update<T>(found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data,
|
||||
incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio,
|
||||
updated_loss_scaling_data, good_out_data, bad_out_data);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
class UpdateLossScalingFunctor<platform::CUDADeviceContext, T> {
|
||||
public:
|
||||
void operator()(const platform::CUDADeviceContext& dev_ctx,
|
||||
const bool* found_inf_data, const T* pre_loss_scaling_data,
|
||||
const int* good_in_data, const int* bad_in_data,
|
||||
const int incr_every_n_steps,
|
||||
const int decr_every_n_nan_or_inf, const float incr_ratio,
|
||||
const float decr_ratio, T* updated_loss_scaling_data,
|
||||
int* good_out_data, int* bad_out_data) const {
|
||||
GpuUpdateLossScaling<T><<<1, 1, 0, dev_ctx.stream()>>>(
|
||||
found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data,
|
||||
incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio,
|
||||
updated_loss_scaling_data, good_out_data, bad_out_data);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class LazyZeroInputs<platform::CUDADeviceContext, T> {
|
||||
public:
|
||||
void operator()(const platform::CUDADeviceContext& dev_ctx,
|
||||
const bool* found_inf_data,
|
||||
const std::vector<const framework::Tensor*>& xs,
|
||||
const std::vector<framework::Tensor*>& outs) const {
|
||||
const auto gpu_place =
|
||||
BOOST_GET_CONST(platform::CUDAPlace, dev_ctx.GetPlace());
|
||||
bool has_inf{false};
|
||||
memory::Copy(platform::CPUPlace(), &has_inf, gpu_place, found_inf_data,
|
||||
sizeof(bool), dev_ctx.stream());
|
||||
if (has_inf) {
|
||||
VLOG(1) << "-- UpdateLossScaling: Infinite values are found in grads. --";
|
||||
for (size_t i = 0; i < xs.size(); ++i) {
|
||||
auto* out = outs[i];
|
||||
T* out_data = out->mutable_data<T>(dev_ctx.GetPlace());
|
||||
int num = out->numel();
|
||||
cudaMemset(out_data, 0, num * sizeof(T));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
using GPU = paddle::platform::CUDADeviceContext;
|
||||
|
||||
REGISTER_OP_CUDA_KERNEL(update_loss_scaling,
|
||||
ops::UpdateLossScalingKernel<GPU, float>,
|
||||
ops::UpdateLossScalingKernel<GPU, double>);
|
||||
@ -0,0 +1,123 @@
|
||||
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
#include "paddle/fluid/framework/operator.h"
|
||||
#include "paddle/fluid/platform/device_context.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
#include "paddle/fluid/platform/errors.h"
|
||||
#include "paddle/fluid/platform/hostdevice.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
using Tensor = framework::Tensor;
|
||||
|
||||
template <typename T>
|
||||
HOSTDEVICE void Update(const bool* found_inf_data,
|
||||
const T* pre_loss_scaling_data, const int* good_in_data,
|
||||
const int* bad_in_data, const int incr_every_n_steps,
|
||||
const int decr_every_n_nan_or_inf,
|
||||
const float incr_ratio, const float decr_ratio,
|
||||
T* updated_loss_scaling_data, int* good_out_data,
|
||||
int* bad_out_data) {
|
||||
if (*found_inf_data) {
|
||||
*good_out_data = 0;
|
||||
*bad_out_data = *bad_in_data + 1;
|
||||
if (*bad_out_data == decr_every_n_nan_or_inf) {
|
||||
T new_loss_scaling = *pre_loss_scaling_data * decr_ratio;
|
||||
*updated_loss_scaling_data = new_loss_scaling < static_cast<T>(1)
|
||||
? static_cast<T>(1)
|
||||
: new_loss_scaling;
|
||||
*bad_out_data = 0;
|
||||
}
|
||||
} else {
|
||||
*bad_out_data = 0;
|
||||
*good_out_data = *good_in_data + 1;
|
||||
if (*good_out_data == incr_every_n_steps) {
|
||||
T new_loss_scaling = *pre_loss_scaling_data * incr_ratio;
|
||||
*updated_loss_scaling_data = std::isfinite(new_loss_scaling)
|
||||
? new_loss_scaling
|
||||
: *pre_loss_scaling_data;
|
||||
*good_out_data = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class UpdateLossScalingFunctor {
|
||||
public:
|
||||
void operator()(const DeviceContext& dev_ctx, const bool* found_inf_data,
|
||||
const T* pre_loss_scaling_data, const int* good_in_data,
|
||||
const int* bad_in_data, const int incr_every_n_steps,
|
||||
const int decr_every_n_nan_or_inf, const float incr_ratio,
|
||||
const float decr_ratio, T* updated_loss_scaling_data,
|
||||
int* good_out_data, int* bad_out_data) const;
|
||||
};
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class LazyZeroInputs {
|
||||
public:
|
||||
void operator()(const DeviceContext& dev_ctx, const bool* found_inf_data,
|
||||
const std::vector<const framework::Tensor*>& xs,
|
||||
const std::vector<framework::Tensor*>& outs) const;
|
||||
};
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class UpdateLossScalingKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& ctx) const override {
|
||||
const auto xs = ctx.MultiInput<framework::Tensor>("X");
|
||||
const auto* found_inf = ctx.Input<Tensor>("FoundInfinite");
|
||||
const auto* pre_loss_scaling = ctx.Input<Tensor>("PrevLossScaling");
|
||||
const auto* good_in = ctx.Input<Tensor>("InGoodSteps");
|
||||
const auto* bad_in = ctx.Input<Tensor>("InBadSteps");
|
||||
auto outs = ctx.MultiOutput<framework::Tensor>("Out");
|
||||
auto* updated_loss_scaling = ctx.Output<Tensor>("LossScaling");
|
||||
auto* good_out = ctx.Output<Tensor>("OutGoodSteps");
|
||||
auto* bad_out = ctx.Output<Tensor>("OutBadSteps");
|
||||
|
||||
PADDLE_ENFORCE_EQ(found_inf->numel(), 1,
|
||||
platform::errors::InvalidArgument(
|
||||
"FoundInfinite must has only one element."));
|
||||
|
||||
const bool* found_inf_data = found_inf->data<bool>();
|
||||
const T* pre_loss_scaling_data = pre_loss_scaling->data<T>();
|
||||
const int* good_in_data = good_in->data<int>();
|
||||
const int* bad_in_data = bad_in->data<int>();
|
||||
|
||||
auto& dev_ctx = ctx.template device_context<DeviceContext>();
|
||||
T* updated_loss_scaling_data =
|
||||
updated_loss_scaling->mutable_data<T>(dev_ctx.GetPlace());
|
||||
int* good_out_data = good_out->mutable_data<int>(dev_ctx.GetPlace());
|
||||
int* bad_out_data = bad_out->mutable_data<int>(dev_ctx.GetPlace());
|
||||
|
||||
const int incr_every_n_steps = ctx.Attr<int>("incr_every_n_steps");
|
||||
const int decr_every_n_nan_or_inf =
|
||||
ctx.Attr<int>("decr_every_n_nan_or_inf");
|
||||
const float incr_ratio = ctx.Attr<float>("incr_ratio");
|
||||
const float decr_ratio = ctx.Attr<float>("decr_ratio");
|
||||
UpdateLossScalingFunctor<DeviceContext, T>{}(
|
||||
dev_ctx, found_inf_data, pre_loss_scaling_data, good_in_data,
|
||||
bad_in_data, incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio,
|
||||
decr_ratio, updated_loss_scaling_data, good_out_data, bad_out_data);
|
||||
LazyZeroInputs<DeviceContext, T>{}(dev_ctx, found_inf_data, xs, outs);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,124 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from paddle.fluid.data_feeder import check_variable_and_dtype, check_type
|
||||
from paddle.fluid.layer_helper import LayerHelper
|
||||
from paddle.fluid.framework import Variable
|
||||
|
||||
__all__ = ['check_finite_and_unscale', 'update_loss_scaling']
|
||||
|
||||
|
||||
def check_finite_and_unscale(x, scale, name=None):
|
||||
"""
|
||||
Check if input X contains all finite data, if yes, scale it by input Scale.
|
||||
|
||||
$$Out = X / scale$$
|
||||
|
||||
If any tensor in X contains Inf or Nan, the Out will generate a indicator.
|
||||
FoundInfinite will be 1 (True), and Out will not be scaled. In this case, the data of
|
||||
Out should not be used, and its data may not be deterministic.
|
||||
Otherwise, FoundInfinite will be 0 (False).
|
||||
Args:
|
||||
x(list|tuple): The input tensors of check_finite_and_unscale operator.
|
||||
scale: The scale of check_finite_and_unscale operator.
|
||||
"""
|
||||
check_type(x, 'x', (tuple, list), 'check_finite_and_unscale')
|
||||
for e in x:
|
||||
check_variable_and_dtype(e, "x", ['float32', 'float64'],
|
||||
'check_finite_and_unscale')
|
||||
|
||||
helper = LayerHelper("check_finite_and_unscale", **locals())
|
||||
found_inf = helper.create_variable_for_type_inference(dtype='bool')
|
||||
|
||||
inputs = {'X': x, 'Scale': scale}
|
||||
outputs = {'Out': x, 'FoundInfinite': found_inf}
|
||||
helper.append_op(
|
||||
type='check_finite_and_unscale', inputs=inputs, outputs=outputs)
|
||||
|
||||
return x, found_inf
|
||||
|
||||
|
||||
def update_loss_scaling(x,
|
||||
found_inf,
|
||||
prev_loss_scaling,
|
||||
num_good_steps,
|
||||
num_bad_steps,
|
||||
incr_every_n_steps,
|
||||
decr_every_n_nan_or_inf,
|
||||
incr_ratio,
|
||||
decr_ratio,
|
||||
name=None):
|
||||
"""
|
||||
Update loss scaling according to overall gradients. If all gradients is
|
||||
finite after incr_every_n_steps, loss scaling will increase by incr_ratio.
|
||||
Otherwise, loss scaling will decrease by decr_ratio after
|
||||
decr_every_n_nan_or_inf steps and each step some gradients are infinite.
|
||||
|
||||
Args:
|
||||
x(list|tuple): The input tensors of update_loss_scaling operator.
|
||||
found_inf (Variable): A boolean variable indicates whether
|
||||
there is any infinite gradient.
|
||||
prev_loss_scaling (Variable): Previous loss scaling.
|
||||
num_good_steps (Variable): A variable accumulates good steps in which
|
||||
all gradients are finite.
|
||||
num_bad_steps (Variable): A variable accumulates bad steps in which
|
||||
some gradients are infinite.
|
||||
incr_every_n_steps (int): A variable represents increasing loss
|
||||
scaling every n consecutive steps with
|
||||
finite gradients.
|
||||
decr_every_n_nan_or_inf (int): A variable represents decreasing
|
||||
loss scaling every n accumulated
|
||||
steps with nan or inf gradients.
|
||||
incr_ratio(float): The multiplier to use when increasing the loss
|
||||
scaling.
|
||||
decr_ratio(float): The less-than-one-multiplier to use when decreasing
|
||||
loss scaling.
|
||||
"""
|
||||
|
||||
check_variable_and_dtype(prev_loss_scaling, "prev_loss_scaling",
|
||||
['float32', 'float64'], "update_loss_scaling")
|
||||
check_type(x, 'x', (tuple, list), 'update_loss_scaling')
|
||||
for e in x:
|
||||
check_variable_and_dtype(e, "x", ['float32', 'float64'],
|
||||
'update_loss_scaling')
|
||||
assert prev_loss_scaling.dtype == e.dtype, "The dtype of prev_loss_scaling should be equal to the dtype of x."
|
||||
|
||||
helper = LayerHelper("update_loss_scaling", **locals())
|
||||
|
||||
inputs = {
|
||||
'X': x,
|
||||
'FoundInfinite': found_inf,
|
||||
'PrevLossScaling': prev_loss_scaling,
|
||||
'InGoodSteps': num_good_steps,
|
||||
'InBadSteps': num_bad_steps
|
||||
}
|
||||
|
||||
outputs = {
|
||||
'Out': x,
|
||||
'LossScaling': prev_loss_scaling,
|
||||
'OutGoodSteps': num_good_steps,
|
||||
'OutBadSteps': num_bad_steps
|
||||
}
|
||||
|
||||
attrs = {
|
||||
'incr_every_n_steps': incr_every_n_steps,
|
||||
'decr_every_n_nan_or_inf': decr_every_n_nan_or_inf,
|
||||
'incr_ratio': incr_ratio,
|
||||
'decr_ratio': decr_ratio,
|
||||
}
|
||||
|
||||
helper.append_op(
|
||||
type='update_loss_scaling', inputs=inputs, outputs=outputs, attrs=attrs)
|
||||
|
||||
return x
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue