You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
358 lines
16 KiB
358 lines
16 KiB
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License. */
|
|
|
|
#include "paddle/fluid/operators/detection/yolov3_loss_op.h"
|
|
#include <memory>
|
|
#include "paddle/fluid/framework/op_registry.h"
|
|
#include "paddle/fluid/imperative/type_defs.h"
|
|
|
|
namespace paddle {
|
|
namespace operators {
|
|
|
|
using framework::Tensor;
|
|
|
|
class Yolov3LossOp : public framework::OperatorWithKernel {
|
|
public:
|
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
|
void InferShape(framework::InferShapeContext* ctx) const override {
|
|
OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "Yolov3LossOp");
|
|
OP_INOUT_CHECK(ctx->HasInput("GTBox"), "Input", "GTBox", "Yolov3LossOp");
|
|
OP_INOUT_CHECK(ctx->HasInput("GTLabel"), "Input", "GTLabel",
|
|
"Yolov3LossOp");
|
|
OP_INOUT_CHECK(ctx->HasOutput("Loss"), "Output", "Loss", "Yolov3LossOp");
|
|
OP_INOUT_CHECK(ctx->HasOutput("ObjectnessMask"), "Output", "ObjectnessMask",
|
|
"Yolov3LossOp");
|
|
OP_INOUT_CHECK(ctx->HasOutput("GTMatchMask"), "Output", "GTMatchMask",
|
|
"Yolov3LossOp");
|
|
|
|
auto dim_x = ctx->GetInputDim("X");
|
|
auto dim_gtbox = ctx->GetInputDim("GTBox");
|
|
auto dim_gtlabel = ctx->GetInputDim("GTLabel");
|
|
auto anchors = ctx->Attrs().Get<std::vector<int>>("anchors");
|
|
int anchor_num = anchors.size() / 2;
|
|
auto anchor_mask = ctx->Attrs().Get<std::vector<int>>("anchor_mask");
|
|
int mask_num = anchor_mask.size();
|
|
auto class_num = ctx->Attrs().Get<int>("class_num");
|
|
|
|
PADDLE_ENFORCE_EQ(dim_x.size(), 4,
|
|
platform::errors::InvalidArgument(
|
|
"Input(X) should be a 4-D tensor. But received "
|
|
"X dimension size(%s)",
|
|
dim_x.size()));
|
|
PADDLE_ENFORCE_EQ(dim_x[2], dim_x[3],
|
|
platform::errors::InvalidArgument(
|
|
"Input(X) dim[3] and dim[4] should be euqal."
|
|
"But received dim[3](%s) != dim[4](%s)",
|
|
dim_x[2], dim_x[3]));
|
|
PADDLE_ENFORCE_EQ(
|
|
dim_x[1], mask_num * (5 + class_num),
|
|
platform::errors::InvalidArgument(
|
|
"Input(X) dim[1] should be equal to (anchor_mask_number * (5 "
|
|
"+ class_num))."
|
|
"But received dim[1](%s) != (anchor_mask_number * "
|
|
"(5+class_num)(%s).",
|
|
dim_x[1], mask_num * (5 + class_num)));
|
|
PADDLE_ENFORCE_EQ(dim_gtbox.size(), 3,
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTBox) should be a 3-D tensor, but "
|
|
"received gtbox dimension size(%s)",
|
|
dim_gtbox.size()));
|
|
PADDLE_ENFORCE_EQ(dim_gtbox[2], 4,
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTBox) dim[2] should be 4",
|
|
"But receive dim[2](%s) != 5. ", dim_gtbox[2]));
|
|
PADDLE_ENFORCE_EQ(
|
|
dim_gtlabel.size(), 2,
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTLabel) should be a 2-D tensor,"
|
|
"But received Input(GTLabel) dimension size(%s) != 2.",
|
|
dim_gtlabel.size()));
|
|
PADDLE_ENFORCE_EQ(
|
|
dim_gtlabel[0], dim_gtbox[0],
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTBox) dim[0] and Input(GTLabel) dim[0] should be same,"
|
|
"But received Input(GTLabel) dim[0](%s) != "
|
|
"Input(GTBox) dim[0](%s)",
|
|
dim_gtlabel[0], dim_gtbox[0]));
|
|
PADDLE_ENFORCE_EQ(
|
|
dim_gtlabel[1], dim_gtbox[1],
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTBox) and Input(GTLabel) dim[1] should be same,"
|
|
"But received Input(GTBox) dim[1](%s) != Input(GTLabel) "
|
|
"dim[1](%s)",
|
|
dim_gtbox[1], dim_gtlabel[1]));
|
|
PADDLE_ENFORCE_GT(anchors.size(), 0,
|
|
platform::errors::InvalidArgument(
|
|
"Attr(anchors) length should be greater then 0."
|
|
"But received anchors length(%s)",
|
|
anchors.size()));
|
|
PADDLE_ENFORCE_EQ(anchors.size() % 2, 0,
|
|
platform::errors::InvalidArgument(
|
|
"Attr(anchors) length should be even integer."
|
|
"But received anchors length(%s)",
|
|
anchors.size()));
|
|
for (size_t i = 0; i < anchor_mask.size(); i++) {
|
|
PADDLE_ENFORCE_LT(
|
|
anchor_mask[i], anchor_num,
|
|
platform::errors::InvalidArgument(
|
|
"Attr(anchor_mask) should not crossover Attr(anchors)."
|
|
"But received anchor_mask[i](%s) > anchor_num(%s)",
|
|
anchor_mask[i], anchor_num));
|
|
}
|
|
PADDLE_ENFORCE_GT(class_num, 0,
|
|
platform::errors::InvalidArgument(
|
|
"Attr(class_num) should be an integer greater then 0."
|
|
"But received class_num(%s) < 0",
|
|
class_num));
|
|
|
|
if (ctx->HasInput("GTScore")) {
|
|
auto dim_gtscore = ctx->GetInputDim("GTScore");
|
|
PADDLE_ENFORCE_EQ(dim_gtscore.size(), 2,
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTScore) should be a 2-D tensor"
|
|
"But received GTScore dimension(%s)",
|
|
dim_gtbox.size()));
|
|
PADDLE_ENFORCE_EQ(
|
|
dim_gtscore[0], dim_gtbox[0],
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTBox) and Input(GTScore) dim[0] should be same"
|
|
"But received GTBox dim[0](%s) != GTScore dim[0](%s)",
|
|
dim_gtbox[0], dim_gtscore[0]));
|
|
PADDLE_ENFORCE_EQ(
|
|
dim_gtscore[1], dim_gtbox[1],
|
|
platform::errors::InvalidArgument(
|
|
"Input(GTBox) and Input(GTScore) dim[1] should be same"
|
|
"But received GTBox dim[1](%s) != GTScore dim[1](%s)",
|
|
dim_gtscore[1], dim_gtbox[1]));
|
|
}
|
|
|
|
std::vector<int64_t> dim_out({dim_x[0]});
|
|
ctx->SetOutputDim("Loss", framework::make_ddim(dim_out));
|
|
|
|
std::vector<int64_t> dim_obj_mask({dim_x[0], mask_num, dim_x[2], dim_x[3]});
|
|
ctx->SetOutputDim("ObjectnessMask", framework::make_ddim(dim_obj_mask));
|
|
|
|
std::vector<int64_t> dim_gt_match_mask({dim_gtbox[0], dim_gtbox[1]});
|
|
ctx->SetOutputDim("GTMatchMask", framework::make_ddim(dim_gt_match_mask));
|
|
}
|
|
|
|
protected:
|
|
framework::OpKernelType GetExpectedKernelType(
|
|
const framework::ExecutionContext& ctx) const override {
|
|
return framework::OpKernelType(
|
|
OperatorWithKernel::IndicateVarDataType(ctx, "X"),
|
|
platform::CPUPlace());
|
|
}
|
|
};
|
|
|
|
class Yolov3LossOpMaker : public framework::OpProtoAndCheckerMaker {
|
|
public:
|
|
void Make() override {
|
|
AddInput("X",
|
|
"The input tensor of YOLOv3 loss operator, "
|
|
"This is a 4-D tensor with shape of [N, C, H, W]."
|
|
"H and W should be same, and the second dimension(C) stores"
|
|
"box locations, confidence score and classification one-hot"
|
|
"keys of each anchor box");
|
|
AddInput("GTBox",
|
|
"The input tensor of ground truth boxes, "
|
|
"This is a 3-D tensor with shape of [N, max_box_num, 5], "
|
|
"max_box_num is the max number of boxes in each image, "
|
|
"In the third dimension, stores x, y, w, h coordinates, "
|
|
"x, y is the center coordinate of boxes and w, h is the "
|
|
"width and height and x, y, w, h should be divided by "
|
|
"input image height to scale to [0, 1].");
|
|
AddInput("GTLabel",
|
|
"The input tensor of ground truth label, "
|
|
"This is a 2-D tensor with shape of [N, max_box_num], "
|
|
"and each element should be an integer to indicate the "
|
|
"box class id.");
|
|
AddInput("GTScore",
|
|
"The score of GTLabel, This is a 2-D tensor in same shape "
|
|
"GTLabel, and score values should in range (0, 1). This "
|
|
"input is for GTLabel score can be not 1.0 in image mixup "
|
|
"augmentation.")
|
|
.AsDispensable();
|
|
AddOutput("Loss",
|
|
"The output yolov3 loss tensor, "
|
|
"This is a 1-D tensor with shape of [N]");
|
|
AddOutput("ObjectnessMask",
|
|
"This is an intermediate tensor with shape of [N, M, H, W], "
|
|
"M is the number of anchor masks. This parameter caches the "
|
|
"mask for calculate objectness loss in gradient kernel.")
|
|
.AsIntermediate();
|
|
AddOutput("GTMatchMask",
|
|
"This is an intermediate tensor with shape of [N, B], "
|
|
"B is the max box number of GT boxes. This parameter caches "
|
|
"matched mask index of each GT boxes for gradient calculate.")
|
|
.AsIntermediate();
|
|
|
|
AddAttr<int>("class_num", "The number of classes to predict.");
|
|
AddAttr<std::vector<int>>("anchors",
|
|
"The anchor width and height, "
|
|
"it will be parsed pair by pair.")
|
|
.SetDefault(std::vector<int>{});
|
|
AddAttr<std::vector<int>>("anchor_mask",
|
|
"The mask index of anchors used in "
|
|
"current YOLOv3 loss calculation.")
|
|
.SetDefault(std::vector<int>{});
|
|
AddAttr<int>("downsample_ratio",
|
|
"The downsample ratio from network input to YOLOv3 loss "
|
|
"input, so 32, 16, 8 should be set for the first, second, "
|
|
"and thrid YOLOv3 loss operators.")
|
|
.SetDefault(32);
|
|
AddAttr<float>("ignore_thresh",
|
|
"The ignore threshold to ignore confidence loss.")
|
|
.SetDefault(0.7);
|
|
AddAttr<bool>("use_label_smooth",
|
|
"Whether to use label smooth. Default True.")
|
|
.SetDefault(true);
|
|
AddAttr<float>("scale_x_y",
|
|
"Scale the center point of decoded bounding "
|
|
"box. Default 1.0")
|
|
.SetDefault(1.);
|
|
AddComment(R"DOC(
|
|
This operator generates yolov3 loss based on given predict result and ground
|
|
truth boxes.
|
|
|
|
The output of previous network is in shape [N, C, H, W], while H and W
|
|
should be the same, H and W specify the grid size, each grid point predict
|
|
given number bounding boxes, this given number, which following will be represented as S,
|
|
is specified by the number of anchor clusters in each scale. In the second dimension(the channel
|
|
dimension), C should be equal to S * (class_num + 5), class_num is the object
|
|
category number of source dataset(such as 80 in coco dataset), so in the
|
|
second(channel) dimension, apart from 4 box location coordinates x, y, w, h,
|
|
also includes confidence score of the box and class one-hot key of each anchor box.
|
|
|
|
Assume the 4 location coordinates are :math:`t_x, t_y, t_w, t_h`, the box predictions
|
|
should be as follows:
|
|
|
|
$$
|
|
b_x = \\sigma(t_x) + c_x
|
|
$$
|
|
$$
|
|
b_y = \\sigma(t_y) + c_y
|
|
$$
|
|
$$
|
|
b_w = p_w e^{t_w}
|
|
$$
|
|
$$
|
|
b_h = p_h e^{t_h}
|
|
$$
|
|
|
|
In the equation above, :math:`c_x, c_y` is the left top corner of current grid
|
|
and :math:`p_w, p_h` is specified by anchors.
|
|
|
|
As for confidence score, it is the logistic regression value of IoU between
|
|
anchor boxes and ground truth boxes, the score of the anchor box which has
|
|
the max IoU should be 1, and if the anchor box has IoU bigger than ignore
|
|
thresh, the confidence score loss of this anchor box will be ignored.
|
|
|
|
Therefore, the yolov3 loss consists of three major parts: box location loss,
|
|
objectness loss and classification loss. The L1 loss is used for
|
|
box coordinates (w, h), sigmoid cross entropy loss is used for box
|
|
coordinates (x, y), objectness loss and classification loss.
|
|
|
|
Each groud truth box finds a best matching anchor box in all anchors.
|
|
Prediction of this anchor box will incur all three parts of losses, and
|
|
prediction of anchor boxes with no GT box matched will only incur objectness
|
|
loss.
|
|
|
|
In order to trade off box coordinate losses between big boxes and small
|
|
boxes, box coordinate losses will be mutiplied by scale weight, which is
|
|
calculated as follows.
|
|
|
|
$$
|
|
weight_{box} = 2.0 - t_w * t_h
|
|
$$
|
|
|
|
Final loss will be represented as follows.
|
|
|
|
$$
|
|
loss = (loss_{xy} + loss_{wh}) * weight_{box}
|
|
+ loss_{conf} + loss_{class}
|
|
$$
|
|
|
|
While :attr:`use_label_smooth` is set to be :attr:`True`, the classification
|
|
target will be smoothed when calculating classification loss, target of
|
|
positive samples will be smoothed to :math:`1.0 - 1.0 / class\_num` and target of
|
|
negetive samples will be smoothed to :math:`1.0 / class\_num`.
|
|
|
|
While :attr:`GTScore` is given, which means the mixup score of ground truth
|
|
boxes, all losses incured by a ground truth box will be multiplied by its
|
|
mixup score.
|
|
)DOC");
|
|
}
|
|
};
|
|
|
|
class Yolov3LossOpGrad : public framework::OperatorWithKernel {
|
|
public:
|
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
|
void InferShape(framework::InferShapeContext* ctx) const override {
|
|
PADDLE_ENFORCE_EQ(
|
|
ctx->HasInput("X"), true,
|
|
platform::errors::NotFound("Input(X) should not be null"));
|
|
PADDLE_ENFORCE_EQ(
|
|
ctx->HasInput(framework::GradVarName("Loss")), true,
|
|
platform::errors::NotFound("Input(Loss@GRAD) should not be null"));
|
|
auto dim_x = ctx->GetInputDim("X");
|
|
if (ctx->HasOutput(framework::GradVarName("X"))) {
|
|
ctx->SetOutputDim(framework::GradVarName("X"), dim_x);
|
|
}
|
|
}
|
|
|
|
protected:
|
|
framework::OpKernelType GetExpectedKernelType(
|
|
const framework::ExecutionContext& ctx) const override {
|
|
return framework::OpKernelType(
|
|
OperatorWithKernel::IndicateVarDataType(ctx, "X"),
|
|
platform::CPUPlace());
|
|
}
|
|
};
|
|
|
|
template <typename T>
|
|
class Yolov3LossGradMaker : public framework::SingleGradOpMaker<T> {
|
|
public:
|
|
using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
|
|
|
|
protected:
|
|
void Apply(GradOpPtr<T> op) const override {
|
|
op->SetType("yolov3_loss_grad");
|
|
op->SetInput("X", this->Input("X"));
|
|
op->SetInput("GTBox", this->Input("GTBox"));
|
|
op->SetInput("GTLabel", this->Input("GTLabel"));
|
|
op->SetInput("GTScore", this->Input("GTScore"));
|
|
op->SetInput(framework::GradVarName("Loss"), this->OutputGrad("Loss"));
|
|
op->SetInput("ObjectnessMask", this->Output("ObjectnessMask"));
|
|
op->SetInput("GTMatchMask", this->Output("GTMatchMask"));
|
|
|
|
op->SetAttrMap(this->Attrs());
|
|
|
|
op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
|
|
op->SetOutput(framework::GradVarName("GTBox"), this->EmptyInputGrad());
|
|
op->SetOutput(framework::GradVarName("GTLabel"), this->EmptyInputGrad());
|
|
op->SetOutput(framework::GradVarName("GTScore"), this->EmptyInputGrad());
|
|
}
|
|
};
|
|
|
|
} // namespace operators
|
|
} // namespace paddle
|
|
|
|
namespace ops = paddle::operators;
|
|
REGISTER_OPERATOR(yolov3_loss, ops::Yolov3LossOp, ops::Yolov3LossOpMaker,
|
|
ops::Yolov3LossGradMaker<paddle::framework::OpDesc>,
|
|
ops::Yolov3LossGradMaker<paddle::imperative::OpBase>);
|
|
REGISTER_OPERATOR(yolov3_loss_grad, ops::Yolov3LossOpGrad);
|
|
REGISTER_OP_CPU_KERNEL(yolov3_loss, ops::Yolov3LossKernel<float>,
|
|
ops::Yolov3LossKernel<double>);
|
|
REGISTER_OP_CPU_KERNEL(yolov3_loss_grad, ops::Yolov3LossGradKernel<float>,
|
|
ops::Yolov3LossGradKernel<double>);
|