You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
504 lines
16 KiB
504 lines
16 KiB
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
#Licensed under the Apache License, Version 2.0 (the "License");
|
|
#you may not use this file except in compliance with the License.
|
|
#You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
#Unless required by applicable law or agreed to in writing, software
|
|
#distributed under the License is distributed on an "AS IS" BASIS,
|
|
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
#See the License for the specific language governing permissions and
|
|
#limitations under the License.
|
|
|
|
from __future__ import print_function
|
|
import unittest
|
|
import numpy as np
|
|
import copy
|
|
from op_test import OpTest
|
|
|
|
|
|
def iou(box_a, box_b, norm):
|
|
"""Apply intersection-over-union overlap between box_a and box_b
|
|
"""
|
|
xmin_a = min(box_a[0], box_a[2])
|
|
ymin_a = min(box_a[1], box_a[3])
|
|
xmax_a = max(box_a[0], box_a[2])
|
|
ymax_a = max(box_a[1], box_a[3])
|
|
|
|
xmin_b = min(box_b[0], box_b[2])
|
|
ymin_b = min(box_b[1], box_b[3])
|
|
xmax_b = max(box_b[0], box_b[2])
|
|
ymax_b = max(box_b[1], box_b[3])
|
|
|
|
area_a = (ymax_a - ymin_a + (norm == False)) * (xmax_a - xmin_a +
|
|
(norm == False))
|
|
area_b = (ymax_b - ymin_b + (norm == False)) * (xmax_b - xmin_b +
|
|
(norm == False))
|
|
if area_a <= 0 and area_b <= 0:
|
|
return 0.0
|
|
|
|
xa = max(xmin_a, xmin_b)
|
|
ya = max(ymin_a, ymin_b)
|
|
xb = min(xmax_a, xmax_b)
|
|
yb = min(ymax_a, ymax_b)
|
|
|
|
inter_area = max(xb - xa + (norm == False),
|
|
0.0) * max(yb - ya + (norm == False), 0.0)
|
|
|
|
iou_ratio = inter_area / (area_a + area_b - inter_area)
|
|
|
|
return iou_ratio
|
|
|
|
|
|
def nms(boxes,
|
|
scores,
|
|
score_threshold,
|
|
nms_threshold,
|
|
top_k=200,
|
|
normalized=True,
|
|
eta=1.0):
|
|
"""Apply non-maximum suppression at test time to avoid detecting too many
|
|
overlapping bounding boxes for a given object.
|
|
Args:
|
|
boxes: (tensor) The location preds for the img, Shape: [num_priors,4].
|
|
scores: (tensor) The class predscores for the img, Shape:[num_priors].
|
|
score_threshold: (float) The confidence thresh for filtering low
|
|
confidence boxes.
|
|
nms_threshold: (float) The overlap thresh for suppressing unnecessary
|
|
boxes.
|
|
top_k: (int) The maximum number of box preds to consider.
|
|
eta: (float) The parameter for adaptive NMS.
|
|
Return:
|
|
The indices of the kept boxes with respect to num_priors.
|
|
"""
|
|
all_scores = copy.deepcopy(scores)
|
|
all_scores = all_scores.flatten()
|
|
selected_indices = np.argwhere(all_scores > score_threshold)
|
|
selected_indices = selected_indices.flatten()
|
|
all_scores = all_scores[selected_indices]
|
|
|
|
sorted_indices = np.argsort(-all_scores, axis=0, kind='mergesort')
|
|
sorted_scores = all_scores[sorted_indices]
|
|
sorted_indices = selected_indices[sorted_indices]
|
|
if top_k > -1 and top_k < sorted_indices.shape[0]:
|
|
sorted_indices = sorted_indices[:top_k]
|
|
sorted_scores = sorted_scores[:top_k]
|
|
|
|
selected_indices = []
|
|
adaptive_threshold = nms_threshold
|
|
for i in range(sorted_scores.shape[0]):
|
|
idx = sorted_indices[i]
|
|
keep = True
|
|
for k in range(len(selected_indices)):
|
|
if keep:
|
|
kept_idx = selected_indices[k]
|
|
overlap = iou(boxes[idx], boxes[kept_idx], normalized)
|
|
keep = True if overlap <= adaptive_threshold else False
|
|
else:
|
|
break
|
|
if keep:
|
|
selected_indices.append(idx)
|
|
if keep and eta < 1 and adaptive_threshold > 0.5:
|
|
adaptive_threshold *= eta
|
|
return selected_indices
|
|
|
|
|
|
def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold,
|
|
nms_top_k, keep_top_k, normalized, shared):
|
|
if shared:
|
|
class_num = scores.shape[0]
|
|
priorbox_num = scores.shape[1]
|
|
else:
|
|
box_num = scores.shape[0]
|
|
class_num = scores.shape[1]
|
|
|
|
selected_indices = {}
|
|
num_det = 0
|
|
for c in range(class_num):
|
|
if c == background: continue
|
|
if shared:
|
|
indices = nms(boxes, scores[c], score_threshold, nms_threshold,
|
|
nms_top_k, normalized)
|
|
else:
|
|
indices = nms(boxes[:, c, :], scores[:, c], score_threshold,
|
|
nms_threshold, nms_top_k, normalized)
|
|
selected_indices[c] = indices
|
|
num_det += len(indices)
|
|
|
|
if keep_top_k > -1 and num_det > keep_top_k:
|
|
score_index = []
|
|
for c, indices in selected_indices.items():
|
|
for idx in indices:
|
|
if shared:
|
|
score_index.append((scores[c][idx], c, idx))
|
|
else:
|
|
score_index.append((scores[idx][c], c, idx))
|
|
|
|
sorted_score_index = sorted(
|
|
score_index, key=lambda tup: tup[0], reverse=True)
|
|
sorted_score_index = sorted_score_index[:keep_top_k]
|
|
selected_indices = {}
|
|
|
|
for _, c, _ in sorted_score_index:
|
|
selected_indices[c] = []
|
|
for s, c, idx in sorted_score_index:
|
|
selected_indices[c].append(idx)
|
|
if not shared:
|
|
for labels in selected_indices:
|
|
selected_indices[labels].sort()
|
|
num_det = keep_top_k
|
|
|
|
return selected_indices, num_det
|
|
|
|
|
|
def lod_multiclass_nms(boxes, scores, background, score_threshold,
|
|
nms_threshold, nms_top_k, keep_top_k, box_lod,
|
|
normalized):
|
|
num_class = boxes.shape[1]
|
|
det_outs = []
|
|
lod = []
|
|
head = 0
|
|
for n in range(len(box_lod[0])):
|
|
box = boxes[head:head + box_lod[0][n]]
|
|
score = scores[head:head + box_lod[0][n]]
|
|
offset = head
|
|
head = head + box_lod[0][n]
|
|
nmsed_outs, nmsed_num = multiclass_nms(
|
|
box,
|
|
score,
|
|
background,
|
|
score_threshold,
|
|
nms_threshold,
|
|
nms_top_k,
|
|
keep_top_k,
|
|
normalized,
|
|
shared=False)
|
|
lod.append(nmsed_num)
|
|
|
|
if nmsed_num == 0:
|
|
continue
|
|
tmp_det_out = []
|
|
for c, indices in nmsed_outs.items():
|
|
for idx in indices:
|
|
xmin, ymin, xmax, ymax = box[idx, c, :]
|
|
tmp_det_out.append([
|
|
c, score[idx][c], xmin, ymin, xmax, ymax,
|
|
offset * num_class + idx * num_class + c
|
|
])
|
|
sorted_det_out = sorted(
|
|
tmp_det_out, key=lambda tup: tup[0], reverse=False)
|
|
det_outs.extend(sorted_det_out)
|
|
|
|
return det_outs, lod
|
|
|
|
|
|
def batched_multiclass_nms(boxes,
|
|
scores,
|
|
background,
|
|
score_threshold,
|
|
nms_threshold,
|
|
nms_top_k,
|
|
keep_top_k,
|
|
normalized=True):
|
|
batch_size = scores.shape[0]
|
|
num_boxes = scores.shape[2]
|
|
det_outs = []
|
|
index_outs = []
|
|
lod = []
|
|
for n in range(batch_size):
|
|
nmsed_outs, nmsed_num = multiclass_nms(
|
|
boxes[n],
|
|
scores[n],
|
|
background,
|
|
score_threshold,
|
|
nms_threshold,
|
|
nms_top_k,
|
|
keep_top_k,
|
|
normalized,
|
|
shared=True)
|
|
lod.append(nmsed_num)
|
|
|
|
if nmsed_num == 0:
|
|
continue
|
|
tmp_det_out = []
|
|
for c, indices in nmsed_outs.items():
|
|
for idx in indices:
|
|
xmin, ymin, xmax, ymax = boxes[n][idx][:]
|
|
tmp_det_out.append([
|
|
c, scores[n][c][idx], xmin, ymin, xmax, ymax,
|
|
idx + n * num_boxes
|
|
])
|
|
sorted_det_out = sorted(
|
|
tmp_det_out, key=lambda tup: tup[0], reverse=False)
|
|
det_outs.extend(sorted_det_out)
|
|
return det_outs, lod
|
|
|
|
|
|
class TestMulticlassNMSOp(OpTest):
|
|
def set_argument(self):
|
|
self.score_threshold = 0.01
|
|
|
|
def setUp(self):
|
|
self.set_argument()
|
|
N = 7
|
|
M = 1200
|
|
C = 21
|
|
BOX_SIZE = 4
|
|
background = 0
|
|
nms_threshold = 0.3
|
|
nms_top_k = 400
|
|
keep_top_k = 200
|
|
score_threshold = self.score_threshold
|
|
|
|
scores = np.random.random((N * M, C)).astype('float32')
|
|
|
|
def softmax(x):
|
|
shiftx = x - np.max(x).clip(-64.)
|
|
exps = np.exp(shiftx)
|
|
return exps / np.sum(exps)
|
|
|
|
scores = np.apply_along_axis(softmax, 1, scores)
|
|
scores = np.reshape(scores, (N, M, C))
|
|
scores = np.transpose(scores, (0, 2, 1))
|
|
|
|
boxes = np.random.random((N, M, BOX_SIZE)).astype('float32')
|
|
boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5
|
|
boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5
|
|
|
|
det_outs, lod = batched_multiclass_nms(boxes, scores, background,
|
|
score_threshold, nms_threshold,
|
|
nms_top_k, keep_top_k)
|
|
lod = [1] if not det_outs else lod
|
|
det_outs = [[-1, 0]] if not det_outs else det_outs
|
|
det_outs = np.array(det_outs)
|
|
nmsed_outs = det_outs[:, :-1].astype('float32')
|
|
|
|
self.op_type = 'multiclass_nms'
|
|
self.inputs = {'BBoxes': boxes, 'Scores': scores}
|
|
self.outputs = {'Out': (nmsed_outs, [lod])}
|
|
self.attrs = {
|
|
'background_label': 0,
|
|
'nms_threshold': nms_threshold,
|
|
'nms_top_k': nms_top_k,
|
|
'keep_top_k': keep_top_k,
|
|
'score_threshold': score_threshold,
|
|
'nms_eta': 1.0,
|
|
'normalized': True,
|
|
}
|
|
|
|
def test_check_output(self):
|
|
self.check_output()
|
|
|
|
|
|
class TestMulticlassNMSOpNoOutput(TestMulticlassNMSOp):
|
|
def set_argument(self):
|
|
# Here set 2.0 to test the case there is no outputs.
|
|
# In practical use, 0.0 < score_threshold < 1.0
|
|
self.score_threshold = 2.0
|
|
|
|
|
|
class TestMulticlassNMSLoDInput(OpTest):
|
|
def set_argument(self):
|
|
self.score_threshold = 0.01
|
|
|
|
def setUp(self):
|
|
self.set_argument()
|
|
M = 1200
|
|
C = 21
|
|
BOX_SIZE = 4
|
|
box_lod = [[1200]]
|
|
background = 0
|
|
nms_threshold = 0.3
|
|
nms_top_k = 400
|
|
keep_top_k = 200
|
|
score_threshold = self.score_threshold
|
|
normalized = False
|
|
|
|
scores = np.random.random((M, C)).astype('float32')
|
|
|
|
def softmax(x):
|
|
shiftx = x - np.max(x).clip(-64.)
|
|
exps = np.exp(shiftx)
|
|
return exps / np.sum(exps)
|
|
|
|
scores = np.apply_along_axis(softmax, 1, scores)
|
|
|
|
boxes = np.random.random((M, C, BOX_SIZE)).astype('float32')
|
|
boxes[:, :, 0] = boxes[:, :, 0] * 10
|
|
boxes[:, :, 1] = boxes[:, :, 1] * 10
|
|
boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10
|
|
boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10
|
|
|
|
det_outs, lod = lod_multiclass_nms(
|
|
boxes, scores, background, score_threshold, nms_threshold,
|
|
nms_top_k, keep_top_k, box_lod, normalized)
|
|
det_outs = np.array(det_outs).astype('float32')
|
|
nmsed_outs = det_outs[:, :-1].astype('float32') if len(
|
|
det_outs) else det_outs
|
|
self.op_type = 'multiclass_nms'
|
|
self.inputs = {
|
|
'BBoxes': (boxes, box_lod),
|
|
'Scores': (scores, box_lod),
|
|
}
|
|
self.outputs = {'Out': (nmsed_outs, [lod])}
|
|
self.attrs = {
|
|
'background_label': 0,
|
|
'nms_threshold': nms_threshold,
|
|
'nms_top_k': nms_top_k,
|
|
'keep_top_k': keep_top_k,
|
|
'score_threshold': score_threshold,
|
|
'nms_eta': 1.0,
|
|
'normalized': normalized,
|
|
}
|
|
|
|
def test_check_output(self):
|
|
self.check_output()
|
|
|
|
|
|
class TestIOU(unittest.TestCase):
|
|
def test_iou(self):
|
|
box1 = np.array([4.0, 3.0, 7.0, 5.0]).astype('float32')
|
|
box2 = np.array([3.0, 4.0, 6.0, 8.0]).astype('float32')
|
|
|
|
expt_output = np.array([2.0 / 16.0]).astype('float32')
|
|
calc_output = np.array([iou(box1, box2, True)]).astype('float32')
|
|
self.assertTrue(np.allclose(calc_output, expt_output))
|
|
|
|
|
|
class TestMulticlassNMS2Op(TestMulticlassNMSOp):
|
|
def setUp(self):
|
|
self.set_argument()
|
|
N = 7
|
|
M = 1200
|
|
C = 21
|
|
BOX_SIZE = 4
|
|
background = 0
|
|
nms_threshold = 0.3
|
|
nms_top_k = 400
|
|
keep_top_k = 200
|
|
score_threshold = self.score_threshold
|
|
|
|
scores = np.random.random((N * M, C)).astype('float32')
|
|
|
|
def softmax(x):
|
|
shiftx = x - np.max(x).clip(-64.)
|
|
exps = np.exp(shiftx)
|
|
return exps / np.sum(exps)
|
|
|
|
scores = np.apply_along_axis(softmax, 1, scores)
|
|
scores = np.reshape(scores, (N, M, C))
|
|
scores = np.transpose(scores, (0, 2, 1))
|
|
|
|
boxes = np.random.random((N, M, BOX_SIZE)).astype('float32')
|
|
boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5
|
|
boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5
|
|
|
|
det_outs, lod = batched_multiclass_nms(boxes, scores, background,
|
|
score_threshold, nms_threshold,
|
|
nms_top_k, keep_top_k)
|
|
det_outs = np.array(det_outs)
|
|
|
|
nmsed_outs = det_outs[:, :-1].astype('float32') if len(
|
|
det_outs) else det_outs
|
|
index_outs = det_outs[:, -1:].astype('int') if len(
|
|
det_outs) else det_outs
|
|
self.op_type = 'multiclass_nms2'
|
|
self.inputs = {'BBoxes': boxes, 'Scores': scores}
|
|
self.outputs = {
|
|
'Out': (nmsed_outs, [lod]),
|
|
'Index': (index_outs, [lod])
|
|
}
|
|
self.attrs = {
|
|
'background_label': 0,
|
|
'nms_threshold': nms_threshold,
|
|
'nms_top_k': nms_top_k,
|
|
'keep_top_k': keep_top_k,
|
|
'score_threshold': score_threshold,
|
|
'nms_eta': 1.0,
|
|
'normalized': True,
|
|
}
|
|
|
|
def test_check_output(self):
|
|
self.check_output()
|
|
|
|
|
|
class TestMulticlassNMS2OpNoOutput(TestMulticlassNMS2Op):
|
|
def set_argument(self):
|
|
# Here set 2.0 to test the case there is no outputs.
|
|
# In practical use, 0.0 < score_threshold < 1.0
|
|
self.score_threshold = 2.0
|
|
|
|
|
|
class TestMulticlassNMS2LoDInput(TestMulticlassNMSLoDInput):
|
|
def setUp(self):
|
|
self.set_argument()
|
|
M = 1200
|
|
C = 21
|
|
BOX_SIZE = 4
|
|
box_lod = [[1200]]
|
|
background = 0
|
|
nms_threshold = 0.3
|
|
nms_top_k = 400
|
|
keep_top_k = 200
|
|
score_threshold = self.score_threshold
|
|
normalized = False
|
|
|
|
scores = np.random.random((M, C)).astype('float32')
|
|
|
|
def softmax(x):
|
|
shiftx = x - np.max(x).clip(-64.)
|
|
exps = np.exp(shiftx)
|
|
return exps / np.sum(exps)
|
|
|
|
scores = np.apply_along_axis(softmax, 1, scores)
|
|
|
|
boxes = np.random.random((M, C, BOX_SIZE)).astype('float32')
|
|
boxes[:, :, 0] = boxes[:, :, 0] * 10
|
|
boxes[:, :, 1] = boxes[:, :, 1] * 10
|
|
boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10
|
|
boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10
|
|
|
|
det_outs, lod = lod_multiclass_nms(
|
|
boxes, scores, background, score_threshold, nms_threshold,
|
|
nms_top_k, keep_top_k, box_lod, normalized)
|
|
|
|
det_outs = np.array(det_outs)
|
|
nmsed_outs = det_outs[:, :-1].astype('float32') if len(
|
|
det_outs) else det_outs
|
|
index_outs = det_outs[:, -1:].astype('int') if len(
|
|
det_outs) else det_outs
|
|
self.op_type = 'multiclass_nms2'
|
|
self.inputs = {
|
|
'BBoxes': (boxes, box_lod),
|
|
'Scores': (scores, box_lod),
|
|
}
|
|
self.outputs = {
|
|
'Out': (nmsed_outs, [lod]),
|
|
'Index': (index_outs, [lod])
|
|
}
|
|
self.attrs = {
|
|
'background_label': 0,
|
|
'nms_threshold': nms_threshold,
|
|
'nms_top_k': nms_top_k,
|
|
'keep_top_k': keep_top_k,
|
|
'score_threshold': score_threshold,
|
|
'nms_eta': 1.0,
|
|
'normalized': normalized,
|
|
}
|
|
|
|
def test_check_output(self):
|
|
self.check_output()
|
|
|
|
|
|
class TestMulticlassNMS2LoDNoOutput(TestMulticlassNMS2LoDInput):
|
|
def set_argument(self):
|
|
# Here set 2.0 to test the case there is no outputs.
|
|
# In practical use, 0.0 < score_threshold < 1.0
|
|
self.score_threshold = 2.0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|