Unify the metrics implementation between low-level and high-level API. (#26158)
* Move paddle/incubate/hapi/metrics to paddle/metric * Add Precision, Recall and Auc metrictest_feature_precision_test_c
qingqing01
5 years ago
committed by
GitHub
parent
da1efe24d5
commit
78ca8cf0c8
@ -1,49 +0,0 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.fluid.framework import Program, program_guard
|
||||
|
||||
|
||||
class TestMetricsDetectionMap(unittest.TestCase):
|
||||
def test_detection_map(self):
|
||||
program = fluid.Program()
|
||||
with program_guard(program):
|
||||
detect_res = fluid.layers.data(
|
||||
name='detect_res',
|
||||
shape=[10, 6],
|
||||
append_batch_size=False,
|
||||
dtype='float32')
|
||||
label = fluid.layers.data(
|
||||
name='label',
|
||||
shape=[10, 1],
|
||||
append_batch_size=False,
|
||||
dtype='float32')
|
||||
box = fluid.layers.data(
|
||||
name='bbox',
|
||||
shape=[10, 4],
|
||||
append_batch_size=False,
|
||||
dtype='float32')
|
||||
map_eval = fluid.metrics.DetectionMAP(
|
||||
detect_res, label, box, class_num=21)
|
||||
cur_map, accm_map = map_eval.get_map_var()
|
||||
self.assertIsNotNone(cur_map)
|
||||
self.assertIsNotNone(accm_map)
|
||||
print(str(program))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -1,233 +0,0 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import six
|
||||
import abc
|
||||
import numpy as np
|
||||
import paddle.fluid as fluid
|
||||
|
||||
import logging
|
||||
|
||||
FORMAT = '%(asctime)s-%(levelname)s: %(message)s'
|
||||
logging.basicConfig(level=logging.INFO, format=FORMAT)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ['Metric', 'Accuracy']
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class Metric(object):
|
||||
"""
|
||||
Base class for metric, encapsulates metric logic and APIs
|
||||
Usage:
|
||||
|
||||
m = SomeMetric()
|
||||
for prediction, label in ...:
|
||||
m.update(prediction, label)
|
||||
m.accumulate()
|
||||
|
||||
Advanced usage for :code:`add_metric_op`
|
||||
Metric calculation can be accelerated by calculating metric states
|
||||
from model outputs and labels by Paddle OPs in :code:`add_metric_op`,
|
||||
metric states will be fetch as numpy array and call :code:`update`
|
||||
with states in numpy format.
|
||||
Metric calculated as follows (operations in Model and Metric are
|
||||
indicated with curly brackets, while data nodes not):
|
||||
inputs & labels || ------------------
|
||||
| ||
|
||||
{model} ||
|
||||
| ||
|
||||
outputs & labels ||
|
||||
| || tensor data
|
||||
{Metric.add_metric_op} ||
|
||||
| ||
|
||||
metric states(tensor) ||
|
||||
| ||
|
||||
{fetch as numpy} || ------------------
|
||||
| ||
|
||||
metric states(numpy) || numpy data
|
||||
| ||
|
||||
{Metric.update} \/ ------------------
|
||||
Examples:
|
||||
|
||||
For :code:`Accuracy` metric, which takes :code:`pred` and :code:`label`
|
||||
as inputs, we can calculate the correct prediction matrix between
|
||||
:code:`pred` and :code:`label` in :code:`add_metric_op`.
|
||||
For examples, prediction results contains 10 classes, while :code:`pred`
|
||||
shape is [N, 10], :code:`label` shape is [N, 1], N is mini-batch size,
|
||||
and we only need to calculate accurary of top-1 and top-5, we could
|
||||
calculated the correct prediction matrix of the top-5 scores of the
|
||||
prediction of each sample like follows, while the correct prediction
|
||||
matrix shape is [N, 5].
|
||||
.. code-block:: python
|
||||
def add_metric_op(pred, label):
|
||||
# sort prediction and slice the top-5 scores
|
||||
pred = fluid.layers.argsort(pred, descending=True)[1][:, :5]
|
||||
# calculate whether the predictions are correct
|
||||
correct = pred == label
|
||||
return fluid.layers.cast(correct, dtype='float32')
|
||||
With the :code:`add_metric_op`, we split some calculations to OPs(which
|
||||
may run on GPU devices, will be faster), and only fetch 1 tensor with
|
||||
shape as [N, 5] instead of 2 tensors with shapes as [N, 10] and [N, 1].
|
||||
:code:`update` can be define as follows:
|
||||
.. code-block:: python
|
||||
def update(self, correct):
|
||||
accs = []
|
||||
for i, k in enumerate(self.topk):
|
||||
num_corrects = correct[:, :k].sum()
|
||||
num_samples = len(correct)
|
||||
accs.append(float(num_corrects) / num_samples)
|
||||
self.total[i] += num_corrects
|
||||
self.count[i] += num_samples
|
||||
return accs
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def reset(self):
|
||||
"""
|
||||
Reset states and result
|
||||
"""
|
||||
raise NotImplementedError("function 'reset' not implemented in {}.".
|
||||
format(self.__class__.__name__))
|
||||
|
||||
@abc.abstractmethod
|
||||
def update(self, *args):
|
||||
"""
|
||||
Update states for metric
|
||||
|
||||
Inputs of :code:`update` is the outputs of :code:`Metric.add_metric_op`,
|
||||
if :code:`add_metric_op` is not defined, the inputs of :code:`update`
|
||||
will be flatten arguments of **output** of mode and **label** from data:
|
||||
:code:`update(output1, output2, ..., label1, label2,...)`
|
||||
|
||||
see :code:`Metric.add_metric_op`
|
||||
"""
|
||||
raise NotImplementedError("function 'update' not implemented in {}.".
|
||||
format(self.__class__.__name__))
|
||||
|
||||
@abc.abstractmethod
|
||||
def accumulate(self):
|
||||
"""
|
||||
Accumulates statistics, computes and returns the metric value
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"function 'accumulate' not implemented in {}.".format(
|
||||
self.__class__.__name__))
|
||||
|
||||
@abc.abstractmethod
|
||||
def name(self):
|
||||
"""
|
||||
Returns metric name
|
||||
"""
|
||||
raise NotImplementedError("function 'name' not implemented in {}.".
|
||||
format(self.__class__.__name__))
|
||||
|
||||
def add_metric_op(self, *args):
|
||||
"""
|
||||
This API is advanced usage to accelerate metric calculating, calulations
|
||||
from outputs of model to the states which should be updated by Metric can
|
||||
be defined here, where Paddle OPs is also supported. Outputs of this API
|
||||
will be the inputs of "Metric.update".
|
||||
|
||||
If :code:`add_metric_op` is defined, it will be called with **outputs**
|
||||
of model and **labels** from data as arguments, all outputs and labels
|
||||
will be concatenated and flatten and each filed as a separate argument
|
||||
as follows:
|
||||
:code:`add_metric_op(output1, output2, ..., label1, label2,...)`
|
||||
|
||||
If :code:`add_metric_op` is not defined, default behaviour is to pass
|
||||
input to output, so output format will be:
|
||||
:code:`return output1, output2, ..., label1, label2,...`
|
||||
|
||||
see :code:`Metric.update`
|
||||
"""
|
||||
return args
|
||||
|
||||
|
||||
class Accuracy(Metric):
|
||||
"""
|
||||
Encapsulates accuracy metric logic
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import paddle
|
||||
import paddle.fluid as fluid
|
||||
import paddle.incubate.hapi as hapi
|
||||
|
||||
fluid.enable_dygraph()
|
||||
|
||||
train_dataset = hapi.datasets.MNIST(mode='train')
|
||||
|
||||
model = hapi.Model(hapi.vision.LeNet(classifier_activation=None))
|
||||
optim = fluid.optimizer.Adam(
|
||||
learning_rate=0.001, parameter_list=model.parameters())
|
||||
model.prepare(
|
||||
optim,
|
||||
loss_function=paddle.nn.CrossEntropyLoss(),
|
||||
metrics=hapi.metrics.Accuracy())
|
||||
|
||||
model.fit(train_dataset, batch_size=64)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, topk=(1, ), name=None, *args, **kwargs):
|
||||
super(Accuracy, self).__init__(*args, **kwargs)
|
||||
self.topk = topk
|
||||
self.maxk = max(topk)
|
||||
self._init_name(name)
|
||||
self.reset()
|
||||
|
||||
def add_metric_op(self, pred, label, *args):
|
||||
pred = fluid.layers.argsort(pred, descending=True)[1][:, :self.maxk]
|
||||
correct = pred == label
|
||||
return fluid.layers.cast(correct, dtype='float32')
|
||||
|
||||
def update(self, correct, *args):
|
||||
accs = []
|
||||
for i, k in enumerate(self.topk):
|
||||
num_corrects = correct[:, :k].sum()
|
||||
num_samples = len(correct)
|
||||
accs.append(float(num_corrects) / num_samples)
|
||||
self.total[i] += num_corrects
|
||||
self.count[i] += num_samples
|
||||
return accs
|
||||
|
||||
def reset(self):
|
||||
self.total = [0.] * len(self.topk)
|
||||
self.count = [0] * len(self.topk)
|
||||
|
||||
def accumulate(self):
|
||||
res = []
|
||||
for t, c in zip(self.total, self.count):
|
||||
res.append(float(t) / c)
|
||||
return res
|
||||
|
||||
def _init_name(self, name):
|
||||
name = name or 'acc'
|
||||
if self.maxk != 1:
|
||||
self._name = ['{}_top{}'.format(name, k) for k in self.topk]
|
||||
else:
|
||||
self._name = [name]
|
||||
|
||||
def name(self):
|
||||
return self._name
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,133 +0,0 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import unittest
|
||||
import numpy as np
|
||||
|
||||
import paddle.fluid as fluid
|
||||
from paddle.fluid.dygraph.base import to_variable
|
||||
|
||||
from paddle.incubate.hapi.metrics import *
|
||||
from paddle.incubate.hapi.utils import to_list
|
||||
|
||||
|
||||
def accuracy(pred, label, topk=(1, )):
|
||||
maxk = max(topk)
|
||||
pred = np.argsort(pred)[:, ::-1][:, :maxk]
|
||||
correct = (pred == np.repeat(label, maxk, 1))
|
||||
|
||||
batch_size = label.shape[0]
|
||||
res = []
|
||||
for k in topk:
|
||||
correct_k = correct[:, :k].sum()
|
||||
res.append(correct_k / batch_size)
|
||||
return res
|
||||
|
||||
|
||||
def convert_to_one_hot(y, C):
|
||||
oh = np.random.choice(np.arange(C), C, replace=False).astype('float32') / C
|
||||
oh = np.tile(oh[np.newaxis, :], (y.shape[0], 1))
|
||||
for i in range(y.shape[0]):
|
||||
oh[i, int(y[i])] = 1.
|
||||
return oh
|
||||
|
||||
|
||||
class TestAccuracyDynamic(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.topk = (1, )
|
||||
self.class_num = 5
|
||||
self.sample_num = 1000
|
||||
self.name = None
|
||||
|
||||
def random_pred_label(self):
|
||||
label = np.random.randint(0, self.class_num,
|
||||
(self.sample_num, 1)).astype('int64')
|
||||
pred = np.random.randint(0, self.class_num,
|
||||
(self.sample_num, 1)).astype('int32')
|
||||
pred_one_hot = convert_to_one_hot(pred, self.class_num)
|
||||
pred_one_hot = pred_one_hot.astype('float32')
|
||||
|
||||
return label, pred_one_hot
|
||||
|
||||
def test_main(self):
|
||||
with fluid.dygraph.guard(fluid.CPUPlace()):
|
||||
acc = Accuracy(topk=self.topk, name=self.name)
|
||||
for _ in range(10):
|
||||
label, pred = self.random_pred_label()
|
||||
label_var = to_variable(label)
|
||||
pred_var = to_variable(pred)
|
||||
state = to_list(acc.add_metric_op(pred_var, label_var))
|
||||
acc.update(* [s.numpy() for s in state])
|
||||
res_m = acc.accumulate()
|
||||
res_f = accuracy(pred, label, self.topk)
|
||||
assert np.all(np.isclose(np.array(res_m, dtype='float64'), np.array(res_f, dtype='float64'), rtol=1e-3)), \
|
||||
"Accuracy precision error: {} != {}".format(res_m, res_f)
|
||||
acc.reset()
|
||||
assert np.sum(acc.total) == 0
|
||||
assert np.sum(acc.count) == 0
|
||||
|
||||
|
||||
class TestAccuracyDynamicMultiTopk(TestAccuracyDynamic):
|
||||
def setUp(self):
|
||||
self.topk = (1, 5)
|
||||
self.class_num = 10
|
||||
self.sample_num = 1000
|
||||
self.name = "accuracy"
|
||||
|
||||
|
||||
class TestAccuracyStatic(TestAccuracyDynamic):
|
||||
def test_main(self):
|
||||
main_prog = fluid.Program()
|
||||
startup_prog = fluid.Program()
|
||||
with fluid.program_guard(main_prog, startup_prog):
|
||||
pred = fluid.data(
|
||||
name='pred', shape=[None, self.class_num], dtype='float32')
|
||||
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
|
||||
acc = Accuracy(topk=self.topk, name=self.name)
|
||||
state = acc.add_metric_op(pred, label)
|
||||
|
||||
exe = fluid.Executor(fluid.CPUPlace())
|
||||
compiled_main_prog = fluid.CompiledProgram(main_prog)
|
||||
|
||||
for _ in range(10):
|
||||
label, pred = self.random_pred_label()
|
||||
state_ret = exe.run(compiled_main_prog,
|
||||
feed={'pred': pred,
|
||||
'label': label},
|
||||
fetch_list=[s.name for s in to_list(state)],
|
||||
return_numpy=True)
|
||||
acc.update(*state_ret)
|
||||
res_m = acc.accumulate()
|
||||
res_f = accuracy(pred, label, self.topk)
|
||||
assert np.all(np.isclose(np.array(res_m, dtype='float64'), np.array(res_f, dtype='float64'), rtol=1e-3)), \
|
||||
"Accuracy precision error: {} != {}".format(res_m, res_f)
|
||||
acc.reset()
|
||||
assert np.sum(acc.total) == 0
|
||||
assert np.sum(acc.count) == 0
|
||||
|
||||
|
||||
class TestAccuracyStaticMultiTopk(TestAccuracyStatic):
|
||||
def setUp(self):
|
||||
self.topk = (1, 5)
|
||||
self.class_num = 10
|
||||
self.sample_num = 1000
|
||||
self.name = "accuracy"
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,6 @@
|
||||
file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
|
||||
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
|
||||
|
||||
foreach(src ${TEST_OPS})
|
||||
py_test(${src} SRCS ${src}.py)
|
||||
endforeach()
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue