diff --git a/mindspore/nn/metrics/__init__.py b/mindspore/nn/metrics/__init__.py index 0abf7de28f..53a6f25ea1 100755 --- a/mindspore/nn/metrics/__init__.py +++ b/mindspore/nn/metrics/__init__.py @@ -36,6 +36,7 @@ from .bleu_score import BleuScore from .cosine_similarity import CosineSimilarity from .occlusion_sensitivity import OcclusionSensitivity from .perplexity import Perplexity +from .confusion_matrix import ConfusionMatrixMetric, ConfusionMatrix __all__ = [ "names", @@ -61,6 +62,8 @@ __all__ = [ "MeanSurfaceDistance", "RootMeanSquareDistance", "Perplexity", + "ConfusionMatrix", + "ConfusionMatrixMetric", ] __factory__ = { @@ -85,6 +88,8 @@ __factory__ = { 'mean_surface_distance': MeanSurfaceDistance, 'root_mean_square_distance': RootMeanSquareDistance, 'perplexity': Perplexity, + 'confusion_matrix': ConfusionMatrix, + 'confusion_matrix_metric': ConfusionMatrixMetric, } diff --git a/mindspore/nn/metrics/confusion_matrix.py b/mindspore/nn/metrics/confusion_matrix.py new file mode 100644 index 0000000000..f3fec795b6 --- /dev/null +++ b/mindspore/nn/metrics/confusion_matrix.py @@ -0,0 +1,685 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ConfusionMatrixMetric & ConfusionMatrix.""" +import numpy as np +from mindspore._checkparam import Validator as validator +from .metric import Metric + + +class ConfusionMatrix(Metric): + r""" + Computes the confusion matrix. The performance matrix of measurement classification model is the model whose output + is binary or multi class. The confusion matrix is calculated. An array of shape [BC4] is returned. + The third dimension represents each channel of each sample in the input batch.Where B is the batch size and C is + the number of classes to be calculated. + + If you only want to find confusion matrix, use this class. If you want to find 'PPV', 'TPR', 'TNR', etc., use class + 'mindspore.metrics.ConfusionMatrixMetric'. + + Args: + num_classes (int): Number of classes in the dataset. + normalize (str): The parameter of calculating ConfusionMatrix supports four Normalization modes, Choose from: + + - **'no_norm'** (None) - No normalization is used. Default: None. + - **'target'** (str) - Normalization based on target value. + - **'prediction'** (str) - Normalization based on predicted value. + - **'all'** (str) - Normalization over the whole matrix. + + threshold (float): A threshold, which is used to compare with the input tensor. Default: 0.5. + + Examples: + >>> x = Tensor(np.array([1, 0, 1, 0])) + >>> y = Tensor(np.array([1, 0, 0, 1])) + >>> metric = nn.ConfusionMatrix(num_classes=2, normalize=NO_NORM, threshold=0.5) + >>> metric.clear() + >>> metric.update(x, y) + >>> output = metric.eval() + >>> print(output) + [[1. 1.] + [1. 1.]] + """ + TARGET = "target" + PREDICTION = "prediction" + ALL = "all" + NO_NORM = "no_norm" + + def __init__(self, num_classes, normalize=NO_NORM, threshold=0.5): + super(ConfusionMatrix, self).__init__() + + self.num_classes = validator.check_value_type("num_classes", num_classes, [int]) + if normalize != ConfusionMatrix.TARGET and normalize != ConfusionMatrix.PREDICTION and \ + normalize != ConfusionMatrix.ALL and normalize is not ConfusionMatrix.NO_NORM: + raise ValueError( + 'The normalize way should be in [all, prediction, label, None], but got {}.'.format(normalize) + ) + + self.normalize = normalize + self.threshold = validator.check_value_type("threshold", threshold, [float]) + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self.confusion_matrix = np.zeros((self.num_classes, self.num_classes)) + self._is_update = False + + def update(self, *inputs): + """ + Update state with y_pred and y. + + Args: + inputs: Input `y_pred` and `y`. `y_pred` and `y` are a `Tensor`, a list or an array. + `y_pred` is the predicted value, `y` is the true value. + The shape of `y_pred` is :math:`(N, C, ...)` or :math:`(N, ...)`. + The shape of `y` is :math:`(N, ...)`. + + Raises: + ValueError: If the number of the inputs is not 2. + """ + if len(inputs) != 2: + raise ValueError('ConfusionMatrix need 2 inputs (y_pred, y), but got {}.'.format(len(inputs))) + + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + + if not (y_pred.ndim == y.ndim or y_pred.ndim == y.ndim + 1): + raise ValueError("y_pred and y should have the same number of dimensions, or the dimension of y_pred " + "equals the dimension of y add 1.") + + if y_pred.ndim == y.ndim + 1: + y_pred = np.argmax(y_pred, axis=1) + + if y_pred.ndim == y.ndim and y_pred.dtype in (np.float16, np.float32, np.float64): + y_pred = (y_pred >= self.threshold).astype(int) + + trans = (y.reshape(-1) * self.num_classes + y_pred.reshape(-1)).astype(int) + bincount = np.bincount(trans, minlength=self.num_classes ** 2) + confusion_matrix = bincount.reshape(self.num_classes, self.num_classes) + self.confusion_matrix += confusion_matrix + self._is_update = True + + def eval(self): + """ + Computes confusion matrix. + + Returns: + numpy.ndarray, the computed result. + """ + + if not self._is_update: + raise RuntimeError('Call the update method before calling eval.') + + confusion_matrix = self.confusion_matrix.astype(float) + + matrix_target = confusion_matrix / confusion_matrix.sum(axis=1, keepdims=True) + matrix_pred = confusion_matrix / confusion_matrix.sum(axis=0, keepdims=True) + matrix_all = confusion_matrix / confusion_matrix.sum() + normalize_dict = {ConfusionMatrix.TARGET: matrix_target, + ConfusionMatrix.PREDICTION: matrix_pred, + ConfusionMatrix.ALL: matrix_all} + + if self.normalize == ConfusionMatrix.NO_NORM: + return confusion_matrix + + matrix = normalize_dict.get(self.normalize) + if matrix[np.isnan(matrix)].size != 0: + matrix[np.isnan(matrix)] = 0 + + return matrix + + +class ConfusionMatrixMetric(Metric): + r""" + The performance matrix of measurement classification model is the model whose output is binary or multi class. + The correlation measure of confusion matrix was calculated from the full-scale tensor, and the average values of + batch, class channel and iteration were collected. This function supports the calculation of all measures described + below: the metric name in parameter metric_name. + + If you want to use confusion matrix to calculate, such as 'PPV', 'TPR', 'TNR', use this class. + If you only want to calculate confusion matrix, please use 'mindspore.metrics.ConfusionMatrix'. + + Args: + skip_channel (bool): Whether to skip the measurement calculation on the first channel of the predicted output. + Default: True. + metric_name (str): The names of indicators are in the following range. Of course, you can also set the industry + common aliases for these indicators. Choose from: + ["sensitivity", "specificity", "precision", "negative predictive value", "miss rate", + "fall out", "false discovery rate", "false omission rate", "prevalence threshold", + "threat score", "accuracy", "balanced accuracy", "f1 score", + "matthews correlation coefficient", "fowlkes mallows index", "informedness", "markedness"]. + calculation_method (bool): If true, the measurement for each sample is calculated first. If it is false, the + confusion matrix of all samples is accumulated first. As for classification task, + 'calculation_method' should be False. Default: False. + decrease (str): Define the mode to reduce the calculation result of one batch of data. Decrease is used only if + calculation_method is True. Default: "mean". Choose from: + ["none", "mean", "sum", "mean_batch", "sum_batch", "mean_channel", "sum_channel"]. + + Examples: + >>> metric = ConfusionMatrixMetric(skip_channel=True, metric_name="tpr", + >>> calculation_method=False, decrease="mean") + >>> metric.clear() + >>> x = Tensor(np.array([[[0], [1]], [[1], [0]]])) + >>> y = Tensor(np.array([[[0], [1]], [[0], [1]]])) + >>> metric.update(x, y) + >>> x = Tensor(np.array([[[0], [1]], [[1], [0]]])) + >>> y = Tensor(np.array([[[0], [1]], [[1], [1]]])) + >>> avg_output = metric.eval() + >>> print(avg_output) + [0.75] + """ + def __init__(self, + skip_channel=True, + metric_name="sensitivity", + calculation_method=False, + decrease="mean"): + super(ConfusionMatrixMetric, self).__init__() + + self.confusion_matrix = _ConfusionMatrix(skip_channel=skip_channel, metric_name=metric_name, + calculation_method=calculation_method, decrease=decrease) + self.skip_channel = validator.check_value_type("skip_channel", skip_channel, [bool]) + self.calculation_method = validator.check_value_type("calculation_method", calculation_method, [bool]) + self.metric_name = validator.check_value_type("metric_name", metric_name, [str]) + decrease_list = ["none", "mean", "sum", "mean_batch", "sum_batch", "mean_channel", "sum_channel"] + decrease = validator.check_value_type("decrease", decrease, [str]) + self.decrease = validator.check_string(decrease, decrease_list, "decrease") + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self._total_num = 0 + self._class_num = 0 + self._total_tp = 0.0 + self._total_fp = 0.0 + self._total_tn = 0.0 + self._total_fn = 0.0 + + def update(self, *inputs): + """ + Update state with predictions and targets. + + inputs: + Input `y_pred` and `y`. `y_pred` and `y` are a `Tensor`, a list or an array. + + - **y_pred** (ndarray) - Input data to compute. It must be one-hot format and first dim is batch. + The shape of `y_pred` is :math:`(N, C, ...)` or :math:`(N, ...)`. + As for classification tasks, `y_pred` should has the shape [BN] where N is larger than 1. + As for segmentation tasks, the shape should be [BNHW] or [BNHWD]. + - **y** (ndarray) - Compute the true value of the measure. It must be one-hot format and first dim is batch. + The shape of `y` is :math:`(N, C, ...)`. + + Raises: + ValueError: If the number of the inputs is not 2. + """ + if len(inputs) != 2: + raise ValueError('ConfusionMatrixMetric need 2 inputs (y_pred, y), but got {}.'.format(len(inputs))) + + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + + if self.calculation_method is True: + score, not_nans = self.confusion_matrix(y_pred, y) + not_nans = int(not_nans.item()) + self._total_num += score.item() * not_nans + self._class_num += not_nans + else: + confusion_matrix = self.confusion_matrix(y_pred, y) + confusion_matrix, _ = _decrease_metric(confusion_matrix, "sum") + self._total_tp += confusion_matrix[0].item() + self._total_fp += confusion_matrix[1].item() + self._total_tn += confusion_matrix[2].item() + self._total_fn += confusion_matrix[3].item() + + def eval(self): + """ + Computes confusion matrix metric. + + Returns: + ndarray, the computed result. + """ + + if self.calculation_method is True: + if self._class_num == 0: + raise RuntimeError("ConfusionMatrixMetric must have at least one example before it can be computed.") + + return self._total_num / self._class_num + + confusion_matrix = np.array([self._total_tp, self._total_fp, self._total_tn, self._total_fn]) + return _compute_confusion_matrix_metric(self.metric_name, confusion_matrix) + + +class _ConfusionMatrix: + """ + Compute confusion matrix related metrics. + + Args: + skip_channel (bool): Whether to skip the measurement calculation on the first channel of the predicted + output. Default: True. + metric_name (str): The names of indicators are in the following range. Of course, you can also set the industry + common aliases for these indicators. + calculation_method (bool): If true, the measurement for each sample is calculated first. If it is false, the + confusion matrix for each image (the output of function '_get_confusion_matrix') + will be returned. In this way, users should achieve the confusion matrixes for all + images during an epochand then use '_compute_confusion_matrix_metric' to calculate + the metric. Default: False. + decrease (Union[DecreaseMetric, str]): ["none", "mean", "sum", "mean_batch", "sum_batch", "mean_channel", + "sum_channel"] + Define the mode to reduce the calculation result of one batch of data. + Decrease is used only if calculation_method is True. Default: "mean". + """ + + def __init__(self, skip_channel=True, metric_name="hit_rate", calculation_method=False, + decrease="mean"): + super().__init__() + self.skip_channel = skip_channel + self.metric_name = metric_name + self.calculation_method = calculation_method + self.decrease = decrease + + def __call__(self, y_pred, y): + """ + 'y_preds' is expected to have binarized predictions and 'y' should be in one-hot format. + + Args: + - **y_pred** (ndarray) - Input data to compute. It must be one-hot format and first dim is batch. + - **y** (ndarray) - Ground truth to compute the metric. It must be one-hot format and first dim is batch. + + Raises: + ValueError: If `metric_name` is empty. + ValueError: when `y_pred` has less than two dimensions. + """ + if not np.all(y.astype(np.uint8) == y): + raise ValueError("y should be a binarized ndarray.") + + dims = y_pred.ndim + if dims < 2: + raise ValueError("y_pred should have at least two dimensions.") + + if dims == 2 or (dims == 3 and y_pred.shape[-1] == 1): + if self.calculation_method: + self.calculation_method = False + + confusion_matrix = _get_confusion_matrix(y_pred=y_pred, y=y, skip_channel=self.skip_channel) + + if self.calculation_method: + if isinstance(self.metric_name, str): + confusion_matrix = _compute_confusion_matrix_metric(self.metric_name, confusion_matrix) + chart, not_nans = _decrease_metric(confusion_matrix, self.decrease) + return chart, not_nans + + if not self.metric_name: + raise ValueError("There should be at least one metric name.") + + results = [] + for metric_name in self.metric_name: + sub_confusion_matrix = _compute_confusion_matrix_metric(metric_name, confusion_matrix) + chart, not_nans = _decrease_metric(sub_confusion_matrix, self.decrease) + results.append(chart) + results.append(not_nans) + return results + + return confusion_matrix + + +def _get_confusion_matrix(y_pred, y, skip_channel=True): + """ + The confusion matrix is calculated. An array of shape [BC4] is returned. The third dimension represents each channel + of each sample in the input batch.Where B is the batch size and C is the number of classes to be calculated. + + Args: + y_pred (ndarray): input data to compute. It must be one-hot format and first dim is batch. + The values should be binarized. + y (ndarray): ground truth to compute the metric. It must be one-hot format and first dim is batch. + The values should be binarized. + skip_channel (bool): whether to skip metric computation on the first channel of the predicted output. + Default: True. + + Raises: + ValueError: when `y_pred` and `y` have different shapes. + """ + + if not skip_channel: + y = y[:, 1:] if y.shape[1] > 1 else y + y_pred = y_pred[:, 1:] if y_pred.shape[1] > 1 else y_pred + + y = y.astype(float) + y_pred = y_pred.astype(float) + validator.check('y_shape', y.shape, 'y_pred_shape', y_pred.shape) + batch_size, n_class = y_pred.shape[:2] + y_pred = y_pred.reshape(batch_size, n_class, -1) + y = y.reshape(batch_size, n_class, -1) + tp = ((y_pred + y) == 2).astype(float) + tn = ((y_pred + y) == 0).astype(float) + tp = tp.sum(axis=2) + tn = tn.sum(axis=2) + p = y.sum(axis=2) + n = y.shape[-1] - p + fn = p - tp + fp = n - tn + + return np.stack([tp, fp, tn, fn], axis=-1) + + +def _decrease_mean(not_nans, chart): + not_nans = not_nans.sum(axis=1) + chart = np.where(not_nans > 0, chart.sum(axis=1) / not_nans, np.zeros(1, dtype=float)) + + not_nans = (not_nans > 0).astype(float).sum(axis=0) + chart = np.where(not_nans > 0, chart.sum(axis=0) / not_nans, np.zeros(1, dtype=float)) + + return not_nans, chart + + +def _decrease_sum(not_nans, chart): + not_nans = not_nans.sum(axis=(0, 1)) + chart = np.sum(chart, axis=(0, 1)) + + return not_nans, chart + + +def _decrease_mean_batch(not_nans, chart): + not_nans = not_nans.sum(axis=0) + chart = np.where(not_nans > 0, chart.sum(axis=0) / not_nans, np.zeros(1, dtype=float)) + + return not_nans, chart + + +def _decrease_sum_batch(not_nans, chart): + not_nans = not_nans.sum(axis=0) + chart = chart.sum(axis=0) + + return not_nans, chart + + +def _decrease_mean_channel(not_nans, chart): + not_nans = not_nans.sum(axis=1) + chart = np.where(not_nans > 0, chart.sum(axis=1) / not_nans, np.zeros(1, dtype=float)) + + return not_nans, chart + + +def _decrease_sum_channel(not_nans, chart): + not_nans = not_nans.sum(axis=1) + chart = chart.sum(axis=1) + + return not_nans, chart + + +def _decrease_none(not_nans, chart): + return not_nans, chart + + +def _decrease_metric(chart, decrease="mean"): + """ + This function is used to reduce the calculated metrics for each class of each example. + + Args: + chart (ndarray): A data table containing the calculated measurement scores for each batch and class. + The first two dims should be batch and class. + decrease (str): Define the mode to reduce computation result of 1 batch data. Decrease will only be employed + when 'calculation_method' is True. Default: "mean". + """ + + nans = np.isnan(chart) + not_nans = (~nans).astype(float) + chart[nans] = 0 + + decrease_dict = {"mean": _decrease_mean(not_nans, chart), + "sum": _decrease_sum(not_nans, chart), + "mean_batch": _decrease_mean_batch, + "sum_batch": _decrease_sum_batch(not_nans, chart), + "mean_channel": _decrease_mean_channel(not_nans, chart), + "sum_channel": _decrease_sum_channel(not_nans, chart), + "none": _decrease_none(not_nans, chart)} + not_nans, chart = decrease_dict.get(decrease) + + return chart, not_nans + + +def _calculate_tpr(tp, p): + """Calculate tpr.""" + return tp, p + + +def _calculate_tnr(tn, n): + """Calculate tnr.""" + return tn, n + + +def _calculate_ppv(tp, fp): + """Calculate ppv.""" + return tp, (tp + fp) + + +def _calculate_npv(tn, fn): + """Calculate npv.""" + return tn, (tn + fn) + + +def _calculate_fnr(fn, p): + """Calculate fnr.""" + return fn, p + + +def _calculate_fpr(fp, n): + """Calculate fpr.""" + return fp, n + + +def _calculate_fdr(tp, fp): + """Calculate fdr.""" + return fp, (fp + tp) + + +def _calculate_for(tn, fn): + """Calculate for.""" + return fn, (fn + tn) + + +def _calculate_pt(tp, tn, p, n): + """Calculate pt.""" + tpr = np.where(p > 0, tp / p, np.array(float("nan"))) + tnr = np.where(n > 0, tn / n, np.array(float("nan"))) + numerator = np.sqrt(tpr * (1.0 - tnr)) + tnr - 1.0 + denominator = tpr + tnr - 1.0 + + return numerator, denominator + + +def _calculate_ts(tp, fp, fn): + """Calculate ts.""" + return tp, (tp + fn + fp) + + +def _calculate_acc(tp, tn, p, n): + """Calculate acc.""" + return (tp + tn), (p + n) + + +def _calculate_ba(tp, tn, p, n): + """Calculate ba.""" + tpr = np.where(p > 0, tp / p, np.array(float("nan"))) + tnr = np.where(n > 0, tn / n, np.array(float("nan"))) + numerator, denominator = (tpr + tnr), 2.0 + + return numerator, denominator + + +def _calculate_f1(tp, fp, fn): + """Calculate f1.""" + return tp * 2.0, (tp * 2.0 + fn + fp) + + +def _calculate_mcc(tp, fp, tn, fn): + """Calculate mcc.""" + numerator = tp * tn - fp * fn + denominator = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + + return numerator, denominator + + +def _calculate_fm(tp, fp, p): + """Calculate fm.""" + tpr = np.where(p > 0, tp / p, np.array(float("nan"))) + ppv = np.where((tp + fp) > 0, tp / (tp + fp), np.array(float("nan"))) + numerator = np.sqrt(ppv * tpr) + denominator = 1.0 + + return numerator, denominator + + +def _calculate_bm(tp, tn, p, n): + """Calculate bm.""" + tpr = np.where(p > 0, tp / p, np.array(float("nan"))) + tnr = np.where(n > 0, tn / n, np.array(float("nan"))) + numerator = tpr + tnr - 1.0 + denominator = 1.0 + + return numerator, denominator + + +def _calculate_mk(tp, fp, tn, fn): + """Calculate mk.""" + ppv = np.where((tp + fp) > 0, tp / (tp + fp), np.array(float("nan"))) + npv = np.where((tn + fn) > 0, tn / (tn + fn), np.array(float("nan"))) + npv = tn / (tn + fn) + numerator = ppv + npv - 1.0 + denominator = 1.0 + + return numerator, denominator + + +def _compute_confusion_matrix_metric(metric_name, confusion_matrix): + """ + This function is used to compute confusion matrix related metric. + + Args: + metric_name (str): Refer to conflusionmatrixmetric 'metric_name'. Some of the metrics have multiple aliases + (as shown in the wikipedia page aforementioned), and you can also input those names instead. + confusion_matrix (ndarray): Refer to '_get_confusion_matrix'. + + Raises: + ValueError: when the size of the last dimension of confusion_matrix is not 4. + NotImplementedError: when specify a not implemented metric_name. + + """ + + metric = _check_metric_name(metric_name) + + input_dim = confusion_matrix.ndim + if input_dim == 1: + confusion_matrix = np.expand_dims(confusion_matrix, 0) + if confusion_matrix.shape[-1] != 4: + raise ValueError("The size of the last dimension of confusion_matrix should be 4.") + + tp = confusion_matrix[..., 0] + fp = confusion_matrix[..., 1] + tn = confusion_matrix[..., 2] + fn = confusion_matrix[..., 3] + p = tp + fn + n = fp + tn + + metric_name_dict = {"tpr": _calculate_tpr(tp, p), + "tnr": _calculate_tnr(tn, n), + "ppv": _calculate_ppv(tp, fp), + "npv": _calculate_npv(tn, fn), + "fnr": _calculate_fnr(fn, p), + "fpr": _calculate_fpr(fp, n), + "fdr": _calculate_fdr(tp, fp), + "for": _calculate_for(tn, fn), + "pt": _calculate_pt(tp, tn, p, n), + "ts": _calculate_ts(tp, fp, fn), + "acc": _calculate_acc(tp, tn, p, n), + "ba": _calculate_ba(tp, tn, p, n), + "f1": _calculate_f1(tp, fp, fn), + "mcc": _calculate_mcc(tp, fp, tn, fn), + "fm": _calculate_fm(tp, fp, p), + "bm": _calculate_bm(tp, tn, p, n), + "mk": _calculate_mk(tp, fp, tn, fn) + } + numerator, denominator = metric_name_dict.get(metric) + + if isinstance(denominator, np.ndarray): + result = np.where(denominator != 0, numerator / denominator, np.array(float("nan"))) + else: + result = numerator / denominator + return result + + +def _check_metric_name(metric_name): + """ + There are many metrics related to confusion matrix, and some of the metrics have more than one names. In addition, + some of the names are very long. Therefore, this function is used to check and simplify the name. + + Returns: + Simplified metric name. + + Raises: + NotImplementedError: when the metric is not implemented. + """ + metric_name = metric_name.replace(" ", "_") + metric_name = metric_name.lower() + metric_name_dict = {"sensitivity": "tpr", + "recall": "tpr", + "hit_rate": "tpr", + "true_positive_rate": "tpr", + "tpr": "tpr", + "specificity": "tnr", + "selectivity": "tnr", + "true_negative_rate": "tnr", + "tnr": "tnr", + "precision": "ppv", + "positive_predictive_value": "ppv", + "ppv": "ppv", + "negative_predictive_value": "npv", + "npv": "npv", + "miss_rate": "fnr", + "false_negative_rate": "fnr", + "fnr": "fnr", + "fall_out": "fpr", + "false_positive_rate": "fpr", + "fpr": "fpr", + "false_discovery_rate": "fdr", + "fdr": "fdr", + "false_omission_rate": "for", + "for": "for", + "prevalence_threshold": "pt", + "pt": "pt", + "threat_score": "ts", + "critical_success_index": "ts", + "ts": "ts", + "csi": "ts", + "accuracy": "acc", + "acc": "acc", + "balanced_accuracy": "ba", + "ba": "ba", + "f1_score": "f1", + "f1": "f1", + "matthews_correlation_coefficient": "mcc", + "mcc": "mcc", + "fowlkes_mallows_index": "fm", + "fm": "fm", + "informedness": "bm", + "bookmaker_informedness": "bm", + "bm": "bm", + "markedness": "mk", + "deltap": "mk", + "mk": "mk" + } + + metric_name_info = metric_name_dict.get(metric_name) + + if metric_name_info is None: + raise NotImplementedError("The metric is not implemented.") + + return metric_name_info diff --git a/mindspore/nn/metrics/dice.py b/mindspore/nn/metrics/dice.py index 56524c7d0f..ad5e8f26af 100644 --- a/mindspore/nn/metrics/dice.py +++ b/mindspore/nn/metrics/dice.py @@ -21,19 +21,19 @@ from .metric import Metric class Dice(Metric): r""" The Dice coefficient is a set similarity metric. It is used to calculate the similarity between two samples. The - value of the Dice coefficient is 1 when the segmentation result is the best and 0 when the segmentation result - is the worst. The Dice coefficient indicates the ratio of the area between two objects to the total area. - The function is shown as follows: + value of the Dice coefficient is 1 when the segmentation result is the best and 0 when the segmentation result + is the worst. The Dice coefficient indicates the ratio of the area between two objects to the total area. + The function is shown as follows: - .. math:: + .. math:: dice = \frac{2 * (pred \bigcap true)}{pred \bigcup true} - Args: - smooth (float): A term added to the denominator to improve numerical stability. Should be greater than 0. - Default: 1e-5. - threshold (float): A threshold, which is used to compare with the input tensor. Default: 0.5. + Args: + smooth (float): A term added to the denominator to improve numerical stability. Should be greater than 0. + Default: 1e-5. + threshold (float): A threshold, which is used to compare with the input tensor. Default: 0.5. - Examples: + Examples: >>> x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) >>> y = Tensor(np.array([[0, 1], [1, 0], [0, 1]])) >>> metric = Dice(smooth=1e-5, threshold=0.5) diff --git a/tests/ut/python/metrics/test_confusion_matrix.py b/tests/ut/python/metrics/test_confusion_matrix.py new file mode 100644 index 0000000000..13d9a23198 --- /dev/null +++ b/tests/ut/python/metrics/test_confusion_matrix.py @@ -0,0 +1,73 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# """test_confusion_matrix""" +import numpy as np +import pytest +from mindspore import Tensor +from mindspore.nn.metrics import ConfusionMatrix + + +def test_confusion_matrix(): + """test_confusion_matrix""" + x = Tensor(np.array([1, 0, 1, 0])) + y = Tensor(np.array([1, 0, 0, 1])) + metric = ConfusionMatrix(num_classes=2) + metric.clear() + metric.update(x, y) + output = metric.eval() + + assert np.allclose(output, np.array([[1, 1], [1, 1]])) + + +def test_confusion_matrix_update_len(): + x = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + metric = ConfusionMatrix(num_classes=2) + metric.clear() + + with pytest.raises(ValueError): + metric.update(x) + + +def test_confusion_matrix_update_dim(): + x = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + y = Tensor(np.array([1, 0])) + metric = ConfusionMatrix(num_classes=2) + metric.clear() + + with pytest.raises(ValueError): + metric.update(x, y) + + +def test_confusion_matrix_init_num_classes(): + with pytest.raises(TypeError): + ConfusionMatrix(num_classes='1') + + +def test_confusion_matrix_init_normalize_value(): + with pytest.raises(ValueError): + ConfusionMatrix(num_classes=2, normalize="wwe") + + +def test_confusion_matrix_init_threshold(): + with pytest.raises(TypeError): + ConfusionMatrix(num_classes=2, normalize='no_norm', threshold=1) + + +def test_confusion_matrix_runtime(): + metric = ConfusionMatrix(num_classes=2) + metric.clear() + + with pytest.raises(RuntimeError): + metric.eval() diff --git a/tests/ut/python/metrics/test_confusion_matrix_metric.py b/tests/ut/python/metrics/test_confusion_matrix_metric.py new file mode 100644 index 0000000000..422ee7f80c --- /dev/null +++ b/tests/ut/python/metrics/test_confusion_matrix_metric.py @@ -0,0 +1,94 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# """test_confusion_matrix_metric""" +import numpy as np +import pytest +from mindspore import Tensor +from mindspore.nn.metrics import ConfusionMatrixMetric + + +def test_confusion_matrix_metric(): + """test_confusion_matrix_metric""" + metric = ConfusionMatrixMetric(skip_channel=True, metric_name="tpr", calculation_method=False) + metric.clear() + x = Tensor(np.array([[[0], [1]], [[1], [0]]])) + y = Tensor(np.array([[[0], [1]], [[0], [1]]])) + metric.update(x, y) + + x = Tensor(np.array([[[0], [1]], [[1], [0]]])) + y = Tensor(np.array([[[0], [1]], [[1], [0]]])) + metric.update(x, y) + output = metric.eval() + + assert np.allclose(output, np.array([0.75])) + + +def test_confusion_matrix_metric_update_len(): + x = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + metric = ConfusionMatrixMetric(skip_channel=True, metric_name="ppv", calculation_method=True) + metric.clear() + + with pytest.raises(ValueError): + metric.update(x) + + +def test_confusion_matrix_metric_update_dim(): + x = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + y = Tensor(np.array([1, 0])) + metric = ConfusionMatrixMetric(skip_channel=True, metric_name="tnr", calculation_method=True) + metric.clear() + + with pytest.raises(ValueError): + metric.update(y, x) + + +def test_confusion_matrix_metric_init_skip_channel(): + with pytest.raises(TypeError): + ConfusionMatrixMetric(skip_channel=1) + + +def test_confusion_matrix_metric_init_compute_sample(): + with pytest.raises(TypeError): + ConfusionMatrixMetric(calculation_method=1) + + +def test_confusion_matrix_metric_init_metric_name_type(): + with pytest.raises(TypeError): + metric = ConfusionMatrixMetric(skip_channel=True, metric_name=1, calculation_method=False) + x = Tensor(np.array([[[0], [1]], [[1], [0]]])) + y = Tensor(np.array([[[0], [1]], [[1], [0]]])) + metric.update(x, y) + output = metric.eval() + + assert np.allclose(output, np.array([0.75])) + + +def test_confusion_matrix_metric_init_metric_name_str(): + with pytest.raises(NotImplementedError): + metric = ConfusionMatrixMetric(skip_channel=True, metric_name="wwwww", calculation_method=False) + x = Tensor(np.array([[[0], [1]], [[1], [0]]])) + y = Tensor(np.array([[[0], [1]], [[1], [0]]])) + metric.update(x, y) + output = metric.eval() + + assert np.allclose(output, np.array([0.75])) + + +def test_confusion_matrix_metric_runtime(): + metric = ConfusionMatrixMetric(skip_channel=True, metric_name="tnr", calculation_method=True) + metric.clear() + + with pytest.raises(RuntimeError): + metric.eval()