Add explainer to provide eXplainable AI tools.

This commit provides APIs for user to use the widely used attribution methods to explain DL models and the evaluation methods to quantify the explanations. With combination of MindInsight, the user can have a friendly visualization on their models.

mindspore/ccsrc/utils/summary.proto

@@ -118,19 +118,11 @@ message Explain {
 	}
 
     message Benchmark{
-        message TotalScore{
+        optional string benchmark_method = 1;
-            optional string benchmark_method = 1;
+        optional string explain_method = 2;
-            optional float score = 2;
+        optional float total_score = 3;
-		}
+        repeated float label_score = 4;
-        message LabelScore{
+    }
-            repeated float score = 1;
-            optional string benchmark_method = 2;
-		}
-
-        optional string explain_method = 1;
-        repeated TotalScore total_score = 2;
-        repeated LabelScore label_score = 3;
-	}
 
     message Metadata{
 		repeated string label = 1;

mindspore/explainer/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Provide ExplainRunner High-level API."""
+
+from ._runner import ExplainRunner
+
+__all__ = ['ExplainRunner']

mindspore/explainer/benchmark/__init__.py

@@ -0,0 +1,23 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Predefined XAI metrics."""
+
+from ._attribution.faithfulness import Faithfulness
+from ._attribution.localization import Localization
+
+__all__ = [
+    "Faithfulness",
+    "Localization"
+]

mindspore/explainer/benchmark/_attribution/__init__.py

@@ -0,0 +1,23 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Predefined XAI metrics"""
+
+from .faithfulness import Faithfulness
+from .localization import Localization
+
+__all__ = [
+    "Faithfulness",
+    "Localization"
+]

mindspore/explainer/benchmark/_attribution/localization.py

@@ -0,0 +1,146 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Localization metrics."""
+import numpy as np
+
+from mindspore.train._utils import check_value_type
+from .metric import AttributionMetric
+from ..._operators import maximum, reshape, Tensor
+from ..._utils import format_tensor_to_ndarray
+
+
+def _get_max_position(saliency):
+    """Get the position of the max pixel of the saliency map."""
+    saliency = saliency.asnumpy()
+    w = saliency.shape[3]
+    saliency = np.reshape(saliency, (len(saliency), -1))
+    max_arg = np.argmax(saliency, axis=1)
+    return max_arg // w, max_arg - (max_arg // w) * w
+
+
+def _mask_out_saliency(saliency, threshold):
+    """Keep the saliency map with value greater than threshold."""
+    max_value = maximum(saliency)
+    mask_out = saliency > (reshape(max_value, (len(saliency), -1, 1, 1)) * threshold)
+    return mask_out
+
+
+class Localization(AttributionMetric):
+    """
+    Provides evaluation on the localization capability of XAI methods.
+
+    We support two metrics for the evaluation os localization capability: "PointingGame" and "IoSR".
+    For metric "PointingGame", the localization capability is calculated as the ratio of data in which the max position
+    of their saliency maps lies within the bounding boxes. Specifically, for a single datum, given the saliency map and
+    its bounding box, if the max point of its saliency map lies within the bounding box, the evaluation result is 1
+    otherwise 0.
+
+    For metric "IoSR" (Intersection over Salient Region), the localization capability is calculated as the intersection
+    of the bounding box and the salient region over the area of the salient region.
+
+    Args:
+        num_labels (int): number of classes in the dataset.
+        metric (str): specific metric to calculate localization capability.
+            Options: "PointingGame", "IoSR".
+            Default: "PointingGame".
+
+    Examples:
+        >>> from mindspore.explainer.benchmark import Localization
+        >>> num_labels = 100
+        >>> localization = Localization(num_labels, "PointingGame")
+    """
+
+    def __init__(self,
+                 num_labels,
+                 metric="PointingGame"
+                 ):
+        super(Localization, self).__init__(num_labels)
+        self._verify_metrics(metric)
+        self._metric = metric
+
+        # Arg for specific metric, for "PointingGame" it should be an integer indicating the tolerance
+        # of "PointingGame", while for "IoSR" it should be a float number
+        # indicating the threshold to choose salient region. Default: 25.
+        if self._metric == "PointingGame":
+            self._metric_arg = 15
+        else:
+            self._metric_arg = 0.5
+
+    @staticmethod
+    def _verify_metrics(metric):
+        """Verify the user defined metric."""
+        supports = ["PointingGame", "IoSR"]
+        if metric not in supports:
+            raise ValueError("Metric should be one of {}".format(supports))
+
+    def evaluate(self, explainer, inputs, targets, saliency=None, mask=None):
+        """
+        Evaluate localization on a single data sample.
+
+        Args:
+            explainer (Explanation): The explainer to be evaluated, see `mindspore/explainer/explanation`.
+            inputs (Tensor): data sample. Currently only support single sample at each call.
+            targets (int): target label to evaluate on.
+            saliency (Tensor): A saliency tensor.
+            mask (Union[Tensor, np.ndarray]): ground truth bounding box/masks for the inputs w.r.t targets.
+
+        Returns:
+            np.ndarray, result of localization evaluated on explainer
+
+        Examples:
+            >>> # init an explainer, the network should contain the output activation function.
+            >>> gradient = Gradient(network)
+            >>> inputs = ms.Tensor(np.random.rand(1, 3, 224, 224), ms.float32)
+            >>> masks = np.zeros(1, 1, 224, 224)
+            >>> masks[:, :, 65: 100, 65: 100] = 1
+            >>> targets = 5
+            >>> # usage 1: input the explainer and the data to be explained,
+            >>> # calculate the faithfulness with the specified metric
+            >>> res = localization.evaluate(gradient, inputs, targets, mask=masks)
+            >>> # usage 2: input the generated saliency map
+            >>> saliency = gradient(inputs, targets)
+            >>> res = localization.evaluate(gradient, inputs, targets, saliency, mask=masks)
+        """
+        self._check_evaluate_param(explainer, inputs, targets, saliency)
+
+        mask_np = format_tensor_to_ndarray(mask)[0]
+
+        if saliency is None:
+            saliency = explainer(inputs, targets)
+
+        if self._metric == "PointingGame":
+            point = _get_max_position(saliency)
+
+            x, y = np.meshgrid(
+                (np.arange(mask_np.shape[1]) - point[0]) ** 2,
+                (np.arange(mask_np.shape[2]) - point[1]) ** 2)
+            max_region = (x + y) < self._metric_arg ** 2
+
+            # if max_region has overlap with mask_np return 1 otherwise 0.
+            result = 1 if (mask_np.astype(bool) & max_region).any() else 0
+
+        elif self._metric == "IoSR":
+            mask_out = _mask_out_saliency(saliency, self._metric_arg)
+            mask_out_np = format_tensor_to_ndarray(mask_out)
+            overlap = np.sum(mask_np.astype(bool) & mask_out_np.astype(bool))
+            saliency_area = np.sum(mask_out_np)
+            result = overlap / saliency_area.clip(min=1e-10)
+        return np.array([result], np.float)
+
+    def _check_evaluate_param_with_mask(self, explainer, inputs, targets, saliency, mask):
+        self._check_evaluate_param(explainer, inputs, targets, saliency)
+        check_value_type('mask', mask, (Tensor, np.ndarray))
+        if len(inputs.shape) != 4:
+            raise ValueError('Argument mask must be 4D Tensor')

mindspore/explainer/benchmark/_attribution/metric.py

@@ -0,0 +1,123 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Base class for XAI metrics."""
+import numpy as np
+
+from mindspore.train._utils import check_value_type
+from ..._operators import Tensor
+from ..._utils import format_tensor_to_ndarray
+from ...explanation._attribution._attribution import Attribution
+
+
+def verify_argument(inputs, arg_name):
+    """Verify the validity of the parsed arguments."""
+    check_value_type(arg_name, inputs, Tensor)
+    if len(inputs.shape) != 4:
+        raise ValueError('Argument {} must be a 4D Tensor.'.format(arg_name))
+    if len(inputs) > 1:
+        raise ValueError('Support single data evaluation only, but got {}.'.format(len(inputs)))
+
+
+def verify_targets(targets, num_labels):
+    """Verify the validity of the parsed targets."""
+    check_value_type('targets', targets, (int, Tensor))
+
+    if isinstance(targets, Tensor):
+        if len(targets.shape) > 1 or (len(targets.shape) == 1 and len(targets) != 1):
+            raise ValueError('Argument targets must be a 1D or 0D Tensor. If it is a 1D Tensor, '
+                             'it should have the length = 1 as we only support single evaluation now.')
+        targets = int(targets.asnumpy()[0]) if len(targets.shape) == 1 else int(targets.asnumpy())
+    if targets > num_labels - 1 or targets < 0:
+        raise ValueError('Parsed targets exceed the label range.')
+
+
+class AttributionMetric:
+    """Super class of XAI metric class used in classification scenarios."""
+
+    def __init__(self, num_labels=None):
+        self._num_labels = num_labels
+        self._global_results = {i: [] for i in range(num_labels)}
+
+    def evaluate(self, explainer, inputs, targets, saliency=None):
+        """This function evaluates on a single sample and return the result."""
+        raise NotImplementedError
+
+    def aggregate(self, result, targets):
+        """Aggregates single result to global_results."""
+        if isinstance(result, float):
+            if isinstance(targets, int):
+                self._global_results[targets].append(result)
+            else:
+                target_np = format_tensor_to_ndarray(targets)
+                if len(target_np) > 1:
+                    raise ValueError("One result can not be aggreated to multiple targets.")
+        else:
+            result_np = format_tensor_to_ndarray(result)
+            if isinstance(targets, int):
+                for res in result_np:
+                    self._global_results[targets].append(float(res))
+            else:
+                target_np = format_tensor_to_ndarray(targets)
+                if len(target_np) != len(result_np):
+                    raise ValueError("Length of result does not match with length of targets.")
+                for tar, res in zip(target_np, result_np):
+                    self._global_results[int(tar)].append(float(res))
+
+    def reset(self):
+        """Resets global_result."""
+        self._global_results = {i: [] for i in range(self._num_labels)}
+
+    @property
+    def class_performances(self):
+        """
+        Get the class performances by global result.
+
+
+        Returns:
+            (:class:`np.ndarray`): :attr:`num_labels`-dimensional vector
+                containing per-class performance.
+        """
+        count = np.array(
+            [len(self._global_results[i]) for i in range(self._num_labels)])
+        result_sum = np.array(
+            [sum(self._global_results[i]) for i in range(self._num_labels)])
+        return result_sum / count.clip(min=1)
+
+    @property
+    def performance(self):
+        """
+        Get the performance by global result.
+
+        Returns:
+            (:class:`float`): mean performance.
+        """
+        count = sum(
+            [len(self._global_results[i]) for i in range(self._num_labels)])
+        result_sum = sum(
+            [sum(self._global_results[i]) for i in range(self._num_labels)])
+        if count == 0:
+            return 0
+        return result_sum / count
+
+    def get_results(self):
+        """Global result of the metric can be return"""
+        return self._global_results
+
+    def _check_evaluate_param(self, explainer, inputs, targets, saliency):
+        """Check the evaluate parameters."""
+        check_value_type('explainer', explainer, Attribution)
+        verify_argument(inputs, 'inputs')
+        verify_targets(targets, self._num_labels)
+        check_value_type('saliency', saliency, (Tensor, type(None)))

mindspore/explainer/explanation/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Predefined Attribution explainers."""
+
+from ._attribution._backprop.gradcam import GradCAM
+from ._attribution._backprop.gradient import Gradient
+from ._attribution._backprop.modified_relu import Deconvolution, GuidedBackprop
+
+__all__ = [
+    'Gradient',
+    'Deconvolution',
+    'GuidedBackprop',
+    'GradCAM',
+]

mindspore/explainer/explanation/_attribution/__init__.py

@@ -0,0 +1,25 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Predefined Attribution explainers."""
+from ._backprop.gradcam import GradCAM
+from ._backprop.gradient import Gradient
+from ._backprop.modified_relu import Deconvolution, GuidedBackprop
+
+__all__ = [
+    'Gradient',
+    'Deconvolution',
+    'GuidedBackprop',
+    'GradCAM',
+]

mindspore/explainer/explanation/_attribution/_attribution.py

@@ -0,0 +1,60 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Attribution."""
+
+from typing import Callable
+
+import mindspore as ms
+
+class Attribution:
+    r"""
+    Basic class of attributing the salient score
+
+    The explainers which explanation through attributing the relevance scores
+    should inherit this class.
+
+    Args:
+        network (ms.nn.Cell): The black-box model to explanation.
+    """
+
+    def __init__(self, network):
+        self._verify_model(network)
+        self._model = network
+
+    @staticmethod
+    def _verify_model(model):
+        """
+        Verify the input `network` for __init__ function.
+        """
+        if not isinstance(model, ms.nn.Cell):
+            raise TypeError("The parsed `network` must be a `mindspore.nn.Cell` object.")
+
+
+    __call__: Callable
+    """
+    The explainers return the explanations by calling directly on the explanation.
+    Derived class should overwrite this implementations for different
+    algorithms.
+
+    Args:
+        input (ms.Tensor): Input tensor to be explained.
+
+    Returns:
+        - saliency map (ms.Tensor): saliency map of the input.
+    """
+
+    @property
+    def model(self):
+        return self._model

mindspore/explainer/explanation/_attribution/_backprop/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Backprop-base _attribution explainer."""
+
+from .gradient import Gradient
+from .gradcam import GradCAM
+from .modified_relu import Deconvolution, GuidedBackprop
+
+__all__ = ['Gradient',
+           'GradCAM',
+           'Deconvolution',
+           'GuidedBackprop']

mindspore/explainer/explanation/_attribution/_backprop/backprop_utils.py

@@ -0,0 +1,49 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Providing utility functions."""
+
+from mindspore.ops.composite import GradOperation
+
+from ...._utils import unify_inputs, unify_targets, generate_one_hot
+
+
+def compute_gradients(model, inputs, targets=None, weights=None):
+    r"""
+    Compute the gradient of output w.r.t input.
+
+    Args:
+        model (`ms.nn.Cell`): Differentiable black-box model.
+        inputs (`ms.Tensor`): Input to calculate gradient and explanation.
+        targets (int, optional): Target label id specifying which category to compute gradient. Default: None.
+        weights (`ms.Tensor`, optional): Custom weights for computing gradients. The shape of weights should match the
+            model outputs. If None is provided, an one-hot weights with one in targets positions will be used instead.
+            Default: None.
+
+    Returns:
+        saliency map (ms.Tensor): Gradient back-propagated to the input.
+    """
+    inputs = unify_inputs(inputs)
+    if targets is None and weights is None:
+        raise ValueError('Must provide one of targets or weights')
+    if weights is None:
+        targets = unify_targets(targets)
+        output = model(*inputs).asnumpy()
+        num_categories = output.shape[-1]
+        weights = generate_one_hot(targets, num_categories)
+
+    grad_op = GradOperation(
+        get_all=True, get_by_list=False, sens_param=True)(model)
+    gradients = grad_op(*inputs, weights)
+    return gradients[0]

mindspore/explainer/explanation/_attribution/_backprop/gradcam.py

@@ -0,0 +1,141 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+""" GradCAM and GuidedGradCAM. """
+
+from mindspore.ops import operations as op
+
+from .backprop_utils import compute_gradients
+from .intermediate_layer import IntermediateLayerAttribution
+from ...._utils import ForwardProbe, retrieve_layer, unify_inputs, unify_targets
+
+
+
+def _gradcam_aggregation(attributions):
+    """
+    Aggregate the gradient and activation to get the final _attribution.
+
+    Args:
+        attributions (Tensor): the _attribution with channel dimension.
+
+    Returns:
+        Tensor: the _attribution with channel dimension aggregated.
+    """
+    sum_ = op.ReduceSum(keep_dims=True)
+    relu_ = op.ReLU()
+    attributions = relu_(sum_(attributions, 1))
+    return attributions
+
+
+class GradCAM(IntermediateLayerAttribution):
+    r"""
+    Provides GradCAM explanation method.
+
+    GradCAM generates saliency map at intermediate layer.
+    ..math:
+        \alpha_k^c = 1/Z \sum_i \sum_j \div{\partial{y^c}}{\partial{A_{i,j}^k}}
+        L_{GradCAM} = ReLu(\sum_k \alpha_k^c A^k)
+    For more details, please refer to the original paper: GradCAM
+    [https://openaccess.thecvf.com/content_ICCV_2017/papers/Selvaraju_Grad-CAM_Visual_Explanations_ICCV_2017_paper.pdf]
+
+    Args:
+        network (Cell): The black-box model to be explained.
+        layer (str): The layer name to generate the explanation at. Default: ''.
+            If default, the explantion will be generated at the input layer.
+
+    Examples:
+        >>> net = resnet50(10)
+        >>> param_dict = load_checkpoint("resnet50.ckpt")
+        >>> load_param_into_net(net, param_dict)
+        >>> # bind net with its output activation if you wish, e.g. nn.Sigmoid(),
+        >>> # you may also use the net itself.
+        >>> net = nn.SequentialCell([net, nn.Sigmoid()])
+        >>> # specify a layer name to generate explanation, usually the layer can be set as the last conv layer.
+        >>> layer_name = '0.layer4'
+        >>> # init GradCAM with a trained network and specify the layer to obtain
+        >>> gradcam = GradCAM(net, layer=layer_name)
+        >>> # parse data and the target label to be explained and get the saliency map
+        >>> inputs = ms.Tensor(np.random.rand([1, 3, 224, 224]), ms.float32)
+        >>> label = 5
+        >>> saliency = gradcam(inputs, label)
+    """
+
+    def __init__(
+            self,
+            network,
+            layer=""):
+        super(GradCAM, self).__init__(network, layer)
+
+        self._saliency_cell = retrieve_layer(self._backward_model, target_layer=layer)
+        self._avgpool = op.ReduceMean(keep_dims=True)
+        self._intermediate_grad = None
+        self._aggregation_fn = _gradcam_aggregation
+        self._resize_mode = 'bilinear'
+
+    def _hook_cell(self):
+        if self._saliency_cell:
+            self._saliency_cell.register_backward_hook(self._cell_hook_fn)
+            self._saliency_cell.enable_hook = True
+        self._intermediate_grad = None
+
+    def _cell_hook_fn(self, _, grad_input, grad_output):
+        """
+        Hook function to deal with the backward gradient.
+
+        The arguments are set as required by Cell.register_back_hook
+        """
+        self._intermediate_grad = grad_input
+
+    def __call__(self, inputs, targets):
+        """
+        Call function for `GradCAM`.
+
+        Args:
+            inputs (Tensor): The input data to be explained, 4D Tensor.
+            targets (Union[Tensor, int]): The label of interest. It should be a 1D or 0D Tensor, or an integer.
+                If `targets` is a 1D Tensor, its length should be the same as `inputs`.
+        """
+        self._verify_data(inputs, targets)
+        self._hook_cell()
+
+        with ForwardProbe(self._saliency_cell) as probe:
+
+            inputs = unify_inputs(inputs)
+            targets = unify_targets(targets)
+
+            gradients = compute_gradients(self._backward_model, *inputs, targets)
+
+            # get intermediate activation
+            activation = (probe.value,)
+
+            if self._layer == "":
+                activation = inputs
+                self._intermediate_grad = unify_inputs(gradients)
+            if self._intermediate_grad is not None:
+                # average pooling on gradients
+                intermediate_grad = unify_inputs(
+                    self._avgpool(self._intermediate_grad[0], (2, 3)))
+            else:
+                raise ValueError("Gradient for intermediate layer is not "
+                                 "obtained")
+            mul = op.Mul()
+            attribution = self._aggregation_fn(
+                mul(*intermediate_grad, *activation))
+            if self._resize:
+                attribution = self._resize_fn(attribution, *inputs,
+                                              mode=self._resize_mode)
+            self._intermediate_grad = None
+
+        return attribution

mindspore/explainer/explanation/_attribution/_backprop/gradient.py

@@ -0,0 +1,129 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Gradient explainer."""
+from copy import deepcopy
+
+from mindspore import nn
+from mindspore.ops import operations as op
+from mindspore.train._utils import check_value_type
+from ...._operators import reshape, sqrt, Tensor
+from .._attribution import Attribution
+from .backprop_utils import compute_gradients
+from ...._utils import unify_inputs, unify_targets
+
+
+def _get_hook(bntype, cache):
+    """Provide backward hook function for BatchNorm layer in eval mode."""
+    var, gamma, eps = cache
+    if bntype == "2d":
+        var = reshape(var, (1, -1, 1, 1))
+        gamma = reshape(gamma, (1, -1, 1, 1))
+    elif bntype == "1d":
+        var = reshape(var, (1, -1, 1))
+        gamma = reshape(gamma, (1, -1, 1))
+
+    def reset_gradient(_, grad_input, grad_output):
+        grad_output = grad_input[0] * gamma / sqrt(var + eps)
+        return grad_output
+
+    return reset_gradient
+
+
+def _abs_max(gradients):
+    """
+    Transform gradients to saliency through abs then take max along
+    channels.
+    """
+    gradients = op.Abs()(gradients)
+    saliency = op.ReduceMax(keep_dims=True)(gradients, axis=1)
+    return saliency
+
+
+class Gradient(Attribution):
+    r"""
+    Provides Gradient explanation method.
+
+    Gradient is the simplest attribution method which uses the naive gradients of outputs w.r.t inputs as the
+    explanation.
+
+    .. math::
+        _attribution = \div{\delta{y}, \delta{x}}
+
+    Args:
+        network (Cell): The black-box model to be explained.
+
+    Examples:
+        >>> net = resnet50(10)
+        >>> param_dict = load_checkpoint("resnet50.ckpt")
+        >>> load_param_into_net(net, param_dict)
+        >>> # bind net with its output activation if you wish, e.g. nn.Sigmoid(),
+        >>> # you may also use the net itself. The saliency map might be slightly different for softmax activation.
+        >>> net = nn.SequentialCell([net, nn.Sigmoid()])
+        >>> # init Gradient with a trained network.
+        >>> gradient = Gradient(net)
+        >>> # parse data and the target label to be explained and get the saliency map
+        >>> inputs = ms.Tensor(np.random.rand([1, 3, 224, 224]), ms.float32)
+        >>> label = 5
+        >>> saliency = gradient(inputs, label)
+    """
+
+    def __init__(self, network):
+        super(Gradient, self).__init__(network)
+        self._backward_model = deepcopy(network)
+        self._backward_model.set_train(False)
+        self._backward_model.set_grad(False)
+        self._hook_bn()
+        self._grad_op = compute_gradients
+        self._aggregation_fn = _abs_max
+
+
+    def __call__(self, inputs, targets):
+        """
+        Call function for `Gradient`.
+
+        Args:
+            inputs (Tensor): The input data to be explained, 4D Tensor.
+            targets (Union[Tensor, int]): The label of interest. It should be a 1D or 0D Tensor, or an integer.
+                If `targets` is a 1D `Tensor`, its length should be the same as `inputs`.
+        """
+        self._verify_data(inputs, targets)
+        inputs = unify_inputs(inputs)
+        targets = unify_targets(targets)
+
+        gradient = self._grad_op(self._backward_model, *inputs, targets)
+        saliency = self._aggregation_fn(gradient)
+        return saliency
+
+    def _hook_bn(self):
+        """Hook BatchNorm layer for `self._backward_model.`"""
+        for _, cell in self._backward_model.cells_and_names():
+            if isinstance(cell, nn.BatchNorm2d):
+                cache = (cell.moving_variance, cell.gamma, cell.eps)
+                cell.register_backward_hook(_get_hook("2d", cache=cache))
+            elif isinstance(cell, nn.BatchNorm1d):
+                cache = (cell.moving_variance, cell.gamma, cell.eps)
+                cell.register_backward_hook(_get_hook("1d", cache=cache))
+
+    @staticmethod
+    def _verify_data(inputs, targets):
+        """Verify the validity of the parsed inputs."""
+        check_value_type('inputs', inputs, Tensor)
+        if len(inputs.shape) != 4:
+            raise ValueError('Argument inputs must be 4D Tensor')
+        check_value_type('targets', targets, (Tensor, int))
+        if isinstance(targets, Tensor):
+            if len(targets.shape) > 1 or (len(targets.shape) == 1 and len(targets) != len(inputs)):
+                raise ValueError('Argument targets must be a 1D or 0D Tensor. If it is a 1D Tensor, '
+                                 'it should have the same length as inputs.')

mindspore/explainer/explanation/_attribution/_backprop/intermediate_layer.py

@@ -0,0 +1,47 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Base class IntermediateLayerAttribution"""
+
+from .gradient import Gradient
+from ...._utils import resize as resize_fn
+
+
+class IntermediateLayerAttribution(Gradient):
+    """
+    Base class for generating _attribution map at intermediate layer.
+
+    Args:
+        network (nn.Cell): DNN model to be explained.
+        layer (str, optional): string that specifies the layer to generate
+            intermediate _attribution. When using default value, the input layer
+            will be specified. Default: ''.
+    """
+
+    def __init__(self, network, layer=''):
+        super(IntermediateLayerAttribution, self).__init__(network)
+
+        # Whether resize the _attribution layer to the input size.
+        self._resize = True
+        # string that specifies the resize mode. Default: 'nearest_neighbor'.
+        self._resize_mode = 'nearest_neighbor'
+
+        self._layer = layer
+
+    @staticmethod
+    def _resize_fn(attributions, inputs, mode):
+        """Resize the intermediate layer _attribution to the same size as inputs."""
+        height, width = inputs.shape[2], inputs.shape[3]
+        return resize_fn(attributions, (height, width), mode)

mindspore/explainer/explanation/_attribution/_backprop/modified_relu.py

@@ -0,0 +1,117 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Explainer with modified ReLU."""
+
+import mindspore.nn as nn
+import mindspore.ops.operations as op
+
+from .gradient import Gradient
+from ...._utils import (
+    unify_inputs,
+    unify_targets,
+)
+
+
+class ModifiedReLU(Gradient):
+    """Basic class for modified ReLU explanation."""
+
+    def __init__(self, network, use_relu_backprop=False):
+        super(ModifiedReLU, self).__init__(network)
+        self.use_relu_backprop = use_relu_backprop
+        self.hooked_list = []
+
+    def __call__(self, inputs, targets):
+        self._verify_data(inputs, targets)
+        inputs = unify_inputs(inputs)
+        targets = unify_targets(targets)
+
+        self._hook_relu_backward()
+        gradients = self._grad_op(self._backward_model, inputs, targets)
+        saliency = self._aggregation_fn(gradients)
+
+        return saliency
+
+    def _hook_relu_backward(self):
+        """Set backward hook for ReLU layers."""
+        for _, cell in self._backward_model.cells_and_names():
+            if isinstance(cell, nn.ReLU):
+                cell.register_backward_hook(self._backward_hook)
+                self.hooked_list.append(cell)
+
+    def _backward_hook(self, _, grad_inputs, grad_outputs):
+        """Hook function for ReLU layers."""
+        inputs = grad_inputs if self.use_relu_backprop else grad_outputs
+        relu = op.ReLU()
+        if isinstance(inputs, tuple):
+            return relu(*inputs)
+        return relu(inputs)
+
+
+class Deconvolution(ModifiedReLU):
+    """
+    Deconvolution explanation.
+
+    To use `Deconvolution`, the `ReLU` operations in the network must be implemented with `mindspore.nn.Cell` object
+    rather than `mindspore.ops.Operations.ReLU`. Otherwise, the results will not be correct.
+
+    Args:
+        network (Cell): The black-box model to be explained.
+
+    Examples:
+        >>> net = resnet50(10)
+        >>> param_dict = load_checkpoint("resnet50.ckpt")
+        >>> load_param_into_net(net, param_dict)
+        >>> # bind net with its output activation if you wish, e.g. nn.Sigmoid(),
+        >>> # you may also use the net itself. The saliency map might be slightly different for softmax activation.
+        >>> net = nn.SequentialCell([net, nn.Sigmoid()])
+        >>> # init Gradient with a trained network.
+        >>> deconvolution = Deconvolution(net)
+        >>> # parse data and the target label to be explained and get the saliency map
+        >>> inputs = ms.Tensor(np.random.rand([1, 3, 224, 224]), ms.float32)
+        >>> label = 5
+        >>> saliency = deconvolution(inputs, label)
+    """
+
+    def __init__(self, network):
+        super(Deconvolution, self).__init__(network, use_relu_backprop=True)
+
+
+class GuidedBackprop(ModifiedReLU):
+    """
+    Guided-Backpropation explanation.
+
+    To use `GuidedBackprop`, the `ReLU` operations in the network must be implemented with `mindspore.nn.Cell` object
+    rather than `mindspore.ops.Operations.ReLU`. Otherwise, the results will not be correct.
+
+    Args:
+        network (Cell): The black-box model to be explained.
+
+    Examples:
+        >>> net = resnet50(10)
+        >>> param_dict = load_checkpoint("resnet50.ckpt")
+        >>> load_param_into_net(net, param_dict)
+        >>> # bind net with its output activation if you wish, e.g. nn.Sigmoid(),
+        >>> # you may also use the net itself. The saliency map might be slightly different for softmax activation.
+        >>> net = nn.SequentialCell([net, nn.Sigmoid()])
+        >>> # init Gradient with a trained network.
+        >>> gbp = GuidedBackprop(net)
+        >>> # parse data and the target label to be explained and get the saliency map
+        >>> inputs = ms.Tensor(np.random.rand([1, 3, 224, 224]), ms.float32)
+        >>> label = 5
+        >>> saliency = gbp(inputs, label)
+    """
+
+    def __init__(self, network):
+        super(GuidedBackprop, self).__init__(network, use_relu_backprop=False)