diff --git a/tests/st/explainer/__init__.py b/tests/st/explainer/__init__.py new file mode 100644 index 0000000000..142283a621 --- /dev/null +++ b/tests/st/explainer/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initialization of tests of explanation related classes.""" diff --git a/tests/st/explainer/benchmark/__init__.py b/tests/st/explainer/benchmark/__init__.py new file mode 100644 index 0000000000..b27beebb79 --- /dev/null +++ b/tests/st/explainer/benchmark/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initialization of tests of mindspore.explainer.benchmark.""" diff --git a/tests/st/explainer/benchmark/_attribution/__init__.py b/tests/st/explainer/benchmark/_attribution/__init__.py new file mode 100644 index 0000000000..5317b762dc --- /dev/null +++ b/tests/st/explainer/benchmark/_attribution/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initialization of tests of in mindspore.explainer.benchmark.""" diff --git a/tests/st/explainer/benchmark/_attribution/test_localization.py b/tests/st/explainer/benchmark/_attribution/test_localization.py new file mode 100644 index 0000000000..d456c1132f --- /dev/null +++ b/tests/st/explainer/benchmark/_attribution/test_localization.py @@ -0,0 +1,134 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests of Localization of mindspore.explainer.benchmark.""" + +from unittest.mock import patch + +import numpy as np +import pytest + +import mindspore as ms +from mindspore import context +from mindspore import nn +from mindspore.explainer.benchmark import Localization +from mindspore.explainer.explanation import Gradient + + +context.set_context(mode=context.PYNATIVE_MODE) + +H, W = 4, 4 +SALIENCY = ms.Tensor(np.random.rand(1, 1, H, W), ms.float32) + + +class CustomNet(nn.Cell): + """Simple net for unit test.""" + + def __init__(self): + super().__init__() + + def construct(self, _): + return ms.Tensor([[0.1, 0.9]], ms.float32) + + +def mock_gradient_call(_, inputs, targets): + del inputs, targets + return SALIENCY + + +class TestLocalization: + """Test on Localization.""" + + def setup_method(self): + self.net = CustomNet() + self.data = ms.Tensor(np.random.rand(1, 1, H, W), ms.float32) + self.target = 1 + + masks_np = np.zeros((1, 1, H, W)) + masks_np[:, :, 1:3, 1:3] = 1 + self.masks_np = masks_np + self.masks = ms.Tensor(masks_np, ms.float32) + + self.explainer = Gradient(self.net) + self.saliency_gt = mock_gradient_call(self.explainer, self.data, self.target) + self.num_class = 2 + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_pointing_game(self): + """Test case for `metric="PointingGame"` without input saliency.""" + with patch.object(Gradient, "__call__", mock_gradient_call): + max_pos = np.argmax(abs(self.saliency_gt.asnumpy().flatten())) + x_gt, y_gt = max_pos // W, max_pos % W + res_gt = self.masks_np[0, 0, x_gt, y_gt] + + pg = Localization(self.num_class, metric="PointingGame") + pg._metric_arg = 1 # make the tolerance smaller to simplify the test + + res = pg.evaluate(self.explainer, self.data, targets=self.target, mask=self.masks) + assert np.max(np.abs(np.array([res_gt]) - res)) < 1e-5 + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_iosr(self): + """Test case for `metric="IoSR"` without input saliency.""" + with patch.object(Gradient, "__call__", mock_gradient_call): + threshold = 0.5 + max_val = np.max(self.saliency_gt.asnumpy()) + sr = (self.saliency_gt.asnumpy() > (max_val * threshold)).astype(int) + res_gt = np.sum(sr * self.masks_np) / (np.sum(sr).clip(1e-10)) + + iosr = Localization(self.num_class, metric="IoSR") + iosr._metric_arg = threshold + + res = iosr.evaluate(self.explainer, self.data, targets=self.target, mask=self.masks) + + assert np.allclose(np.array([res_gt]), res) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_pointing_game_with_saliency(self): + """Test metric PointingGame with input saliency.""" + max_pos = np.argmax(abs(self.saliency_gt.asnumpy().flatten())) + x_gt, y_gt = max_pos // W, max_pos % W + res_gt = self.masks_np[0, 0, x_gt, y_gt] + + pg = Localization(self.num_class, metric="PointingGame") + pg._metric_arg = 1 # make the tolerance smaller to simplify the test + + res = pg.evaluate(self.explainer, self.data, targets=self.target, mask=self.masks, saliency=self.saliency_gt) + assert np.allclose(np.array([res_gt]), res) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_iosr_with_saliency(self): + """Test metric IoSR with input saliency map.""" + threshold = 0.5 + max_val = np.max(self.saliency_gt.asnumpy()) + sr = (self.saliency_gt.asnumpy() > (max_val * threshold)).astype(int) + res_gt = np.sum(sr * self.masks_np) / (np.sum(sr).clip(1e-10)) + + iosr = Localization(self.num_class, metric="IoSR") + + res = iosr.evaluate(self.explainer, self.data, targets=self.target, mask=self.masks, saliency=self.saliency_gt) + + assert np.allclose(np.array([res_gt]), res) diff --git a/tests/st/explainer/explanation/__init__.py b/tests/st/explainer/explanation/__init__.py new file mode 100644 index 0000000000..b56c349088 --- /dev/null +++ b/tests/st/explainer/explanation/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initialization of tests of mindspore.explainer.explanation.""" diff --git a/tests/st/explainer/explanation/_attribution/__init__.py b/tests/st/explainer/explanation/_attribution/__init__.py new file mode 100644 index 0000000000..aa25136448 --- /dev/null +++ b/tests/st/explainer/explanation/_attribution/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initialization of tests of explainers of mindspore.explainer.explanation.""" diff --git a/tests/st/explainer/explanation/_attribution/_backprop/__init__.py b/tests/st/explainer/explanation/_attribution/_backprop/__init__.py new file mode 100644 index 0000000000..80f20deaa6 --- /dev/null +++ b/tests/st/explainer/explanation/_attribution/_backprop/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initialization of tests of back-propagation based explainers.""" diff --git a/tests/st/explainer/explanation/_attribution/_backprop/test_gradcam.py b/tests/st/explainer/explanation/_attribution/_backprop/test_gradcam.py new file mode 100644 index 0000000000..e7b71937c5 --- /dev/null +++ b/tests/st/explainer/explanation/_attribution/_backprop/test_gradcam.py @@ -0,0 +1,104 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests of GradCAM of mindspore.explainer.explanation.""" + +from unittest.mock import patch + +import numpy as np +import pytest + +import mindspore as ms +from mindspore import context +import mindspore.ops.operations as op +from mindspore import nn +from mindspore.explainer.explanation import GradCAM +from mindspore.explainer.explanation._attribution._backprop.gradcam import _gradcam_aggregation as aggregation + + +context.set_context(mode=context.PYNATIVE_MODE) + + +class SimpleAvgLinear(nn.Cell): + """Simple linear model for the unit test.""" + + def __init__(self): + super().__init__() + self.avgpool = nn.AvgPool2d(2, 2) + self.flatten = nn.Flatten() + self.fc2 = nn.Dense(4, 3) + + def construct(self, x): + x = self.avgpool(x) + x = self.flatten(x) + return self.fc2(x) + + +def resize_fn(attributions, inputs, mode): + """Mocked resize function for test.""" + del inputs, mode + return attributions + + +class TestGradCAM: + """Test GradCAM.""" + + def setup_method(self): + self.net = SimpleAvgLinear() + self.data = ms.Tensor(np.random.random(size=(1, 1, 4, 4)), ms.float32) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_gradcam_attribution(self): + """Test __call__ method in GradCAM.""" + with patch.object(GradCAM, "_resize_fn", side_effect=resize_fn): + layer = "avgpool" + + gradcam = GradCAM(self.net, layer=layer) + + data = ms.Tensor(np.random.random(size=(1, 1, 4, 4)), ms.float32) + num_classes = 3 + activation = self.net.avgpool(data) + reshape = op.Reshape() + for x in range(num_classes): + target = ms.Tensor([x], ms.int32) + attribution = gradcam(data, target) + # intermediate grad should be reshape of weight of fc2 + intermediate_grad = self.net.fc2.weight.data[x] + reshaped = reshape(intermediate_grad, (1, 1, 2, 2)) + gap_grad = self.net.avgpool(reshaped) + res = aggregation(gap_grad * activation) + assert np.allclose(res.asnumpy(), attribution.asnumpy(), atol=1e-5, rtol=1e-3) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_layer_default(self): + """Test layer argument of GradCAM.""" + with patch.object(GradCAM, "_resize_fn", side_effect=resize_fn): + gradcam = GradCAM(self.net) + num_classes = 3 + sum_ = op.ReduceSum() + for x in range(num_classes): + target = ms.Tensor([x], ms.int32) + attribution = gradcam(self.data, target) + + # intermediate_grad should be reshape of weight of fc2 + intermediate_grad = self.net.fc2.weight.data[x] + avggrad = float(sum_(intermediate_grad).asnumpy() / 16) + res = aggregation(avggrad * self.data) + assert np.allclose(res.asnumpy(), attribution.asnumpy(), atol=1e-5, rtol=1e-3) diff --git a/tests/st/explainer/explanation/_attribution/_backprop/test_gradient.py b/tests/st/explainer/explanation/_attribution/_backprop/test_gradient.py new file mode 100644 index 0000000000..a71e7fa049 --- /dev/null +++ b/tests/st/explainer/explanation/_attribution/_backprop/test_gradient.py @@ -0,0 +1,74 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests of Gradient of mindspore.explainer.explanation.""" + +import numpy as np +import pytest + +import mindspore as ms +from mindspore import context +import mindspore.ops.operations as P +from mindspore import nn +from mindspore.explainer.explanation import Gradient + + +context.set_context(mode=context.PYNATIVE_MODE) + + +class SimpleLinear(nn.Cell): + """Simple linear model for the unit test.""" + + def __init__(self): + super().__init__() + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc2 = nn.Dense(16, 3) + + def construct(self, x): + x = self.relu(x) + x = self.flatten(x) + return self.fc2(x) + + +class TestGradient: + """Test Gradient.""" + + def setup_method(self): + """Setup the test case.""" + self.net = SimpleLinear() + self.relu = P.ReLU() + self.abs_ = P.Abs() + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_gradient(self): + """Test gradient __call__ function.""" + data = (ms.Tensor(np.random.random(size=(1, 1, 4, 4)), + ms.float32) - 0.5) * 2 + explainer = Gradient(self.net) + + num_classes = 3 + reshape = P.Reshape() + for x in range(num_classes): + target = ms.Tensor([x], ms.int32) + + attribution = explainer(data, target) + + # intermediate_grad should be reshape of weight of fc2 + grad = self.net.fc2.weight.data[x] + grad = self.abs_(reshape(grad, (1, 1, 4, 4)) * (self.abs_(self.relu(data) / data))) + assert np.allclose(grad.asnumpy(), attribution.asnumpy()) diff --git a/tests/st/explainer/explanation/_attribution/_backprop/test_modified_relu.py b/tests/st/explainer/explanation/_attribution/_backprop/test_modified_relu.py new file mode 100644 index 0000000000..f365bc32d4 --- /dev/null +++ b/tests/st/explainer/explanation/_attribution/_backprop/test_modified_relu.py @@ -0,0 +1,92 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests of Deconvolution and GuidedBackprop of mindspore.explainer.explanation.""" + +import numpy as np +import pytest + +import mindspore as ms +import mindspore.ops.operations as P +from mindspore import context +from mindspore import nn +from mindspore.explainer.explanation import Deconvolution, GuidedBackprop + + +context.set_context(mode=context.PYNATIVE_MODE) + + +class SimpleLinear(nn.Cell): + """Simple linear model for the unit test.""" + + def __init__(self): + super().__init__() + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc2 = nn.Dense(16, 3) + + def construct(self, x): + x = self.relu(x) + x = self.flatten(x) + return self.fc2(x) + + +class TestModifiedReLU: + """Test on modified_relu module, Deconvolution and GuidedBackprop specifically.""" + def setup_method(self): + """Setup the test case.""" + self.net = SimpleLinear() + self.relu = P.ReLU() + self.abs_ = P.Abs() + self.reshape = P.Reshape() + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_deconvolution(self): + """Test deconvolution attribution.""" + data = (ms.Tensor(np.random.random(size=(1, 1, 4, 4)), + ms.float32) - 0.5) * 2 + deconv = Deconvolution(self.net) + + num_classes = 3 + for x in range(num_classes): + target = ms.Tensor([x], ms.int32) + + attribution = deconv(data, target) + + # intermediate_grad should be reshape of weight of fc2 + grad = self.net.fc2.weight.data[x] + grad = self.abs_(self.relu(self.reshape(grad, (1, 1, 4, 4)))) + assert np.allclose(attribution.asnumpy(), grad.asnumpy()) + + def test_guided_backprop(self): + """Test deconvolution attribution.""" + data = (ms.Tensor(np.random.random(size=(1, 1, 4, 4)), + ms.float32) - 0.5) * 2 + explainer = GuidedBackprop(self.net) + + num_classes = 3 + for x in range(num_classes): + target = ms.Tensor([x], ms.int32) + + attribution = explainer(data, target) + + # intermediate_grad should be reshape of weight of fc2 + grad = self.net.fc2.weight.data[x] + grad = self.reshape(grad, (1, 1, 4, 4)) + guided_grad = self.abs_(self.relu(grad * (self.abs_(self.relu(data) / data)))) + + assert np.allclose(guided_grad.asnumpy(), attribution.asnumpy()) diff --git a/tests/st/explainer/test_runner.py b/tests/st/explainer/test_runner.py new file mode 100644 index 0000000000..8577a2ae24 --- /dev/null +++ b/tests/st/explainer/test_runner.py @@ -0,0 +1,200 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests on mindspore.explainer.ImageClassificationRunner.""" + +import os +import shutil +from random import random +from unittest.mock import patch + +import numpy as np +import pytest +from PIL import Image + +from mindspore import context +import mindspore as ms +import mindspore.nn as nn +from mindspore.dataset import GeneratorDataset +from mindspore.explainer import ImageClassificationRunner +from mindspore.explainer._image_classification_runner import _normalize +from mindspore.explainer.benchmark import Faithfulness +from mindspore.explainer.explanation import Gradient +from mindspore.train.summary import SummaryRecord + +CONST = random() +NUMDATA = 2 + +context.set_context(mode=context.PYNATIVE_MODE) + +def image_label_bbox_generator(): + for i in range(NUMDATA): + image = np.arange(i, i + 16 * 3).reshape((3, 4, 4)) / 50 + label = np.array(i) + bbox = np.array([1, 1, 2, 2]) + yield (image, label, bbox) + + +class SimpleNet(nn.Cell): + """ + Simple model for the unit test. + """ + + def __init__(self): + super(SimpleNet, self).__init__() + self.reshape = ms.ops.operations.Reshape() + + def construct(self, x): + prob = ms.Tensor([0.1, 0.9], ms.float32) + prob = self.reshape(prob, (1, 2)) + return prob + + +class ActivationFn(nn.Cell): + """ + Simple activation function for unit test. + """ + + def __init__(self): + super(ActivationFn, self).__init__() + + def construct(self, x): + return x + + +def mock_gradient_call(_, inputs, targets): + return inputs[:, 0:1, :, :] + + +def mock_faithfulness_evaluate(_, explainer, inputs, targets, saliency): + return CONST * targets + + +def mock_make_rgba(array): + return array.asnumpy() + + +class TestRunner: + """Test on Runner.""" + + def setup_method(self): + self.dataset = GeneratorDataset(image_label_bbox_generator, ["image", "label", "bbox"]) + self.labels = ["label_{}".format(i) for i in range(2)] + self.network = SimpleNet() + self.summary_dir = "summary_test_temp" + self.explainer = [Gradient(self.network)] + self.activation_fn = ActivationFn() + self.benchmarkers = [Faithfulness(num_labels=len(self.labels), + metric="NaiveFaithfulness", + activation_fn=self.activation_fn)] + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_run_saliency_no_benchmark(self): + """Test case when argument benchmarkers is not parsed.""" + res = [] + runner = ImageClassificationRunner(summary_dir=self.summary_dir, data=(self.dataset, self.labels), + network=self.network, activation_fn=self.activation_fn) + + def mock_summary_add_value(_, plugin, name, value): + res.append((plugin, name, value)) + + with patch.object(SummaryRecord, "add_value", mock_summary_add_value), \ + patch.object(Gradient, "__call__", mock_gradient_call): + runner.register_saliency(self.explainer) + runner.run() + + # test on meta data + idx = 0 + assert res[idx][0] == "explainer" + assert res[idx][1] == "metadata" + assert res[idx][2].metadata.label == self.labels + assert res[idx][2].metadata.explain_method == ["Gradient"] + + # test on inference data + for i in range(NUMDATA): + idx += 1 + data_np = np.arange(i, i + 3 * 16).reshape((3, 4, 4)) / 50 + assert res[idx][0] == "explainer" + assert res[idx][1] == "sample" + assert res[idx][2].sample_id == i + original_path = os.path.join(self.summary_dir, res[idx][2].image_path) + with open(original_path, "rb") as f: + image_data = np.asarray(Image.open(f)) / 255.0 + original_image = _normalize(np.transpose(data_np, [1, 2, 0])) + assert np.allclose(image_data, original_image, rtol=3e-2, atol=3e-2) + + idx += 1 + assert res[idx][0] == "explainer" + assert res[idx][1] == "inference" + assert res[idx][2].sample_id == i + assert res[idx][2].ground_truth_label == [i] + + diff = np.array(res[idx][2].inference.ground_truth_prob) - np.array([[0.1, 0.9][i]]) + assert np.max(np.abs(diff)) < 1e-6 + assert res[idx][2].inference.predicted_label == [1] + diff = np.array(res[idx][2].inference.predicted_prob) - np.array([0.9]) + assert np.max(np.abs(diff)) < 1e-6 + + # test on explanation data + for i in range(NUMDATA): + idx += 1 + data_np = np.arange(i, i + 3 * 16).reshape((3, 4, 4)) / 50 + saliency_np = data_np[0, :, :] + assert res[idx][0] == "explainer" + assert res[idx][1] == "explanation" + assert res[idx][2].sample_id == i + assert res[idx][2].explanation[0].explain_method == "Gradient" + + assert res[idx][2].explanation[0].label in [i, 1] + + heatmap_path = os.path.join(self.summary_dir, res[idx][2].explanation[0].heatmap_path) + assert os.path.exists(heatmap_path) + + with open(heatmap_path, "rb") as f: + heatmap_data = np.asarray(Image.open(f)) / 255.0 + heatmap_image = _normalize(saliency_np) + assert np.allclose(heatmap_data, heatmap_image, atol=3e-2, rtol=3e-2) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_run_saliency_with_benchmark(self): + """Test case when argument benchmarkers is parsed.""" + res = [] + + def mock_summary_add_value(_, plugin, name, value): + res.append((plugin, name, value)) + + runner = ImageClassificationRunner(summary_dir=self.summary_dir, data=(self.dataset, self.labels), + network=self.network, activation_fn=self.activation_fn) + + with patch.object(SummaryRecord, "add_value", mock_summary_add_value), \ + patch.object(Gradient, "__call__", mock_gradient_call), \ + patch.object(Faithfulness, "evaluate", mock_faithfulness_evaluate): + runner.register_saliency(self.explainer, self.benchmarkers) + runner.run() + + idx = 3 * NUMDATA + 1 # start index of benchmark data + assert res[idx][0] == "explainer" + assert res[idx][1] == "benchmark" + assert abs(res[idx][2].benchmark[0].total_score - 2 / 3 * CONST) < 1e-6 + diff = np.array(res[idx][2].benchmark[0].label_score) - np.array([i * CONST for i in range(NUMDATA)]) + assert np.max(np.abs(diff)) < 1e-6 + + def teardown_method(self): + shutil.rmtree(self.summary_dir) diff --git a/tests/st/explainer/test_utils.py b/tests/st/explainer/test_utils.py new file mode 100644 index 0000000000..b8e4d8c9e1 --- /dev/null +++ b/tests/st/explainer/test_utils.py @@ -0,0 +1,119 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Unit test on mindspore.explainer._utils.""" + +import numpy as np +import pytest + +import mindspore as ms +import mindspore.nn as nn + +from mindspore.explainer._utils import ( + ForwardProbe, + rank_pixels, + retrieve_layer, + retrieve_layer_by_name) +from mindspore.explainer.explanation._attribution._backprop.backprop_utils import GradNet, get_bp_weights + + +class CustomNet(nn.Cell): + """Simple net for test.""" + + def __init__(self): + super(CustomNet, self).__init__() + self.fc1 = nn.Dense(10, 10) + self.fc2 = nn.Dense(10, 10) + self.fc3 = nn.Dense(10, 10) + self.fc4 = nn.Dense(10, 10) + + def construct(self, inputs): + out = self.fc1(inputs) + out = self.fc2(out) + out = self.fc3(out) + out = self.fc4(out) + return out + + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_rank_pixels(): + """Test on rank_pixels.""" + saliency = np.array([[4., 3., 1.], [5., 9., 1.]]) + descending_target = np.array([[0, 1, 2], [1, 0, 2]]) + ascending_target = np.array([[2, 1, 0], [1, 2, 0]]) + descending_rank = rank_pixels(saliency) + ascending_rank = rank_pixels(saliency, descending=False) + assert (descending_rank - descending_target).any() == 0 + assert (ascending_rank - ascending_target).any() == 0 + + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_retrieve_layer_by_name(): + """Test on rank_pixels.""" + model = CustomNet() + target_layer_name = 'fc3' + target_layer = retrieve_layer_by_name(model, target_layer_name) + + assert target_layer is model.fc3 + + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_retrieve_layer_by_name_no_name(): + """Test on retrieve layer.""" + model = CustomNet() + target_layer = retrieve_layer_by_name(model, '') + + assert target_layer is model + + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_forward_probe(): + """Test case for ForwardProbe.""" + model = CustomNet() + model.set_grad() + inputs = np.random.random((1, 10)) + inputs = ms.Tensor(inputs, ms.float32) + gt_activation = model.fc3(model.fc2(model.fc1(inputs))).asnumpy() + + targets = 1 + weights = get_bp_weights(model, inputs, targets=targets) + + gradnet = GradNet(model) + grad_before_probe = gradnet(inputs, weights).asnumpy() + + # Probe forward tensor + saliency_layer = retrieve_layer(model, 'fc3') + + with ForwardProbe(saliency_layer) as probe: + grad_after_probe = gradnet(inputs, weights).asnumpy() + activation = probe.value.asnumpy() + + grad_after_unprobe = gradnet(inputs, weights).asnumpy() + + assert np.array_equal(gt_activation, activation) + assert np.array_equal(grad_before_probe, grad_after_probe) + assert np.array_equal(grad_before_probe, grad_after_unprobe) + assert probe.value is None