From a8f392ca4280af55c8e7026e54ed856267acd54b Mon Sep 17 00:00:00 2001 From: Danish Farid Date: Wed, 24 Jun 2020 12:41:41 -0400 Subject: [PATCH] updated test util file + new BoundingBoxCheck + fixed VOCDataset annotations Style Error fix fixed PyLint problem reverting testVOC2012 due to CI break for existing test reverting testVOC2012 due to CI break for existing test-2 updated old error messages to confirm with global standard addressing PR 2355 Comments - 1 addressing PR 2355 Comments - 2 addressing PR 2355 Comments - 3 --- .../random_vertical_flip_with_bbox_op.cc | 2 +- mindspore/ccsrc/dataset/kernels/tensor_op.h | 8 +- .../dataset/testVOC2012/Annotations/129.xml | 27 +++ .../ut/data/dataset/testVOC2012_2/README.txt | 2 + .../test_random_crop_and_resize_with_bbox.py | 4 +- .../dataset/test_random_crop_with_bbox.py | 24 +-- .../test_random_vertical_flip_with_bbox.py | 204 +++--------------- tests/ut/python/dataset/util.py | 123 +++++++++++ 8 files changed, 197 insertions(+), 197 deletions(-) create mode 100644 tests/ut/data/dataset/testVOC2012/Annotations/129.xml create mode 100644 tests/ut/data/dataset/testVOC2012_2/README.txt diff --git a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc index d88c009559..c6aa8450a8 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc @@ -41,7 +41,7 @@ Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow * RETURN_IF_NOT_OK(input[1]->GetUnsignedIntAt(&boxHeight, {i, 3})); // get height of bbox // subtract (curCorner + height) from (max) for new Corner position - newBoxCorner_y = (imHeight - 1) - (boxCorner_y + boxHeight); + newBoxCorner_y = (imHeight - 1) - ((boxCorner_y + boxHeight) - 1); RETURN_IF_NOT_OK(input[1]->SetItemAt({i, 1}, newBoxCorner_y)); } diff --git a/mindspore/ccsrc/dataset/kernels/tensor_op.h b/mindspore/ccsrc/dataset/kernels/tensor_op.h index 293d4a4f99..9aae50d6b0 100644 --- a/mindspore/ccsrc/dataset/kernels/tensor_op.h +++ b/mindspore/ccsrc/dataset/kernels/tensor_op.h @@ -45,14 +45,18 @@ #define BOUNDING_BOX_CHECK(input) \ do { \ + if (input.size() != 2) { \ + return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ + "Requires Image and Bounding Boxes, likely missed bounding boxes."); \ + } \ if (input[1]->shape().Size() < 2) { \ return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ - "Bounding boxes shape should have at least two dims"); \ + "Bounding boxes shape should have at least two dimensions."); \ } \ uint32_t num_of_features = input[1]->shape()[1]; \ if (num_of_features < 4) { \ return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ - "Bounding boxes should be have at least 4 features"); \ + "Bounding boxes should be have at least 4 features."); \ } \ uint32_t num_of_boxes = input[1]->shape()[0]; \ uint32_t img_h = input[0]->shape()[0]; \ diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/129.xml b/tests/ut/data/dataset/testVOC2012/Annotations/129.xml new file mode 100644 index 0000000000..3bb822f545 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/129.xml @@ -0,0 +1,27 @@ + + VOC2012 + 129.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 375 + 3 + + 1 + + dog + Frontal + 0 + 0 + + 1124 + 437 + 1684 + 2669 + + + diff --git a/tests/ut/data/dataset/testVOC2012_2/README.txt b/tests/ut/data/dataset/testVOC2012_2/README.txt new file mode 100644 index 0000000000..6410067c69 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012_2/README.txt @@ -0,0 +1,2 @@ +Custom VOC2012-like dataset with valid annotations for images. +Created to test BoundingBox Augmentation Ops - June 2020. \ No newline at end of file diff --git a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py index 3dd97d2512..90269a7027 100644 --- a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py @@ -305,8 +305,8 @@ def test_c_random_resized_crop_with_bbox_op_bad(): if __name__ == "__main__": - test_c_random_resized_crop_with_bbox_op(False) - test_c_random_resized_crop_with_bbox_op_edge(False) + test_c_random_resized_crop_with_bbox_op(plot_vis=True) + test_c_random_resized_crop_with_bbox_op_edge(plot_vis=True) test_c_random_resized_crop_with_bbox_op_invalid() test_c_random_resized_crop_with_bbox_op_invalid2() test_c_random_resized_crop_with_bbox_op_bad() diff --git a/tests/ut/python/dataset/test_random_crop_with_bbox.py b/tests/ut/python/dataset/test_random_crop_with_bbox.py index d1e2e08419..7f5fa46512 100644 --- a/tests/ut/python/dataset/test_random_crop_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_with_bbox.py @@ -142,7 +142,7 @@ def gen_bbox_edge(im, bbox): return im, bbox -def c_random_crop_with_bbox_op(plot_vis=False): +def test_random_crop_with_bbox_op_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes """ @@ -176,7 +176,7 @@ def c_random_crop_with_bbox_op(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op2(plot_vis=False): +def test_random_crop_with_bbox_op2_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes With Fill Value @@ -212,7 +212,7 @@ def c_random_crop_with_bbox_op2(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op3(plot_vis=False): +def test_random_crop_with_bbox_op3_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes With Padding Mode passed @@ -247,7 +247,7 @@ def c_random_crop_with_bbox_op3(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op_edge(plot_vis=False): +def test_random_crop_with_bbox_op_edge_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes Testing for an Edge case @@ -289,7 +289,7 @@ def c_random_crop_with_bbox_op_edge(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op_invalid(): +def test_random_crop_with_bbox_op_invalid_c(): """ Checking for invalid params passed to Aug Constructor """ @@ -319,7 +319,7 @@ def c_random_crop_with_bbox_op_invalid(): assert "Size should be a single integer" in str(err) -def c_random_crop_with_bbox_op_bad(): +def test_random_crop_with_bbox_op_bad_c(): # Should Fail - Errors logged to logger for ix, badFunc in enumerate(badGenFuncs): try: @@ -352,9 +352,9 @@ def c_random_crop_with_bbox_op_bad(): if __name__ == "__main__": - c_random_crop_with_bbox_op(False) - c_random_crop_with_bbox_op2(False) - c_random_crop_with_bbox_op3(False) - c_random_crop_with_bbox_op_edge(False) - c_random_crop_with_bbox_op_invalid() - c_random_crop_with_bbox_op_bad() + test_random_crop_with_bbox_op_c(plot_vis=True) + test_random_crop_with_bbox_op2_c(plot_vis=True) + test_random_crop_with_bbox_op3_c(plot_vis=True) + test_random_crop_with_bbox_op_edge_c(plot_vis=True) + test_random_crop_with_bbox_op_invalid_c() + test_random_crop_with_bbox_op_bad_c() diff --git a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py index e0c8b455f4..b1bb4bc459 100644 --- a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py +++ b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py @@ -15,15 +15,12 @@ """ Testing RandomVerticalFlipWithBBox op """ -import numpy as np - -import matplotlib.pyplot as plt -import matplotlib.patches as patches import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c_vision from mindspore import log as logger +from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox # updated VOC dataset with correct annotations DATA_DIR = "../data/dataset/testVOC2012_2" @@ -46,106 +43,11 @@ def fix_annotate(bboxes): return bboxes -def add_bounding_boxes(ax, bboxes): - for bbox in bboxes: - rect = patches.Rectangle((bbox[0], bbox[1]), - bbox[2], bbox[3], - linewidth=1, edgecolor='r', facecolor='none') - # Add the patch to the Axes - ax.add_patch(rect) - - -def vis_check(orig, aug): - if not isinstance(orig, list) or not isinstance(aug, list): - return False - if len(orig) != len(aug): - return False - return True - - -def visualize(orig, aug): - - if not vis_check(orig, aug): - return - - plotrows = 3 - compset = int(len(orig)/plotrows) - - orig, aug = np.array(orig), np.array(aug) - - orig = np.split(orig[:compset*plotrows], compset) + [orig[compset*plotrows:]] - aug = np.split(aug[:compset*plotrows], compset) + [aug[compset*plotrows:]] - - for ix, allData in enumerate(zip(orig, aug)): - base_ix = ix * plotrows # will signal what base level we're on - fig, axs = plt.subplots(len(allData[0]), 2) - fig.tight_layout(pad=1.5) - - for x, (dataA, dataB) in enumerate(zip(allData[0], allData[1])): - cur_ix = base_ix + x - - axs[x, 0].imshow(dataA["image"]) - add_bounding_boxes(axs[x, 0], dataA["annotation"]) - axs[x, 0].title.set_text("Original" + str(cur_ix+1)) - print("Original **\n ", str(cur_ix+1), " :", dataA["annotation"]) - - axs[x, 1].imshow(dataB["image"]) - add_bounding_boxes(axs[x, 1], dataB["annotation"]) - axs[x, 1].title.set_text("Augmented" + str(cur_ix+1)) - print("Augmented **\n", str(cur_ix+1), " ", dataB["annotation"], "\n") - - plt.show() - -# Functions to pass to Gen for creating invalid bounding boxes - - -def gen_bad_bbox_neg_xy(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [-50, -50, im_w - 10, im_h - 10] - return im, bbox - - -def gen_bad_bbox_overflow_width(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [0, 0, im_w + 10, im_h - 10] - return im, bbox - - -def gen_bad_bbox_overflow_height(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [0, 0, im_w - 10, im_h + 10] - return im, bbox - - -def gen_bad_bbox_wrong_shape(im, bbox): - bbox = np.array([[0, 0, 0]]).astype(bbox.dtype) - return im, bbox - - -badGenFuncs = [gen_bad_bbox_neg_xy, - gen_bad_bbox_overflow_width, - gen_bad_bbox_overflow_height, - gen_bad_bbox_wrong_shape] - -assertVal = ["min_x", - "is out of bounds of the image", - "is out of bounds of the image", - "4 features"] - - -# Gen Edge case BBox -def gen_bbox_edge(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [0, 0, im_w, im_h] - return im, bbox - - -def c_random_vertical_flip_with_bbox_op(plot_vis=False): +def test_random_vertical_flip_with_bbox_op_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes to compare and test """ - # Load dataset dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) @@ -175,10 +77,10 @@ def c_random_vertical_flip_with_bbox_op(plot_vis=False): augSamp.append(Aug) if plot_vis: - visualize(unaugSamp, augSamp) + visualize_with_bounding_boxes(unaugSamp, augSamp) -def c_random_vertical_flip_with_bbox_op_rand(plot_vis=False): +def test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes to compare and test @@ -213,54 +115,12 @@ def c_random_vertical_flip_with_bbox_op_rand(plot_vis=False): augSamp.append(Aug) if plot_vis: - visualize(unaugSamp, augSamp) - + visualize_with_bounding_boxes(unaugSamp, augSamp) -def c_random_vertical_flip_with_bbox_op_edge(plot_vis=False): - # Should Pass - # Load dataset - dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", - decode=True, shuffle=False) - - dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", - decode=True, shuffle=False) - test_op = c_vision.RandomVerticalFlipWithBBox(0.6) - - # maps to fix annotations to HQ standard - dataVoc1 = dataVoc1.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - dataVoc2 = dataVoc2.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - - # Modify BBoxes to serve as valid edge cases - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[gen_bbox_edge]) - - # map to apply ops - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) - - unaugSamp, augSamp = [], [] - - for unAug, Aug in zip(dataVoc1.create_dict_iterator(), dataVoc2.create_dict_iterator()): - unaugSamp.append(unAug) - augSamp.append(Aug) - - if plot_vis: - visualize(unaugSamp, augSamp) - - -def c_random_vertical_flip_with_bbox_op_invalid(): +def test_random_vertical_flip_with_bbox_op_invalid_c(): # Should Fail # Load dataset - dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) @@ -286,41 +146,25 @@ def c_random_vertical_flip_with_bbox_op_invalid(): assert "Input is not" in str(err) -def c_random_vertical_flip_with_bbox_op_bad(): - # Should Fail - Errors logged to logger - for ix, badFunc in enumerate(badGenFuncs): - try: - dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", - decode=True, shuffle=False) - - test_op = c_vision.RandomVerticalFlipWithBBox(1) - - dataVoc2 = dataVoc2.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[badFunc]) - - # map to apply ops - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) - - for _ in dataVoc2.create_dict_iterator(): - break # first sample will cause exception +def test_random_vertical_flip_with_bbox_op_bad_c(): + """ + Test RandomHorizontalFlipWithBBox op with invalid bounding boxes + """ + logger.info("test_random_horizontal_bbox_invalid_bounds_c") + test_op = c_vision.RandomVerticalFlipWithBBox(1) - except RuntimeError as err: - logger.info("Got an exception in DE: {}".format(str(err))) - assert assertVal[ix] in str(err) + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WidthOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.HeightOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.NegativeXY, "min_x") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WrongShape, "4 features") if __name__ == "__main__": - c_random_vertical_flip_with_bbox_op(False) - c_random_vertical_flip_with_bbox_op_rand(False) - c_random_vertical_flip_with_bbox_op_edge(False) - c_random_vertical_flip_with_bbox_op_invalid() - c_random_vertical_flip_with_bbox_op_bad() + test_random_vertical_flip_with_bbox_op_c(plot_vis=True) + test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=True) + test_random_vertical_flip_with_bbox_op_invalid_c() + test_random_vertical_flip_with_bbox_op_bad_c() diff --git a/tests/ut/python/dataset/util.py b/tests/ut/python/dataset/util.py index 11335e120b..00a2c7ef57 100644 --- a/tests/ut/python/dataset/util.py +++ b/tests/ut/python/dataset/util.py @@ -16,7 +16,9 @@ import hashlib import json import os +from enum import Enum import matplotlib.pyplot as plt +import matplotlib.patches as patches import numpy as np # import jsbeautifier import mindspore.dataset as ds @@ -284,3 +286,124 @@ def config_get_set_num_parallel_workers(num_parallel_workers_new): logger.info("num_parallel_workers: original = {} new = {} ".format(num_parallel_workers_original, num_parallel_workers_new)) return num_parallel_workers_original + + +def visualize_with_bounding_boxes(orig, aug, plot_rows=3): + """ + Take a list of un-augmented and augmented images with "annotation" bounding boxes + Plot images to compare test correct BBox augment functionality + :param orig: list of original images and bboxes (without aug) + :param aug: list of augmented images and bboxes + :param plot_rows: number of rows on plot (rows = samples on one plot) + :return: None + """ + + def add_bounding_boxes(ax, bboxes): + for bbox in bboxes: + rect = patches.Rectangle((bbox[0], bbox[1]), + bbox[2], bbox[3], + linewidth=1, edgecolor='r', facecolor='none') + # Add the patch to the Axes + ax.add_patch(rect) + + # Quick check to confirm correct input parameters + if not isinstance(orig, list) or not isinstance(aug, list): + return + if len(orig) != len(aug) or not orig: + return + + comp_set = int(len(orig)/plot_rows) + orig, aug = np.array(orig), np.array(aug) + + if len(orig) > plot_rows: + orig = np.split(orig[:comp_set*plot_rows], comp_set) + [orig[comp_set*plot_rows:]] + aug = np.split(aug[:comp_set*plot_rows], comp_set) + [aug[comp_set*plot_rows:]] + else: + orig = [orig] + aug = [aug] + + for ix, allData in enumerate(zip(orig, aug)): + base_ix = ix * plot_rows # will signal what base level we're on + + sub_plot_count = 2 if (len(allData[0]) < 2) else len(allData[0]) # if 1 image remains, create subplot for 2 to simplify axis selection + fig, axs = plt.subplots(sub_plot_count, 2) + fig.tight_layout(pad=1.5) + + for x, (dataA, dataB) in enumerate(zip(allData[0], allData[1])): + cur_ix = base_ix + x + + axs[x, 0].imshow(dataA["image"]) + add_bounding_boxes(axs[x, 0], dataA["annotation"]) + axs[x, 0].title.set_text("Original" + str(cur_ix+1)) + logger.info("Original **\n{} : {}".format(str(cur_ix+1), dataA["annotation"])) + + axs[x, 1].imshow(dataB["image"]) + add_bounding_boxes(axs[x, 1], dataB["annotation"]) + axs[x, 1].title.set_text("Augmented" + str(cur_ix+1)) + logger.info("Augmented **\n{} : {}\n".format(str(cur_ix+1), dataB["annotation"])) + + plt.show() + + +class InvalidBBoxType(Enum): + """ + Defines Invalid Bounding Bbox types for test cases + """ + WidthOverflow = 1 + HeightOverflow = 2 + NegativeXY = 3 + WrongShape = 4 + + +def check_bad_bbox(data, test_op, invalid_bbox_type, expected_error): + """ + :param data: de object detection pipeline + :param test_op: Augmentation Op to test on image + :param invalid_bbox_type: type of bad box + :param expected_error: error expected to get due to bad box + :return: None + """ + + def add_bad_annotation(img, bboxes, invalid_bbox_type_): + """ + Used to generate erroneous bounding box examples on given img. + :param img: image where the bounding boxes are. + :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format + :param box_type_: type of bad box + :return: bboxes with bad examples added + """ + height = img.shape[0] + width = img.shape[1] + if invalid_bbox_type_ == InvalidBBoxType.WidthOverflow: + # use box that overflows on width + return img, np.array([[0, 0, width + 1, height, 0, 0, 0]]).astype(np.uint32) + + if invalid_bbox_type_ == InvalidBBoxType.HeightOverflow: + # use box that overflows on height + return img, np.array([[0, 0, width, height + 1, 0, 0, 0]]).astype(np.uint32) + + if invalid_bbox_type_ == InvalidBBoxType.NegativeXY: + # use box with negative xy + return img, np.array([[-10, -10, width, height, 0, 0, 0]]).astype(np.uint32) + + if invalid_bbox_type_ == InvalidBBoxType.WrongShape: + # use box that has incorrect shape + return img, np.array([[0, 0, width - 1]]).astype(np.uint32) + return img, bboxes + + try: + # map to use selected invalid bounding box type + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=lambda img, bboxes: add_bad_annotation(img, bboxes, invalid_bbox_type)) + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + for _, _ in enumerate(data.create_dict_iterator()): + break + except RuntimeError as error: + logger.info("Got an exception in DE: {}".format(str(error))) + assert expected_error in str(error)