add warp affine

mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/CMakeLists.txt

@@ -2,4 +2,5 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc"
 set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
 add_library(lite-cv OBJECT
             image_process.cc
+            warp_affine.cc
             lite_mat.cc)

mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc

@@ -28,11 +28,34 @@
 #endif
 #endif
 
-namespace mindspore {
+#ifdef PLATFORM_ARM64
-namespace dataset {
+#define R2GRAY 9798
+#define G2GRAY 19235
+#define B2GRAY 3735
+#define GRAYSHIFT 15
+#define GRAYSHIFT_DELTA (1 << (GRAYSHIFT - 1))
+#else
+#define R2GRAY 77
+#define G2GRAY 150
+#define B2GRAY 29
+#define GRAYSHIFT 8
+#endif
 
+#define YSCALE 0x0101
+#define UTOB (-128)
+#define UTOG 25
+#define VTOR (-102)
+#define VTOG 52
+#define YTOG 18997
+#define YTOGB (-1160)
+#define BTOB (UTOB * 128 + YTOGB)
+#define BTOG (UTOG * 128 + VTOG * 128 + YTOGB)
+#define BTOR (VTOR * 128 + YTOGB)
 #define Equ(a, b) ((std::fabs((a) - (b)) < 1e-6))
 
+namespace mindspore {
+namespace dataset {
+
 static inline void InitBilinearWeight(int *data_ptr, int16_t *weight_ptr, double scale, int dst_length, int src_length,
                                       int a) {
   const int RESIZE_SCALE = 1 << 11;
@@ -374,6 +397,79 @@ static bool ConvertYUV420SPToBGR(const uint8_t *data, LDataType data_type, bool
   return true;
 }
 
+#ifdef PLATFORM_ARM64
+static uint8x8_t RGBToGray(const uint16x8_t &r_value, const uint16x8_t &g_value, const uint16x8_t &b_value,
+                           const uint16x4_t &r2y_value, const uint16x4_t &g2y_value, const uint16x4_t &b2y_value) {
+  uint32x4_t dst0_value = vmull_u16(vget_low_u16(g_value), g2y_value);
+  uint32x4_t dst1_value = vmull_u16(vget_high_u16(g_value), g2y_value);
+
+  dst0_value = vmlal_u16(dst0_value, vget_low_u16(r_value), r2y_value);
+  dst1_value = vmlal_u16(dst1_value, vget_high_u16(r_value), r2y_value);
+
+  dst0_value = vmlal_u16(dst0_value, vget_low_u16(b_value), b2y_value);
+  dst1_value = vmlal_u16(dst1_value, vget_high_u16(b_value), b2y_value);
+
+  uint8x8_t v_gray = vqmovn_u16(vcombine_u16(vrshrn_n_u32(dst0_value, GRAYSHIFT), vrshrn_n_u32(dst1_value, GRAYSHIFT)));
+
+  return v_gray;
+}
+
+static bool ConvertRGBAToGRAY_Neon(const uint8_t *srcBase, uint8_t *dstBase, int w, int h) {
+  const uint32_t r_to_gray = R2GRAY;
+  const uint32_t g_to_gray = G2GRAY;
+  const uint32_t b_to_gray = B2GRAY;
+
+  uint16x4_t r2y_value = vdup_n_u16(R2GRAY);
+  uint16x4_t g2y_value = vdup_n_u16(G2GRAY);
+  uint16x4_t b2y_value = vdup_n_u16(B2GRAY);
+
+  size_t w16b = w >= 15 ? w - 15 : 0;
+  size_t w8b = w >= 7 ? w - 7 : 0;
+
+  for (size_t i = 0; i < h; ++i) {
+    const uint8_t *src_ptr = srcBase + w * i * 4;
+    uint8_t *dst_ptr = dstBase + w * i * 4;
+    size_t src_j = 0u;
+    size_t dst_j = 0u;
+
+    for (; dst_j < w16b; src_j += 64, dst_j += 16) {
+      uint8x16x4_t src_value0 = vld4q_u8(src_ptr + src_j);
+
+      // 0
+      uint16x8_t r_value = vmovl_u8(vget_low_u8(src_value0.val[0]));
+      uint16x8_t g_value = vmovl_u8(vget_low_u8(src_value0.val[1]));
+      uint16x8_t b_value = vmovl_u8(vget_low_u8(src_value0.val[2]));
+      uint8x8_t gray_value0 = RGBToGray(r_value, g_value, b_value, r2y_value, g2y_value, b2y_value);
+
+      r_value = vmovl_u8(vget_high_u8(src_value0.val[0]));
+      g_value = vmovl_u8(vget_high_u8(src_value0.val[1]));
+      b_value = vmovl_u8(vget_high_u8(src_value0.val[2]));
+      uint8x8_t gray_value1 = RGBToGray(r_value, g_value, b_value, r2y_value, g2y_value, b2y_value);
+
+      vst1q_u8(dst_ptr + dst_j, vcombine_u8(gray_value0, gray_value1));
+    }
+
+    if (dst_j < w8b) {
+      uint8x8x4_t v_src = vld4_u8(src_ptr + src_j);
+      uint16x8_t r_value = vmovl_u8(v_src.val[0]);
+      uint16x8_t g_value = vmovl_u8(v_src.val[1]);
+      uint16x8_t b_value = vmovl_u8(v_src.val[2]);
+      uint8x8_t gray_value = RGBToGray(r_value, g_value, b_value, r2y_value, g2y_value, b2y_value);
+
+      vst1_u8(dst_ptr + dst_j, gray_value);
+      src_j += 32;
+      dst_j += 8;
+    }
+
+    for (; dst_j < w; src_j += 4, dst_j++) {
+      uint32_t val = src_ptr[src_j] * r_to_gray + src_ptr[src_j + 1] * g_to_gray + src_ptr[src_j + 2] * b_to_gray;
+      dst_ptr[dst_j] = U32TOU8CAST((val + GRAYSHIFT_DELTA) >> GRAYSHIFT);
+    }
+  }
+  return true;
+}
+#endif
+
 static bool ConvertRGBAToGRAY(const unsigned char *data, LDataType data_type, int w, int h, LiteMat &mat) {
   if (data_type == LDataType::UINT8) {
     mat.Init(w, h, 1, LDataType::UINT8);
@@ -382,6 +478,9 @@ static bool ConvertRGBAToGRAY(const unsigned char *data, LDataType data_type, in
     }
     unsigned char *ptr = mat;
     const unsigned char *data_ptr = data;
+#ifdef PLATFORM_ARM64
+    ConvertRGBAToGRAY_Neon(data_ptr, ptr, w, h);
+#else
     for (int y = 0; y < h; y++) {
       for (int x = 0; x < w; x++) {
         *ptr = (data_ptr[2] * B2GRAY + data_ptr[1] * G2GRAY + data_ptr[0] * R2GRAY) >> GRAYSHIFT;
@@ -389,6 +488,7 @@ static bool ConvertRGBAToGRAY(const unsigned char *data, LDataType data_type, in
         data_ptr += 4;
       }
     }
+#endif
   } else {
     return false;
   }

mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h

@@ -30,22 +30,6 @@ namespace dataset {
 #define INT16_CAST(X) \
   static_cast<int16_t>(::std::min(::std::max(static_cast<int>(X + (X >= 0.f ? 0.5f : -0.5f)), -32768), 32767));
 
-#define R2GRAY 77
-#define G2GRAY 150
-#define B2GRAY 29
-#define GRAYSHIFT 8
-
-#define YSCALE 0x0101
-#define UTOB (-128)
-#define UTOG 25
-#define VTOR (-102)
-#define VTOG 52
-#define YTOG 18997
-#define YTOGB (-1160)
-#define BTOB (UTOB * 128 + YTOGB)
-#define BTOG (UTOG * 128 + VTOG * 128 + YTOGB)
-#define BTOR (VTOR * 128 + YTOGB)
-
 enum PaddBorderType { PADD_BORDER_CONSTANT = 0, PADD_BORDER_REPLICATE = 1 };
 
 struct BoxesConfig {
@@ -107,6 +91,14 @@ void ConvertBoxes(std::vector<std::vector<float>> &boxes, const std::vector<std:
 std::vector<int> ApplyNms(const std::vector<std::vector<float>> &all_boxes, std::vector<float> &all_scores, float thres,
                           int max_boxes);
 
+/// \brief affine image by linear
+bool WarpAffineBilinear(const LiteMat &src, LiteMat &dst, const LiteMat &M, int dst_w, int dst_h,
+                        PaddBorderType borderType, std::vector<uint8_t> &borderValue);
+
+/// \brief perspective image by linear
+bool WarpPerspectiveBilinear(const LiteMat &src, LiteMat &dst, const LiteMat &M, int dst_w, int dst_h,
+                             PaddBorderType borderType, std::vector<uint8_t> &borderValue);
+
 }  // namespace dataset
 }  // namespace mindspore
 #endif  // IMAGE_PROCESS_H_

mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc

@@ -39,6 +39,7 @@ LiteMat::LiteMat() {
   size_ = 0;
   data_type_ = LDataType::UINT8;
   ref_count_ = nullptr;
+  setSteps(0, 0, 0);
 }
 
 LiteMat::LiteMat(int width, LDataType data_type) {
@@ -52,6 +53,7 @@ LiteMat::LiteMat(int width, LDataType data_type) {
   data_type_ = LDataType::UINT8;
   ref_count_ = nullptr;
   size_ = 0;
+  setSteps(0, 0, 0);
   Init(width, data_type);
 }
 
@@ -66,6 +68,7 @@ LiteMat::LiteMat(int width, int height, LDataType data_type) {
   data_type_ = LDataType::UINT8;
   ref_count_ = nullptr;
   size_ = 0;
+  setSteps(0, 0, 0);
   Init(width, height, data_type);
 }
 
@@ -80,6 +83,7 @@ LiteMat::LiteMat(int width, int height, void *p_data, LDataType data_type) {
   data_type_ = LDataType::UINT8;
   ref_count_ = nullptr;
   size_ = 0;
+  setSteps(0, 0, 0);
   Init(width, height, p_data, data_type);
 }
 
@@ -94,6 +98,7 @@ LiteMat::LiteMat(int width, int height, int channel, LDataType data_type) {
   data_type_ = LDataType::UINT8;
   ref_count_ = nullptr;
   size_ = 0;
+  setSteps(0, 0, 0);
   Init(width, height, channel, data_type);
 }
 
@@ -108,6 +113,7 @@ LiteMat::LiteMat(int width, int height, int channel, void *p_data, LDataType dat
   data_type_ = LDataType::UINT8;
   ref_count_ = nullptr;
   size_ = 0;
+  setSteps(0, 0, 0);
   Init(width, height, channel, p_data, data_type);
 }
 
@@ -130,11 +136,18 @@ LiteMat::LiteMat(const LiteMat &m) {
   data_type_ = m.data_type_;
   ref_count_ = m.ref_count_;
   size_ = 0;
+  setSteps(m.steps_[0], m.steps_[1], m.steps_[2]);
   if (ref_count_) {
     addRef(ref_count_, 1);
   }
 }
 
+void LiteMat::setSteps(int c0, int c1, int c2) {
+  steps_[0] = c0;
+  steps_[1] = c1;
+  steps_[2] = c2;
+}
+
 LiteMat &LiteMat::operator=(const LiteMat &m) {
   if (this == &m) {
     return *this;
@@ -154,7 +167,8 @@ LiteMat &LiteMat::operator=(const LiteMat &m) {
   dims_ = m.dims_;
   data_type_ = m.data_type_;
   ref_count_ = m.ref_count_;
-  size_ = 0;
+  setSteps(m.steps_[0], m.steps_[1], m.steps_[2]);
+  size_ = m.size_;
   return *this;
 }
 
@@ -171,6 +185,7 @@ void LiteMat::Init(int width, LDataType data_type) {
   data_ptr_ = AlignMalloc(size_);
   ref_count_ = new int[1];
   *ref_count_ = 1;
+  steps_[0] = elem_size_;
 }
 
 void LiteMat::Init(int width, int height, LDataType data_type) {
@@ -186,6 +201,8 @@ void LiteMat::Init(int width, int height, LDataType data_type) {
   data_ptr_ = AlignMalloc(size_);
   ref_count_ = new int[1];
   *ref_count_ = 1;
+  steps_[1] = elem_size_;
+  steps_[0] = width_ * steps_[1];
 }
 
 void LiteMat::Init(int width, int height, void *p_data, LDataType data_type) {
@@ -199,6 +216,8 @@ void LiteMat::Init(int width, int height, void *p_data, LDataType data_type) {
   size_ = c_step_ * channel_ * elem_size_;
   data_ptr_ = p_data;
   ref_count_ = nullptr;
+  steps_[1] = elem_size_;
+  steps_[0] = width_ * steps_[1];
 }
 
 void LiteMat::Init(int width, int height, int channel, LDataType data_type) {
@@ -214,6 +233,10 @@ void LiteMat::Init(int width, int height, int channel, LDataType data_type) {
   data_ptr_ = AlignMalloc(size_);
   ref_count_ = new int[1];
   *ref_count_ = 1;
+
+  steps_[2] = elem_size_;
+  steps_[1] = channel * steps_[2];
+  steps_[0] = width_ * steps_[1];
 }
 
 void LiteMat::Init(int width, int height, int channel, void *p_data, LDataType data_type) {
@@ -227,6 +250,9 @@ void LiteMat::Init(int width, int height, int channel, void *p_data, LDataType d
   size_ = c_step_ * channel_ * elem_size_;
   data_ptr_ = p_data;
   ref_count_ = nullptr;
+  steps_[2] = elem_size_;
+  steps_[1] = channel * steps_[2];
+  steps_[0] = width_ * steps_[1];
 }
 
 bool LiteMat::IsEmpty() const { return data_ptr_ == nullptr || c_step_ * channel_ == 0; }
@@ -248,6 +274,7 @@ void LiteMat::Release() {
   c_step_ = 0;
   ref_count_ = 0;
   size_ = 0;
+  setSteps(0, 0, 0);
 }
 
 void *LiteMat::AlignMalloc(unsigned int size) {
@@ -271,6 +298,31 @@ void LiteMat::AlignFree(void *ptr) {
 
 inline void LiteMat::InitElemSize(LDataType data_type) { elem_size_ = data_type.SizeInBytes(); }
 
+bool LiteMat::GetROI(int x, int y, int w, int h, LiteMat &m) {
+  if (x < 0 || y < 0 || x + w > width_ || h + y > height_) {
+    return false;
+  }
+  if (!m.IsEmpty()) {
+    m.Release();
+  }
+
+  if (ref_count_) {
+    addRef(ref_count_, 1);
+  }
+
+  m.height_ = h;
+  m.width_ = w;
+  m.dims_ = dims_;
+  m.elem_size_ = elem_size_;
+  m.data_ptr_ = reinterpret_cast<uint8_t *>(data_ptr_) + y * steps_[0] + x * elem_size_ * channel_;
+  m.channel_ = channel_;
+  m.c_step_ = c_step_;
+  m.data_type_ = data_type_;
+  m.ref_count_ = ref_count_;
+  m.setSteps(steps_[0], steps_[1], steps_[2]);
+  return true;
+}
+
 template <typename T>
 inline void SubtractImpl(const T *src0, const T *src1, T *dst, int64_t total_size) {
   for (int64_t i = 0; i < total_size; i++) {

mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h

@@ -24,6 +24,7 @@ namespace mindspore {
 namespace dataset {
 
 #define ALIGN 16
+#define MAX_DIMS 3
 
 template <typename T>
 struct Chn1 {
@@ -121,6 +122,8 @@ enum LPixelType {
   NV122BGR = 7,
 };
 
+enum WARP_BORDER_MODE { WARP_BORDER_MODE_CONSTANT };
+
 class LDataType {
  public:
   enum Type : uint8_t {
@@ -137,6 +140,7 @@ class LDataType {
     FLOAT16,
     FLOAT32,
     FLOAT64,
+    DOUBLE,
     NUM_OF_TYPES
   };
 
@@ -179,6 +183,7 @@ class LDataType {
     2,  // FLOAT16
     4,  // FLOAT32
     8,  // FLOAT64
+    8,  // DOUBLE
   };
 
   Type type_;
@@ -213,6 +218,8 @@ class LiteMat {
 
   void Init(int width, int height, int channel, void *p_data, LDataType data_type = LDataType::UINT8);
 
+  bool GetROI(int x, int y, int w, int h, LiteMat &dst);  // NOLINT
+
   bool IsEmpty() const;
 
   void Release();
@@ -229,6 +236,14 @@ class LiteMat {
     return reinterpret_cast<const T *>(data_ptr_);
   }
 
+  template <typename T>
+  inline T *ptr(int w) const {
+    if (IsEmpty()) {
+      return nullptr;
+    }
+    return reinterpret_cast<T *>(reinterpret_cast<unsigned char *>(data_ptr_) + steps_[0] * w);
+  }
+
  private:
   /// \brief apply for memory alignment
   void *AlignMalloc(unsigned int size);
@@ -241,6 +256,8 @@ class LiteMat {
   /// \brief add reference
   int addRef(int *p, int value);
 
+  void setSteps(int c0, int c1, int c2);
+
  public:
   void *data_ptr_ = nullptr;
   int elem_size_;
@@ -252,6 +269,7 @@ class LiteMat {
   size_t size_;
   LDataType data_type_;
   int *ref_count_;
+  size_t steps_[MAX_DIMS];
 };
 
 /// \brief Calculates the difference between the two images for each element