add NMStoCharacter, change MergeCharacterToPlate

include/easypr/core/character.hpp

@@ -42,6 +42,16 @@ namespace easypr {
     inline void setCharacterScore(double param) { m_score = param; }
     inline double getCharacterScore() const { return m_score; }
 
+    bool operator < (const CCharacter& other) const
+    {
+      return (m_score > other.m_score);
+    }
+
+    bool operator < (const CCharacter& other)
+    {
+      return (m_score > other.m_score);
+    }
+
   private:
     //! character mat
 

include/easypr/core/chars_identify.h

@@ -11,10 +11,11 @@ class CharsIdentify {
  public:
   static CharsIdentify* instance();
 
-  int classify(cv::Mat f, bool isChinses = false);
+  int classify(cv::Mat f, float& maxVal, bool isChinses = false);
 
   std::pair<std::string, std::string> identify(cv::Mat input, bool isChinese = false);
 
+  bool isCharacter(cv::Mat input, std::string& label, float& maxVal, bool isChinese = false);
  private:
   CharsIdentify();
 

include/easypr/core/core_func.h

@@ -12,7 +12,7 @@ namespace easypr {
 
 enum Color { BLUE, YELLOW, WHITE, UNKNOWN };
 
-enum LocateType { SOBEL, COLOR, OTHER };
+enum LocateType { SOBEL, COLOR, CMSER, OTHER };
 
 //! 根据一幅图像与颜色模板获取对应的二值图
 //! 输入RGB图像, 颜色模板（蓝色、黄色）
@@ -56,6 +56,10 @@ int ThresholdOtsu(Mat mat);
 
 Mat ProjectedHistogram(Mat img, int t);
 
+Mat preprocessChar(Mat in, int char_size);
+
+Rect GetChineseRect(const Rect rectSpe);
+
 } /*! \namespace easypr*/
 
 #endif  // EASYPR_CORE_COREFUNC_H_

include/easypr/core/plate.hpp

@@ -18,9 +18,7 @@ namespace easypr {
 
   class CPlate {
   public:
-    bool bColored;   
-
-    CPlate();
+    CPlate() { }
 
     CPlate(const CPlate& plate)
     {

include/easypr/core/plate_judge.h

@@ -1,7 +1,7 @@
 #ifndef EASYPR_CORE_PLATEJUDGE_H_
 #define EASYPR_CORE_PLATEJUDGE_H_
 
-#include "easypr/core/plate.h"
+#include "easypr/core/plate.hpp"
 #include "easypr/core/feature.h"
 #include "easypr/core/core_func.h"
 

include/easypr/core/plate_locate.h

@@ -14,7 +14,7 @@
 #ifndef EASYPR_CORE_PLATELOCATE_H_
 #define EASYPR_CORE_PLATELOCATE_H_
 
-#include "easypr/core/plate.h"
+#include "easypr/core/plate.hpp"
 #include "easypr/core/core_func.h"
 
 /*! \namespace easypr
@@ -77,6 +77,9 @@ class CPlateLocate {
   int plateSobelLocate(Mat src, std::vector<CPlate>& candPlates, int index = 0);
   int sobelOperT(const Mat& in, Mat& out, int blurSize, int morphW, int morphH);
 
+  int plateMserLocate(Mat src, std::vector<CPlate>& candPlates, int index = 0);
+
+
   //! Color搜索
 
   int colorSearch(const Mat& src, const Color r, Mat& out,

src/core/chars_identify.cpp

@@ -19,20 +19,20 @@ namespace easypr {
     kv_->load("etc/province_mapping");
   }
 
-  int CharsIdentify::classify(cv::Mat f, bool isChinses){
+  int CharsIdentify::classify(cv::Mat f, float& maxVal, bool isChinses){
     int result = -1;
 
     cv::Mat output(1, kCharsTotalNumber, CV_32FC1);
     ann_->predict(f, output);
 
+    maxVal = -2.f;
     if (!isChinses)
     {
       result = 0;
-      float maxVal = -2;
       for (int j = 0; j < kCharactersNumber; j++)
       {
         float val = output.at<float>(j);
-        //cout << "j:" << j << "val:"<< val << endl;
+        // std::cout << "j:" << j << "val:" << val << std::endl;
         if (val > maxVal)
         {
           maxVal = val;
@@ -43,11 +43,10 @@ namespace easypr {
     else
     {
       result = kCharactersNumber;
-      float maxVal = -2;
       for (int j = kCharactersNumber; j < kCharsTotalNumber; j++)
       {
         float val = output.at<float>(j);
-        //cout << "j:" << j << "val:"<< val << endl;
+        //std::cout << "j:" << j << "val:" << val << std::endl;
         if (val > maxVal)
         {
           maxVal = val;
@@ -55,13 +54,38 @@ namespace easypr {
         }
       }
     }
+    //std::cout << "maxVal:" << maxVal << std::endl;
     return result;
   }
 
+  bool CharsIdentify::isCharacter(cv::Mat input, std::string& label, float& maxVal, bool isChinese) {
+    cv::Mat feature = features(input, kPredictSize);
+    auto index = static_cast<int>(classify(feature, maxVal, isChinese));
+
+    if (isChinese)
+      std::cout << "maxVal:" << maxVal << std::endl;
+
+    if (maxVal >= 0.9) {
+      if (index < kCharactersNumber) {
+        label = std::make_pair(kChars[index], kChars[index]).second;
+      }
+      else {
+        const char* key = kChars[index];
+        std::string s = key;
+        std::string province = kv_->get(s);
+        label = std::make_pair(s, province).second;
+      }
+      return true;
+    }
+    else
+      return false;
+  }
+
 
   std::pair<std::string, std::string> CharsIdentify::identify(cv::Mat input, bool isChinese) {
     cv::Mat feature = features(input, kPredictSize);
-    auto index = static_cast<int>(classify(feature, isChinese));
+    float maxVal = -2;
+    auto index = static_cast<int>(classify(feature, maxVal, isChinese));
     if (index < kCharactersNumber) {
       return std::make_pair(kChars[index], kChars[index]);
     }

src/core/core_func.cpp

@@ -773,4 +773,44 @@ Mat ProjectedHistogram(Mat img, int t) {
   return mhist;
 }
 
+Mat preprocessChar(Mat in, int char_size) {
+  // Remap image
+  int h = in.rows;
+  int w = in.cols;
+
+  //统一每个字符的大小
+
+  int charSize = char_size;
+
+  Mat transformMat = Mat::eye(2, 3, CV_32F);
+  int m = max(w, h);
+  transformMat.at<float>(0, 2) = float(m / 2 - w / 2);
+  transformMat.at<float>(1, 2) = float(m / 2 - h / 2);
+
+  Mat warpImage(m, m, in.type());
+  warpAffine(in, warpImage, transformMat, warpImage.size(), INTER_LINEAR,
+    BORDER_CONSTANT, Scalar(0));
+
+  //！ 将所有的字符调整成统一的尺寸
+
+  Mat out;
+  resize(warpImage, out, Size(charSize, charSize));
+
+  return out;
+}
+
+Rect GetChineseRect(const Rect rectSpe) {
+  int height = rectSpe.height;
+  float newwidth = rectSpe.width * 1.2f;
+  int x = rectSpe.x;
+  int y = rectSpe.y;
+
+  int newx = x - int(newwidth * 1.2);
+  newx = newx > 0 ? newx : 0;
+
+  Rect a(newx, y, int(newwidth), height);
+
+  return a;
+}
+
 }

src/core/plate_detect.cpp

@@ -32,8 +32,6 @@ namespace easypr {
       for (size_t i = 0; i < sobel_result_Plates.size(); i++) 
       {
         CPlate plate = sobel_result_Plates[i];
-
-        plate.bColored = false;
         plate.setPlateLocateType(SOBEL);
 
         all_result_Plates.push_back(plate);

src/core/plate_locate.cpp

@@ -764,6 +764,30 @@ int CPlateLocate::plateColorLocate(Mat src, vector<CPlate> &candPlates,
   return 0;
 }
 
+
+//! MSER plate locate
+
+int CPlateLocate::plateMserLocate(Mat src, vector<CPlate> &candPlates, int index) {
+
+  vector<CPlate> plates;
+  Mat src_b;
+
+  //// 查找蓝色车牌
+  //// 查找颜色匹配车牌
+
+  //colorSearch(src, BLUE, src_b, rects_color_blue, index);
+
+  //// 进行抗扭斜处理
+
+  //deskew(src, src_b, rects_color_blue, plates);
+
+
+  for (size_t i = 0; i < plates.size(); i++) {
+    candPlates.push_back(plates[i]);
+  }
+  return 0;
+}
+
 //! Sobel运算
 //! 输入彩色图像，输出二值化图像
 

src/core/plate_recognize.cpp

@@ -228,50 +228,61 @@ namespace easypr {
     return true;
   }
 
-  void MergeCharToPlate(const std::vector<CCharacter>& vecRect, std::vector<Rect>& vecRectPlate, 
-    int mat_width, int mat_height){
-     // recognize char
+  //! ·Ç¼«´óÖµÒÖÖÆ
+  void NMStoCharacter(std::vector<CCharacter> &inVec, std::vector<CCharacter> &resultVec, double overlap) {
+
+    std::sort(inVec.begin(), inVec.end());
+
+    std::vector<CCharacter>::iterator it = inVec.begin();
+    for (; it != inVec.end(); ++it) {
+      CCharacter charSrc = *it;
+      //std::cout << "plateScore:" << plateSrc.getPlateScore() << std::endl;
+      Rect rectSrc = charSrc.getCharacterPos();
+
+      std::vector<CCharacter>::iterator itc = it + 1;
+
+      for (; itc != inVec.end();) {
+        CCharacter charComp = *itc;
+        Rect rectComp = charComp.getCharacterPos();
+        Rect rectInter = rectSrc & rectComp;
+        Rect rectUnion = rectSrc | rectComp;
+        double r = double(rectInter.area()) / double(rectUnion.area());
+        if (r > overlap) {
+          itc = inVec.erase(itc);
+        }
+        else {
+          ++itc;
+        }
+      }
+    }
+
+    resultVec = inVec;
+  }
 
+
+  void MergeCharToPlate(std::vector<CCharacter>& vecRect, std::vector<std::vector<CCharacter>>& vecRectPlate){
      std::vector<int> labels;
 
-     int numbers = partition(vecRect, labels, &compareCharRect);
+     std::vector<CCharacter> charVec;
+
+     double overlap = 0.5;
+     NMStoCharacter(vecRect, charVec, overlap);
+
+     int numbers = partition(charVec, labels, &compareCharRect);
      for (size_t j = 0; j < size_t(numbers); j++) {
-        std::vector<Rect> charRects;
+       std::vector<CCharacter> charRects;
 
-        for (size_t t = 0; t < vecRect.size(); t++) {
+       for (size_t t = 0; t < charVec.size(); t++) {
           int label = labels[t];
 
           if (label == j)
-            charRects.push_back(vecRect[t].getCharacterPos());
+            charRects.push_back(charVec[t]);
         }
 
         if (charRects.size() == 0 || charRects.size() < 5)
           continue;
 
-       int leftmax = mat_width;
-       int rightmax = 0;
-       int topmax = mat_height;
-       int bottommax = 0;
-
-       for (size_t p = 0; p < charRects.size(); p++) {
-         Rect rect = charRects[p];
-         int left = rect.tl().x;
-         int right = rect.br().x;
-
-         int top = rect.tl().y;
-         int bottom = rect.br().y;
-
-         if (left < leftmax) leftmax = left;
-         if (right > rightmax) rightmax = right;
-         if (top < topmax) topmax = top;
-         if (bottom > bottommax) bottommax = bottom;
-       }
-
-       Rect grouprect(leftmax, topmax, rightmax - leftmax + 1, bottommax - topmax + 1);
-
-       if (verifyPlateSize(grouprect)) {
-         vecRectPlate.push_back(grouprect);
-       }
+        vecRectPlate.push_back(charRects);       
      }
   }
 
@@ -284,6 +295,8 @@ namespace easypr {
     std::vector<Mat> channelImages;
     std::vector<int> flags;
 
+
+    // only conside blue plate
     if (1) {
       Mat grayImage;
       cvtColor(src, grayImage, COLOR_BGR2GRAY);
@@ -314,6 +327,7 @@ namespace easypr {
       double scale_ratio = 1;
       Mat image = scaleImage(channelImage, Size(scale_size, scale_size), scale_ratio);
       Mat result = image;
+      cvtColor(result, result, COLOR_GRAY2BGR);
 
       int imageArea = image.rows * image.cols;
 
@@ -329,29 +343,107 @@ namespace easypr {
 
       for (size_t index = 0; index < size; index++) {
         Rect rect = all_boxes[i][index];
-
-        Mat region = image(rect);
-        Mat binary_region;
-        threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
-
+    
         if (verifyCharSizes(rect)) {
           float aspect = float(rect.width) / float(rect.height);
           //if (aspect < 0.2) {
           //  cv::rectangle(result, rect, Scalar(0, 0, 255));
           //}
 
-          CCharacter character;
-          character.setCharacterPos(rect);
-          character.setCharacterMat(binary_region);
-          charVec.push_back(character);
+          Mat region = image(rect);
+          Mat binary_region;
+          threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
+
+          Mat charInput = preprocessChar(binary_region, 20);
+          std::string label = "";
+          float maxVal = -2.f;
+          bool isCharacter = CharsIdentify::instance()->isCharacter(charInput, label, maxVal);
+
+          if (isCharacter) {
+            cv::rectangle(result, rect, Scalar(255, 0, 0));
+
+            CCharacter character;
+            character.setCharacterPos(rect);
+            character.setCharacterMat(binary_region);
+            character.setCharacterStr(label);
+            character.setCharacterScore(maxVal);
+            charVec.push_back(character);
+          }    
         }
       }
 
-      std::vector<Rect> vecRectPlate;
-      MergeCharToPlate(charVec, vecRectPlate, image.rows, image.cols);
+      std::vector<std::vector<CCharacter>> vecRectPlate;
+      MergeCharToPlate(charVec, vecRectPlate);
 
       for (size_t plate_i = 0; plate_i < vecRectPlate.size(); plate_i++) {
-        cv::rectangle(result, vecRectPlate[plate_i], Scalar(0, 0, 255));
+        std::vector<CCharacter> chars = vecRectPlate[plate_i];
+
+        std::sort(chars.begin(), chars.end(),
+          [](const CCharacter& r1, const CCharacter& r2) { return r1.getCharacterPos().x < r2.getCharacterPos().x; });
+
+        Rect specificRect = chars[0].getCharacterPos();
+        Rect chineseRect = GetChineseRect(specificRect);
+
+        cv::rectangle(result, chineseRect, Scalar(0, 255, 0));
+
+        Mat region = image(chineseRect);
+        Mat binary_region;
+        threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
+
+        Mat charInput = preprocessChar(binary_region, 20);
+        std::string label = "";
+        float maxVal = -2.f;
+        bool isCharacter = CharsIdentify::instance()->isCharacter(charInput, label, maxVal, true);
+
+        if (isCharacter) {
+          CCharacter character;
+          character.setCharacterPos(chineseRect);
+          character.setCharacterMat(binary_region);
+          character.setCharacterStr(label);
+          character.setCharacterScore(maxVal);
+          chars.push_back(character);
+        }
+
+        int leftmax = image.rows;
+        int rightmax = 0;
+        int topmax = image.cols;
+        int bottommax = 0;
+
+        std::string plateLicense = "";
+
+        for (size_t p = 0; p < chars.size(); p++) {
+          Rect rect = chars[p].getCharacterPos();
+          int left = rect.tl().x;
+          int right = rect.br().x;
+
+          int top = rect.tl().y;
+          int bottom = rect.br().y;
+
+          if (left < leftmax) leftmax = left;
+          if (right > rightmax) rightmax = right;
+          if (top < topmax) topmax = top;
+          if (bottom > bottommax) bottommax = bottom;
+
+          if (p != chars.size() - 1)
+            plateLicense.append(chars[p].getCharacterStr());
+        }
+
+        Rect grouprect(leftmax, topmax, rightmax - leftmax + 1, bottommax - topmax + 1); 
+        Point2f center(float(leftmax + grouprect.width / 2), float(topmax + grouprect.height / 2));
+
+        plateLicense = label + plateLicense;
+
+          if (verifyPlateSize(grouprect)) {
+            
+            cv::rectangle(result, grouprect, Scalar(0, 0, 255));
+
+            CPlate plate;
+            plate.setPlateLocateType(LocateType::CMSER);
+            plate.setPlatePos(RotatedRect(center, Size2f(float(grouprect.width), float(grouprect.height)), 0));
+            plate.setPlateMat(image(grouprect));
+            plate.setPlateStr(plateLicense);
+            licenseVec.push_back(plate);
+          }
       }
 
       if (1) {

test/accuracy.hpp

@@ -41,7 +41,7 @@ namespace easypr {
 
           string plateStr = plateNode.getText();
 
-          RotatedRect rr(Point2f(x, y), Size2f(width, height), angle);
+          RotatedRect rr(Point2f(float(x), float(y)), Size2f(float(width), float(height)), angle);
 
           CPlate plate;
           plate.setPlateStr(plateStr);

vcprojs/libeasypr.vcxproj

@@ -151,7 +151,6 @@
     <ClCompile Include="..\src\core\chars_segment.cpp" />
     <ClCompile Include="..\src\core\core_func.cpp" />
     <ClCompile Include="..\src\core\feature.cpp" />
-    <ClCompile Include="..\src\core\plate.cpp" />
     <ClCompile Include="..\src\core\plate_detect.cpp" />
     <ClCompile Include="..\src\core\plate_judge.cpp" />
     <ClCompile Include="..\src\core\plate_locate.cpp" />
@@ -177,7 +176,7 @@
     <ClInclude Include="..\include\easypr\core\chars_segment.h" />
     <ClInclude Include="..\include\easypr\core\core_func.h" />
     <ClInclude Include="..\include\easypr\core\feature.h" />
-    <ClInclude Include="..\include\easypr\core\plate.h" />
+    <ClInclude Include="..\include\easypr\core\plate.hpp" />
     <ClInclude Include="..\include\easypr\core\plate_detect.h" />
     <ClInclude Include="..\include\easypr\core\plate_judge.h" />
     <ClInclude Include="..\include\easypr\core\plate_locate.h" />

vcprojs/libeasypr.vcxproj.filters

@@ -69,9 +69,6 @@
     <ClCompile Include="..\src\core\feature.cpp">
       <Filter>源文件\core</Filter>
     </ClCompile>
-    <ClCompile Include="..\src\core\plate.cpp">
-      <Filter>源文件\core</Filter>
-    </ClCompile>
     <ClCompile Include="..\src\core\plate_detect.cpp">
       <Filter>源文件\core</Filter>
     </ClCompile>
@@ -161,9 +158,6 @@
     <ClInclude Include="..\include\easypr\core\feature.h">
       <Filter>头文件\easypr\core</Filter>
     </ClInclude>
-    <ClInclude Include="..\include\easypr\core\plate.h">
-      <Filter>头文件\easypr\core</Filter>
-    </ClInclude>
     <ClInclude Include="..\include\easypr\core\plate_detect.h">
       <Filter>头文件\easypr\core</Filter>
     </ClInclude>
@@ -191,6 +185,9 @@
     <ClInclude Include="..\include\easypr\core\character.hpp">
       <Filter>头文件\easypr\core</Filter>
     </ClInclude>
+    <ClInclude Include="..\include\easypr\core\plate.hpp">
+      <Filter>头文件\easypr\core</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\LICENSE" />