add search for weak seed and slide window for chinese

accuracy.txt

@@ -1732,3 +1732,18 @@ Recall:0%, Precise:0%, Fscore:0%.
 Recall:0%, Precise:0%, Fscore:0%.
 平均字符差距:0,  完全匹配数:0,  完全匹配率:0%  
 总时间:170秒,  平均执行时间:6.8秒
+2016-06-22 17:03:56
+总图片数:25,  未识出图片:25,  定位率:0%  
+Recall:0%, Precise:0%, Fscore:0%.
+平均字符差距:0,  完全匹配数:0,  完全匹配率:0%  
+总时间:240秒,  平均执行时间:9.6秒
+2016-06-22 17:19:36
+总图片数:25,  未识出图片:25,  定位率:0%  
+Recall:0%, Precise:0%, Fscore:0%.
+平均字符差距:0,  完全匹配数:0,  完全匹配率:0%  
+总时间:319秒,  平均执行时间:12.76秒
+2016-06-22 20:48:19
+总图片数:200,  未识出图片:6,  定位率:97%  
+Recall:83.2984%, Precise:86.2716%, Fscore:84.7589%.
+平均字符差距:0.845361,  完全匹配数:127,  完全匹配率:65.4639%  
+总时间:218秒,  平均执行时间:1.09秒

include/easypr/core/character.hpp

@@ -52,13 +52,10 @@ namespace easypr {
     inline void setIsChinese(bool param) { m_isChinese = param; }
     inline bool getIsChinese() const { return m_isChinese; }
 
-    //inline void setIsStrong(bool param) { isStrong = param; }
     inline bool getIsStrong() const { return m_score >= 0.9; }
 
-    //inline void setIsWeak(bool param) { isWeak = param; }
     inline bool getIsWeak() const { return m_score < 0.9 && m_score >= 0.5; }
 
-    //inline void setIsLittle(bool param) { isLittle = param; }
     inline bool getIsLittle() const { return m_score < 0.5; }
 
     bool operator < (const CCharacter& other) const

include/easypr/core/core_func.h

@@ -81,7 +81,7 @@ Mat scaleImage(const Mat& image, const Size& maxSize, double& scale_ratio);
 RotatedRect scaleBackRRect(const RotatedRect& rr, const float scale_ratio);
 
 //! use verify size to first generate char candidates
-Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect);
+Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect, Color color, int index = 0, bool showDebug = false);
 
 /** @brief convert form mser point to image.
 

include/easypr/core/plate.hpp

@@ -28,6 +28,9 @@ namespace easypr {
       m_platePos = plate.m_platePos;
       m_plateStr = plate.m_plateStr;
       m_locateType = plate.m_locateType;
+      m_line = plate.m_line;
+      m_leftPoint = plate.m_leftPoint;
+      m_rightPoint = plate.m_rightPoint;
     }
 
     inline void setPlateMat(Mat param) { m_plateMat = param; }
@@ -45,6 +48,15 @@ namespace easypr {
     inline void setPlateScore(double param) { m_score = param; }
     inline double getPlateScore() const { return m_score; }
 
+    inline void setPlateLine(Vec4f param) { m_line = param; }
+    inline Vec4f getPlateLine() const { return m_line; }
+
+    inline void setPlateLeftPoint(Point param) { m_leftPoint = param; }
+    inline Point getPlateLeftPoint() const { return m_leftPoint; }
+
+    inline void setPlateRightPoint(Point param) { m_rightPoint = param; }
+    inline Point getPlateRightPoint() const { return m_rightPoint; }
+
     bool operator < (const CPlate& plate) const
     {
       return (m_score < plate.m_score);
@@ -57,25 +69,27 @@ namespace easypr {
 
   private:
     //! plate mat
-
     Mat m_plateMat;
 
     //! plate rect
-
     RotatedRect m_platePos;
 
     //! plate license
-
     String m_plateStr;
 
     //! plate locate type
-
     LocateType m_locateType;
 
     //! plate likely
-
     double m_score;
 
+    //! middle line
+    Vec4f m_line;
+
+    //! left point and right point;
+    Point m_leftPoint;
+    Point m_rightPoint;
+
   };
 
 } /*! \namespace easypr*/

include/easypr/core/plate_locate.h

@@ -82,7 +82,7 @@ class CPlateLocate {
 
   //! mser search
   int mserSearch(const Mat& src, const Color r, Mat& out, 
-    std::vector<RotatedRect>& outRects, int index = 0);
+    std::vector<RotatedRect>& outRects, int index = 0, bool showDebug = false);
 
   //! 未使用函数与代码
   //! 开始------------

src/core/chars_identify.cpp

@@ -68,6 +68,9 @@ namespace easypr {
   void CharsIdentify::classify(std::vector<CCharacter>& charVec){
     size_t charVecSize = charVec.size();
 
+    if (charVecSize == 0)
+      return;
+
     Mat featureRows;
     for (size_t index = 0; index < charVecSize; index++) {
       Mat charInput = charVec[index].getCharacterMat();
@@ -158,10 +161,11 @@ namespace easypr {
     cv::Mat feature = charFeatures(input, kPredictSize);
     auto index = static_cast<int>(classify(feature, maxVal, isChinese));
 
-    if (isChinese)
-      std::cout << "maxVal:" << maxVal << std::endl;
+    if (isChinese) {
+      //std::cout << "maxVal:" << maxVal << std::endl;
+    }
 
-    if (maxVal >= 0.9) {
+    if (maxVal >= 0.9 || (isChinese && maxVal >= 0.25)) {
       if (index < kCharactersNumber) {
         label = std::make_pair(kChars[index], kChars[index]).second;
       }

src/core/core_func.cpp

@@ -1065,7 +1065,7 @@ void mergeCharToGroup(std::vector<CCharacter> vecRect,
 }
 
 //! use verify size to first generate char candidates
-Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect) {
+Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect, Color color, int img_index, bool showDebug) {
   Mat image = src;
 
   std::vector<std::vector<Point>> all_contours;
@@ -1102,7 +1102,7 @@ Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect) {
     if (verifyCharSizes(rect)) {
       Mat mserMat = adaptive_image_from_points(contour, rect, Size(char_size, char_size));
       Mat charInput = preprocessChar(mserMat, char_size);
-      
+
       CCharacter charCandidate;
       charCandidate.setCharacterPos(rect);
       charCandidate.setCharacterMat(charInput);
@@ -1118,9 +1118,9 @@ Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect) {
   std::vector<CCharacter> littleSeedVec;
 
   size_t charCan_size = charVec.size();
-  for (size_t index = 0; index < charCan_size; index++) {
-    CCharacter& charCandidate = charVec[index];
-    Rect postion = charCandidate.getCharacterPos();
+  for (size_t char_index = 0; char_index < charCan_size; char_index++) {
+    CCharacter& charCandidate = charVec[char_index];
+    Rect rect = charCandidate.getCharacterPos();
     double score = charCandidate.getCharacterScore();
     if (charCandidate.getIsStrong()) {
       strongSeedVec.push_back(charCandidate);
@@ -1131,7 +1131,6 @@ Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect) {
     else if (charCandidate.getIsLittle()) {
       littleSeedVec.push_back(charCandidate);
     }
-
   }
 
   //nms
@@ -1144,29 +1143,179 @@ Mat mserCharMatch(const Mat &src, Mat &match, std::vector<Rect>& out_charRect) {
   mergeCharToGroup(nmsStrongSeedVec, charGroupVec);
 
   //draw the line of the group
+  std::vector<Vec4f> lineVec;
   for (auto charGroup : charGroupVec) {
+    Rect plateResult = charGroup[0].getCharacterPos();
     std::vector<Point> points;
     Vec4f line;
     int maxarea = 0;
     Rect maxrect;
+    
+    int leftx = image.cols;
+    Point leftPoint(leftx, 0);
 
+    std::vector<CCharacter> mserCharacter;
     for (auto character : charGroup) {
       Rect charRect = character.getCharacterPos();
+      plateResult |= charRect;
       Point center(charRect.tl().x + charRect.width / 2, charRect.tl().y + charRect.height / 2);
       points.push_back(center);
-
-      cv::circle(result, center, 3, Scalar(0, 255, 0), 2);
+      mserCharacter.push_back(character);
+      //cv::circle(result, center, 3, Scalar(0, 255, 0), 2);
       if (charRect.area() > maxarea) {
         maxrect = charRect;
         maxarea = charRect.area();
       }
+      if (center.x < leftPoint.x) {
+        leftPoint = center;
+      }
     }
 
     if (points.size() >= 2) {
       fitLine(Mat(points), line, CV_DIST_L2, 0, 0.01, 0.01);
+      lineVec.push_back(line);
+
       float k = line[1] / line[0];
+      std::cout << "k:" << k << std::endl;
+
       float step = 10.f * (float)maxrect.width;
       cv::line(result, Point2f(line[2] - step, line[3] - k*step), Point2f(line[2] + step, k*step + line[3]), Scalar(255, 255, 255));
+      //cv::circle(result, leftPoint, 3, Scalar(0, 0, 255), 2);
+
+      float x_1 = line[2];
+      float y_1 = line[3];
+
+      std::vector<CCharacter> searchWeakSeedVec;
+      int weakLeftx = leftPoint.x;
+      Point weakLeftPoint(weakLeftx, 0);
+
+      //draw weak seed and little seed from line;
+      //search for mser rect
+      std::cout << "search for mser rect:" << std::endl;
+      if (0) {
+        std::stringstream ss(std::stringstream::in | std::stringstream::out);
+        ss << "resources/image/tmp/" << img_index << "_1_" << "searcgMserRect.jpg";
+        imwrite(ss.str(), result);
+      }
+      
+      for (auto weakSeed : charVec) {
+        Rect weakRect = weakSeed.getCharacterPos();
+        Point weakCenter(weakRect.tl().x + weakRect.width / 2, weakRect.tl().y + weakRect.height / 2);
+
+        float x_2 = (float)weakCenter.x;
+
+        // point in line;
+        float y_2l = k * (x_2 - x_1) + y_1;
+        float y_2 = (float)weakCenter.y;
+
+        float y_diff_ratio = abs(y_2l - y_2) / maxrect.height;
+
+        //std::cout << "weakRect y_diff_ratio:" << y_diff_ratio << std::endl;
+        if (y_diff_ratio < 0.1 && x_2 < leftPoint.x) {
+
+          float width_1 = float(maxrect.width);
+          float height_1 = float(maxrect.height);
+
+          float width_2 = float(weakRect.width);
+          float height_2 = float(weakRect.height);
+
+          float height_diff = abs(height_1 - height_2);
+          double height_diff_ratio = height_diff / min(height_1, height_2);
+
+          if (height_diff_ratio < 0.25) {
+            //cv::rectangle(result, weakRect, Scalar(0, 255, 0));
+            searchWeakSeedVec.push_back(weakSeed);
+            mserCharacter.push_back(weakSeed);
+            plateResult |= weakRect;
+            if (x_2 < weakLeftx) {
+              weakLeftx = (int)x_2;
+            }
+          }         
+        }
+      }
+
+
+      std::cout << "judege the left is chinese:" << std::endl;
+      bool leftIsChinese = false;
+      if (1) {
+        std::sort(mserCharacter.begin(), mserCharacter.end(),
+          [](const CCharacter& r1, const CCharacter& r2) {
+          return r1.getCharacterPos().tl().x < r2.getCharacterPos().tl().x;
+        });
+
+        CCharacter leftChar = mserCharacter[0];
+        Rect theRect = leftChar.getCharacterPos();
+        cv::rectangle(result, theRect, Scalar(255, 0, 0), 1);
+
+        Mat region = image(theRect);
+        Mat binary_region;
+        threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); 
+
+        Mat charInput = preprocessChar(binary_region, 20);
+        if (0) {
+          imshow("charInput", charInput);
+          waitKey(0);
+          destroyWindow("charInput");
+        }
+
+        std::string label = "";
+        float maxVal = -2.f;
+        leftIsChinese = CharsIdentify::instance()->isCharacter(charInput, label, maxVal, true);
+        std::cout << "isChinese:" << leftIsChinese << std::endl;
+        std::cout << "chinese:" << label << std::endl;
+        std::cout << " score:" << maxVal << std::endl;
+      }
+      
+
+      //search for sliding window
+      if (!leftIsChinese) {
+        std::cout << "search for sliding window:" << std::endl;
+        int slideLength = maxrect.width;
+        int slideStep = 3;
+        int fromX = weakLeftx - maxrect.width;
+        std::vector<CCharacter> chineseCandidate;
+        for (int slideX = 0; slideX < slideLength; slideX += slideStep) {
+          float x_slide = float(fromX - slideX);
+          float y_slide = k * (x_slide - x_1) + y_1;
+          Point2f p_slide(x_slide, y_slide);
+          //cv::circle(result, p_slide, 2, Scalar(0, 0, 255), 1);
+          Rect rect(Point2f(x_slide - maxrect.width / 2, y_slide - maxrect.height / 2), maxrect.size());
+
+          if (rect.tl().x < 0 || rect.tl().y < 0 || rect.br().x >= image.cols || rect.br().y >= image.rows)
+            continue;
+
+          Mat region = image(rect);
+          Mat binary_region;
+
+          threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
+
+          Mat charInput = preprocessChar(binary_region, 20);
+
+          CCharacter charCandidate;
+          charCandidate.setCharacterPos(rect);
+          charCandidate.setCharacterMat(charInput);
+          charCandidate.setIsChinese(true);
+          chineseCandidate.push_back(charCandidate);
+        }
+
+        CharsIdentify::instance()->classify(chineseCandidate);
+        std::vector<CCharacter> nmsChineseVec;
+        double overlapThresh = 0.1;
+        NMStoCharacter(chineseCandidate, nmsChineseVec, overlapThresh);
+
+        for (auto chinese : nmsChineseVec) {
+          Rect rect = chinese.getCharacterPos();
+          if (chinese.getCharacterScore() > 0.5) {
+            cv::rectangle(result, rect, Scalar(0, 0, 255), 1);
+            plateResult |= rect;
+            std::cout << "chinese:" << chinese.getCharacterStr();
+            std::cout << "__score:" << chinese.getCharacterScore() << std::endl;
+          }
+        }
+      }
+      
+      cv::rectangle(result, plateResult, Scalar(0, 0, 255), 1);
+      match(plateResult) = 255;
     }
   }
 

src/core/plate_locate.cpp

@@ -81,8 +81,8 @@ bool CPlateLocate::verifySizes(RotatedRect mr) {
 
 
 // !基于HSV空间的颜色搜索方法
-int CPlateLocate::mserSearch(const Mat &src, const Color r, Mat &out,
-  vector<RotatedRect> &outRects, int index) {
+int CPlateLocate::mserSearch(const Mat &src, const Color color, Mat &out,
+  vector<RotatedRect> &outRects, int index, bool showDebug) {
   Mat match_grey;
   Mat result = src.clone();
   cvtColor(result, result, COLOR_GRAY2BGR);
@@ -99,7 +99,7 @@ int CPlateLocate::mserSearch(const Mat &src, const Color r, Mat &out,
   // 进行颜色查找
 
   //mserMatch(src, match_grey, r, plateRects, charRects);
-  mserCharMatch(src, match_grey, charRects);
+  mserCharMatch(src, match_grey, charRects, color, index, showDebug);
 
   if (m_debug) {
     utils::imwrite("resources/image/tmp/match_grey.jpg", match_grey);
@@ -960,14 +960,14 @@ int CPlateLocate::plateColorLocate(Mat src, vector<CPlate> &candPlates,
 
 int CPlateLocate::plateMserLocate(Mat src, vector<CPlate> &candPlates, int index) {
   std::vector<Mat> channelImages;
-  std::vector<int> flags;
+  std::vector<Color> flags;
 
   // only conside blue plate
   if (1) {
     Mat grayImage;
     cvtColor(src, grayImage, COLOR_BGR2GRAY);
     channelImages.push_back(grayImage);
-    flags.push_back(0);
+    flags.push_back(BLUE);
 
     //Mat singleChannelImage;
     //extractChannel(src, singleChannelImage, 2);
@@ -975,7 +975,7 @@ int CPlateLocate::plateMserLocate(Mat src, vector<CPlate> &candPlates, int index
     //flags.push_back(0);
 
     channelImages.push_back(255 - grayImage);
-    flags.push_back(1);
+    flags.push_back(YELLOW);
   }
 
   int scale_size = 1024;
@@ -990,7 +990,7 @@ int CPlateLocate::plateMserLocate(Mat src, vector<CPlate> &candPlates, int index
     Mat image = scaleImage(channelImage, Size(scale_size, scale_size), scale_ratio);
 
     vector<RotatedRect> rects;
-    mserSearch(image, BLUE, src_b, rects, index);
+    mserSearch(image, flags[i], src_b, rects, index);
 
     for (size_t j = 0; j < rects.size(); ++j) {
       RotatedRect mserRect = scaleBackRRect(rects[j], (float)scale_ratio);

test/accuracy.hpp

@@ -97,8 +97,8 @@ namespace easypr {
       // 设置要处理的一张图片中最多有多少车牌
 
       pr.setMaxPlates(4);
-      //pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_SOBEL);
-      pr.setDetectType(PR_DETECT_CMSER);
+      pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_SOBEL);
+      //pr.setDetectType(PR_DETECT_CMSER);
 
       //CPlateDetect pd;
       //pd.setDetectType(PR_DETECT_CMSER);