* change MSER segment

.gitignore

@@ -17,7 +17,7 @@ resources/image/native_test/*.py
 resources/image/phone_test
 *.vsp
 *.psess
-*.DStore
+*.DS_Store
 #ignore thumbnails created by windows
 Thumbs.db
 #Ignore files build by Visual Studio

include/easypr/api.hpp

@@ -44,21 +44,26 @@ static std::vector<std::string> plate_recognize(const char* image,
   assert(!img.empty());
 
   CPlateRecognize pr;
-  pr.setResultShow(true);
+  pr.setResultShow(false);
   pr.setLifemode(true);
   pr.setMaxPlates(1);
   pr.setDetectType(PR_DETECT_CMSER | PR_DETECT_COLOR);
 
-  pr.LoadSVM("resources/model/svm.xml");
-  pr.LoadANN("resources/model/ann.xml");
-  pr.LoadChineseANN("resources/model/ann_chinese.xml");
-
   std::vector<std::string> results;
   std::vector<CPlate> plates;
   pr.plateRecognize(img, plates, 0);
 
   for (auto plate : plates) {
     results.push_back(plate.getPlateStr());
+
+  }
+
+  if (plates.size() == 1) {
+    if (1) {
+      std::stringstream ss(std::stringstream::in | std::stringstream::out);
+      ss << "result.jpg";
+      imwrite(ss.str(), plates.at(0).getPlateMat());
+    }
   }
 
   return std::move(results);

include/easypr/core/chars_identify.h

@@ -11,12 +11,12 @@
 namespace easypr {
 
 class CharsIdentify {
- public:
+public:
   static CharsIdentify* instance();
 
   int classify(cv::Mat f, float& maxVal, bool isChinses = false, bool isAlphabet = false);
   void classify(cv::Mat featureRows, std::vector<int>& out_maxIndexs,
-    std::vector<float>& out_maxVals, std::vector<bool> isChineseVec);
+                std::vector<float>& out_maxVals, std::vector<bool> isChineseVec);
   void classify(std::vector<CCharacter>& charVec);
 
   void classifyChinese(std::vector<CCharacter>& charVec);
@@ -24,7 +24,7 @@ class CharsIdentify {
 
   std::pair<std::string, std::string> identify(cv::Mat input, bool isChinese = false, bool isAlphabet = false);
   int identify(std::vector<cv::Mat> inputs, std::vector<std::pair<std::string, std::string>>& outputs,
-    std::vector<bool> isChineseVec);
+               std::vector<bool> isChineseVec);
 
   std::pair<std::string, std::string> identifyChinese(cv::Mat input, float& result, bool& isChinese);
   std::pair<std::string, std::string> identifyChineseGray(cv::Mat input, float& result, bool& isChinese);
@@ -35,7 +35,7 @@ class CharsIdentify {
   void LoadChineseModel(std::string path);
   void LoadGrayModel(std::string path);
 
- private:
+private:
   CharsIdentify();
   annCallback extractFeature;
   static CharsIdentify* instance_;

include/easypr/core/core_func.h

@@ -94,7 +94,7 @@ Mat adaptive_image_from_points(const std::vector<Point>& points,
   const Scalar& forgroundColor = Scalar(255, 255, 255), bool gray = true);
 
 // Calculate a rect have same length and width and remains the center
-Rect adaptive_charrect_from_rect(const Rect& rect, int maxwidth, int maxheight);
+Rect adaptive_charrect_from_rect(const Rect& rect, int maxwidth, int maxheight, bool useExtendHeight = false);
 
 // calc safe rect
 bool calcSafeRect(const RotatedRect& roi_rect, const Mat& src,
@@ -117,8 +117,12 @@ Rect rectEnlarge(const Rect& src, const int mat_width, const int mat_height);
 // write images to temp folder
 void writeTempImage(const Mat& outImg, const string path);
 
-// remove the small lines in character like "zh-cuan"
-bool judegMDOratio2(const Mat &image, const Rect &rect, std::vector<Point> &contour, Mat &result);
+// remove small hor lines in the plate
+bool judegMDOratio2(const Mat &image, const Rect &rect, std::vector<Point> &contour, Mat &result, const float thresh = 1.f,
+                    bool useExtendHeight = false);
+
+// clear top and bottom borders
+void clearBorder(const Mat &img, Rect& cropRect);
 
 //! non-maximum surpresion for 1d array
 template<typename T>

result/accuracy.txt

@@ -2698,3 +2698,13 @@ Recall:88.4994%, Precise:66.935%, Fscore:76.2213%.
 Recall:84.5843%, Precise:79.6612%, Fscore:82.049%.
 0-error:62.963%,  1-error:74.0741%,  Chinese-precise:78.1893%  
 总共时间: 186秒,  平均执行时间:0.726563秒
+2017-04-16 21:04:06
+总图片数:1,  Plates count:1,  未定位车牌:0,  检出率:100%  
+Recall:88.2833%, Precise:88.2833%, Fscore:88.2833%.
+0-error:0%,  1-error:100%,  Chinese-precise:100%  
+总共时间: 9秒,  平均执行时间:9秒
+2017-04-16 21:04:59
+总图片数:1,  Plates count:1,  未定位车牌:0,  检出率:100%  
+Recall:89.4983%, Precise:89.4983%, Fscore:89.4983%.
+0-error:100%,  1-error:100%,  Chinese-precise:100%  
+总共时间: 14秒,  平均执行时间:14秒

src/core/chars_recognise.cpp

@@ -24,7 +24,7 @@ int CCharsRecognise::charsRecognise(Mat plate, std::string& plateLicense) {
         isChinses = true;
         auto character = CharsIdentify::instance()->identifyChinese(charMat, maxVal, judge);
         plateLicense.append(character.second);
-      } 
+      }
       else {
         isChinses = false;
         auto character = CharsIdentify::instance()->identify(charMat, isChinses);
@@ -45,7 +45,7 @@ int CCharsRecognise::charsRecognise(CPlate& plate, std::string& plateLicense) {
   std::vector<Mat> matChars;
   std::vector<Mat> grayChars;
   Mat plateMat = plate.getPlateMat();
-  if (1) writeTempImage(plateMat, "plateMat/plate");
+  if (0) writeTempImage(plateMat, "plateMat/plate");
   Color color;
   if (plate.getPlateLocateType() == CMSER) {
     color = plate.getPlateColor();
@@ -56,15 +56,15 @@ int CCharsRecognise::charsRecognise(CPlate& plate, std::string& plateLicense) {
     Mat tmpMat = plateMat(Rect_<double>(w * 0.1, h * 0.1, w * 0.8, h * 0.8));
     color = getPlateType(tmpMat, true);
   }
-  
-  int result = m_charsSegment->charsSegmentUsingProject(plateMat, matChars, grayChars, color);
+
+  int result = m_charsSegment->charsSegmentUsingMSER(plateMat, matChars, grayChars, color);
 
   if (result == 0) {
     int num = matChars.size();
     for (int j = 0; j < num; j++)
     {
       Mat charMat = matChars.at(j);
-      bool isChinses = false;   
+      bool isChinses = false;
 
       std::pair<std::string, std::string> character;
       float maxVal;
@@ -72,7 +72,6 @@ int CCharsRecognise::charsRecognise(CPlate& plate, std::string& plateLicense) {
         isChinses = true;
         bool judge = true;
         Mat grayChar = grayChars.at(j);
-        //character = CharsIdentify::instance()->identifyChinese(charMat, maxVal, judge);
         if (color != Color::BLUE)
           grayChar = 255 - grayChar;
         character = CharsIdentify::instance()->identifyChineseGray(grayChar, maxVal, judge);
@@ -89,8 +88,9 @@ int CCharsRecognise::charsRecognise(CPlate& plate, std::string& plateLicense) {
         character = CharsIdentify::instance()->identify(charMat, isChinses, isAbc);
         plateLicense.append(character.second);
       }
-      else {     
+      else {
         isChinses = false;
+        SHOW_IMAGE(charMat, 0);
         character = CharsIdentify::instance()->identify(charMat, isChinses);
         plateLicense.append(character.second);
       }
@@ -105,7 +105,7 @@ int CCharsRecognise::charsRecognise(CPlate& plate, std::string& plateLicense) {
       return -1;
     }
   }
-  
+
   return result;
 }
 

src/core/core_func.cpp

@@ -407,6 +407,51 @@ namespace easypr {
     return true;
   }
 
+
+void clearBorder(const Mat &img, Rect& cropRect) {
+  int r = img.rows;
+  int c = img.cols;
+  Mat boder = Mat::zeros(1, r, CV_8UC1);
+  const int noJunpCount_thresh = int(0.15f * c);
+
+  // if nojumpcount >
+  for (int i = 0; i < r; i++) {
+    int nojumpCount = 0;
+    int isBorder = 0;
+    for (int j = 0; j < c - 1; j++) {
+      if (img.at<char>(i, j) == img.at<char>(i, j + 1))
+        nojumpCount++;
+      if (nojumpCount > noJunpCount_thresh) {
+        nojumpCount = 0;
+        isBorder = 1;
+        break;
+      }
+    }
+    boder.at<char>(i) = (char) isBorder;
+  }
+
+  const int mintop = int(0.1f * r);
+  const int maxtop = int(0.9f * r);
+
+  int minMatTop = 0;
+  int maxMatTop = r - 1;
+
+  for (int i = 0; i < mintop; i++) {
+    if (boder.at<char>(i) == 1) {
+      minMatTop = i;
+    }
+  }
+
+  for (int i = r - 1; i > maxtop; i--) {
+    if (boder.at<char>(i) == 1) {
+      maxMatTop = i;
+    }
+  }
+
+  cropRect = Rect(0, minMatTop, c, maxMatTop - minMatTop + 1);
+
+}
+
   void clearLiuDing(Mat mask, int &top, int &bottom) {
     const int x = 7;
 
@@ -1217,23 +1262,31 @@ namespace easypr {
   }
 
 
-  bool judegMDOratio2(const Mat &image, const Rect &rect, std::vector<Point> &contour, Mat &result) {
+  bool judegMDOratio2(const Mat &image, const Rect &rect, std::vector<Point> &contour, Mat &result, const float thresh,
+    bool useExtendHeight) {
+
     Mat mser = image(rect);
     Mat mser_mat;
-    threshold(mser, mser_mat, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
+    cv::threshold(mser, mser_mat, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
 
-    Rect normalRect = adaptive_charrect_from_rect(rect, image.cols, image.rows);
+    Rect normalRect = adaptive_charrect_from_rect(rect, image.cols, image.rows, useExtendHeight);
     Mat region = image(normalRect);
     Mat thresh_mat;
-    threshold(region, thresh_mat, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
+    cv::threshold(region, thresh_mat, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
 
     // count mser diff ratio
-    int countdiff = countNonZero(thresh_mat) - countNonZero(mser_mat);
+    int countdiff = abs(countNonZero(thresh_mat) - countNonZero(mser_mat));
 
     float MserDiffOstuRatio = float(countdiff) / float(rect.area());
 
-    if (MserDiffOstuRatio > 1) {
-      //cv::rectangle(result, rect, Scalar(0, 0, 0), 2);
+    if (MserDiffOstuRatio > thresh) {
+      //std::cout << "MserDiffOstuRatio:" << MserDiffOstuRatio << std::endl;
+      /*imshow("tmpMat", mser_mat);
+      waitKey(0);
+      imshow("tmpMat", thresh_mat);
+      waitKey(0);*/
+
+      cv::rectangle(result, normalRect, Scalar(0, 0, 0), 2);
       return false;
     }
 
@@ -2152,21 +2205,25 @@ namespace easypr {
     }
   }
 
-  Rect adaptive_charrect_from_rect(const Rect &rect, int maxwidth, int maxheight) {
+  Rect adaptive_charrect_from_rect(const Rect &rect, int maxwidth, int maxheight, bool useExtendHeight) {
     int expendWidth = 0;
+    int extendHeight = 0;
 
     if (rect.height > 3 * rect.width) {
-      expendWidth = (rect.height / 2 - rect.width) / 2;
+      expendWidth = int((int(rect.height * 0.5f) - rect.width) * 0.5f);
+      if (useExtendHeight) {
+        extendHeight = int(rect.height * 0.3f);
+      }
     }
 
     //Rect resultRect(rect.tl().x - expendWidth, rect.tl().y,
     //  rect.width + expendWidth * 2, rect.height);
 
     int tlx = rect.tl().x - expendWidth > 0 ? rect.tl().x - expendWidth : 0;
-    int tly = rect.tl().y;
+    int tly = rect.tl().y - extendHeight > 0 ? rect.tl().y - extendHeight : 0;
 
     int brx = rect.br().x + expendWidth < maxwidth ? rect.br().x + expendWidth : maxwidth;
-    int bry = rect.br().y;
+    int bry = rect.br().y + extendHeight < maxheight ? rect.br().y + extendHeight : maxheight;
 
     Rect resultRect(tlx, tly, brx - tlx, bry - tly);
     return resultRect;

src/core/plate_recognize.cpp

@@ -22,6 +22,7 @@ int CPlateRecognize::plateRecognize(const Mat& src, std::vector<CPlate> &plateVe
     std::stringstream ss(std::stringstream::in | std::stringstream::out);
     ss << "resources/image/tmp/phone/phone_img_" << img_index << ".jpg";
     imwrite(ss.str(), img);
+    return -1;
   }
 
   // 1. plate detect
@@ -32,12 +33,7 @@ int CPlateRecognize::plateRecognize(const Mat& src, std::vector<CPlate> &plateVe
     for (size_t j = 0; j < num; j++) {
       CPlate item = plateVec.at(j);
       Mat plateMat = item.getPlateMat();
-
-      if (0) {
-        imshow("plate", plateMat);
-        waitKey(0);
-        destroyWindow("plate");
-      }
+      //SHOW_IMAGE(plateMat, 1);
 
       // scale the rect to src;
       item.setPlateScale(scale);

src/train/create_data.cpp

@@ -25,6 +25,7 @@ namespace easypr {
   // shift an image
   Mat translateImg(Mat img, int offsetx, int offsety, int bk){
     Mat dst;
+    Mat test = Mat_<double>(2, 3) << 2;
     //cout << test << endl;
     Mat trans_mat = (Mat_<double>(2, 3) << 1, 0, offsetx, 0, 1, offsety);
     //cout << trans_mat << endl;

test/accuracy.hpp

@@ -101,16 +101,20 @@ namespace easypr {
 
       // parameters
       const bool filesNatureSort = true;
-      const int max_plates = 1;
+      const int max_plates = 4;
       const int isGenerateGT = 1;
 
       // set the parameters of CPlateRecognize
       CPlateRecognize pr;
       pr.setResultShow(false);
       pr.setLifemode(true);
-      pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_CMSER);
-      pr.setMaxPlates(4);
- 
+      //  pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_CMSER | PR_DETECT_SOBEL );
+      //  pr.setMaxPlates(4);
+
+        pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_CMSER);
+        pr.setMaxPlates(1);
+
+      //pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_SOBEL);
       // load the maching learning model
       //pr.LoadSVM("resources/model/svm.xml");
       pr.LoadANN("resources/model/ann.xml");
@@ -157,6 +161,11 @@ namespace easypr {
       vector<float> icdar2003_precise_all;
       icdar2003_precise_all.reserve(size * max_plates);
 
+      // use to generate the plate trainging data, for training end2end chars recognize model
+      std::stringstream plate_ss(std::stringstream::in | std::stringstream::out);
+      int batch_i = 3;
+      int plate_out_i = 0;
+
       time_t begin, end;
       time(&begin);
 
@@ -274,6 +283,13 @@ namespace easypr {
               // if isGenerateG, then writes the text string of plate license 
               if (isGenerateGT) {
                 rectangleNode.updateText(matchPlate->getPlateStr().c_str());
+
+                // use to generate the plate trainging data, for training end2end chars recognize model
+                std::stringstream ss(std::stringstream::in | std::stringstream::out);
+                ss << "resources/image/tmp/plateMat/plate_" << batch_i << "_" << plate_out_i << ".jpg";
+                imwrite(ss.str(), matchPlate->getPlateMat());
+                plate_ss << "plate_" << batch_i << "_" << plate_out_i << ".jpg" << ":" << license << endl;
+                plate_out_i++;
               }
 
               int diff = utils::levenshtein_distance(license, matchLicense);
@@ -502,6 +518,19 @@ namespace easypr {
       else {
         cout << "Unable to open file";
       }
+
+      // use to generate the plate trainging data, for training end2end chars recognize model
+      if(isGenerateGT) {
+        ofstream myfile("result/plateGroundTruth.txt", ios::app);
+        if (myfile.is_open()) {
+          myfile << plate_ss.str();
+          myfile.close();
+        }
+        else {
+          cout << "Unable to open file";
+        }
+      }
+
       return 0;
     }