diff --git a/accuracy.txt b/accuracy.txt index 22cfdcc..9011fd7 100644 --- a/accuracy.txt +++ b/accuracy.txt @@ -1722,3 +1722,8 @@ Recall:31.1421%, Precise:35.8134%, Fscore:33.3148%. Recall:83.2984%, Precise:86.2716%, Fscore:84.7589%. ƽַ:0.845361, ȫƥ:127, ȫƥ:65.4639% ʱ:216, ƽִʱ:1.08 +2016-06-21 15:39:31 +ͼƬ:25, δʶͼƬ:25, λ:0% +Recall:0%, Precise:0%, Fscore:0%. +ƽַ:0, ȫƥ:0, ȫƥ:0% +ʱ:231, ƽִʱ:9.24 diff --git a/include/easypr/core/character.hpp b/include/easypr/core/character.hpp index d0f088c..dbf1641 100644 --- a/include/easypr/core/character.hpp +++ b/include/easypr/core/character.hpp @@ -9,7 +9,8 @@ ////////////////////////////////////////////////////////////////////////// #ifndef EASYPR_CORE_CHARACTER_H_ #define EASYPR_CORE_CHARACTER_H_ -#include "core_func.h" + +#include "easypr/core/core_func.h" /*! \namespace easypr Namespace where all the C++ EasyPR functionality resides @@ -20,6 +21,11 @@ namespace easypr { public: CCharacter() { + m_characterMat = Mat(); + m_characterPos = Rect(); + m_characterStr = ""; + m_score = 0; + m_isChinese = false; } CCharacter(const CCharacter& other) @@ -28,6 +34,7 @@ namespace easypr { m_characterPos = other.m_characterPos; m_characterStr = other.m_characterStr; m_score = other.m_score; + m_isChinese = other.m_isChinese; } inline void setCharacterMat(Mat param) { m_characterMat = param; } @@ -42,6 +49,18 @@ namespace easypr { inline void setCharacterScore(double param) { m_score = param; } inline double getCharacterScore() const { return m_score; } + inline void setIsChinese(bool param) { m_isChinese = param; } + inline bool getIsChinese() const { return m_isChinese; } + + //inline void setIsStrong(bool param) { isStrong = param; } + inline bool getIsStrong() const { return m_score >= 0.9; } + + //inline void setIsWeak(bool param) { isWeak = param; } + inline bool getIsWeak() const { return m_score < 0.9 && m_score >= 0.5; } + + //inline void setIsLittle(bool param) { isLittle = param; } + inline bool getIsLittle() const { return m_score < 0.5; } + bool operator < (const CCharacter& other) const { return (m_score > other.m_score); @@ -66,8 +85,16 @@ namespace easypr { double m_score; //! weather is chinese - bool isChinese; + bool m_isChinese; + + ////! m_score >= 0.9 + //bool isStrong; + + ////! m_score < 0.9 && m_score >= 0.5 + //bool isWeak; + ////! m_score < 0.5 + //bool isLittle; }; } /*! \namespace easypr*/ diff --git a/include/easypr/core/chars_identify.h b/include/easypr/core/chars_identify.h index 750e0ec..7d02538 100644 --- a/include/easypr/core/chars_identify.h +++ b/include/easypr/core/chars_identify.h @@ -2,6 +2,7 @@ #define EASYPR_CORE_CHARSIDENTIFY_H_ #include "easypr/util/kv.h" +#include "easypr/core/character.hpp" #include #include "opencv2/opencv.hpp" @@ -14,6 +15,8 @@ class CharsIdentify { int classify(cv::Mat f, float& maxVal, bool isChinses = false); void classify(cv::Mat featureRows, std::vector& out_maxIndexs, std::vector& out_maxVals, std::vector isChineseVec); + void classify(std::vector& charVec); + std::pair identify(cv::Mat input, bool isChinese = false); int identify(std::vector inputs, std::vector>& outputs, diff --git a/include/easypr/core/plate.hpp b/include/easypr/core/plate.hpp index 332de68..82ae6b4 100644 --- a/include/easypr/core/plate.hpp +++ b/include/easypr/core/plate.hpp @@ -9,7 +9,8 @@ ////////////////////////////////////////////////////////////////////////// #ifndef EASYPR_CORE_PLATE_H_ #define EASYPR_CORE_PLATE_H_ -#include "core_func.h" + +#include "easypr/core/core_func.h" /*! \namespace easypr Namespace where all the C++ EasyPR functionality resides diff --git a/resources/image/native_test/沪ALB022.jpg b/resources/image/extreme_test/other/沪ALB022.jpg similarity index 100% rename from resources/image/native_test/沪ALB022.jpg rename to resources/image/extreme_test/other/沪ALB022.jpg diff --git a/resources/image/native_test/沪B683J8.jpg b/resources/image/extreme_test/other/沪B683J8.jpg similarity index 100% rename from resources/image/native_test/沪B683J8.jpg rename to resources/image/extreme_test/other/沪B683J8.jpg diff --git a/resources/image/native_test/渝B777C9.jpg b/resources/image/extreme_test/other/渝B777C9.jpg similarity index 100% rename from resources/image/native_test/渝B777C9.jpg rename to resources/image/extreme_test/other/渝B777C9.jpg diff --git a/resources/image/test.jpg b/resources/image/test.jpg index 26c88d3..7c70737 100644 Binary files a/resources/image/test.jpg and b/resources/image/test.jpg differ diff --git a/result/Result.xml b/result/Result.xml index b6797fd..b22c42b 100644 --- a/result/Result.xml +++ b/result/Result.xml @@ -1,1231 +1,103 @@ - A88731 - - :A88731 - - - - CX8888 - - :CX8888 - - - - H99999 - - :H99999 - - - - FA3215 - - :FA3215 - :AT1203 - - - - A019W2 - - :A019J2 - - - - A09X20 - - :A09X20 - - - - A82M83 - - :A82M83 - - - - A88888 - - :A88888 - - - - AA662F - - :AA662F - - - - AEK882 - - :AEK882 - - - - AL0Q87 - - :AL0Q87 - - - - C28888 - - :C28888 - - - - A51V39 - - :A51V39 - :E372XN - - - - AGH092 - - :AGH092 - - - - AP0910 - - :AP0910 - - - - J32500 - - :132500 - - - - JS6999 - - :JS699L - - - - K62933 - - :XXXX11 - :K62933 - - - - KT5583 - - :KT5583 - - - - AHP676 - - :UUL11M - :AHP676 - - - - DTG667 - - :1JZ399 - :DTG661 - - - - E14579 - - :C5Y45Z - - - - E22602 - - :E22602 - :0LZ686 - - - - E24494 - - :E24494 - - - - E28437 - - :GTL921 - :E28437 - - - - G68991 - - :MRW368 - :G68991 - - - - H65817 - - :H65817 - - - - JZ3999 - - :E24494 - :7JZ399 - - - - KNG518 - - :KNG518 - δ֪:1111Q1 - - - - LS2999 - - :LS2999 - - - - MJY929 - - :MSY678 - - - - MN0888 - - :MN0881 - - - - NRL118 - - :NRL118 - - - - RB7992 - - :RB7992 - - - - A03168 - - δ֪:A03168 - - - - A12210 - - :A12210 - - - - A12903 - - δ֪:A12903 - - - - A13840 - - δ֪:A13840 - - - - A21027 - - :A21027 - - - - A50819 - - :A50879 - - - - A54614 - - :Q54614 - - - - A68952 - - :A68952 - - - - A72220 - - :ԥA72220 - - - - A88888 - - :A88888 - - - - A95044 - - :1Q9504 - - - - AA2270 - - :ALZZ7C - - - - AA4586 - - δ֪:AAA586 - - - - AA5547 - - :AA5547 - - - - AC1847 - - :AC18A7 - - - - AM690M - - :11A690 - - - - B7C289 - - :B7C289 - - - - F397C0 - - :F397C0 - - - - F99999 - - :F99999 - - - - G70000 - - :G70000 - - - - GZB388 - - :GZB388 - - - - GZJ021 - - :GZJ021 - - - - JP291Q - - :JP291Q - - - - LD1930 - - :LD1930 - - - - B3587ѧ - - :B3587L - - - - A05H22 - - :A05H22 - - - - A07G31 - - :A07G31 - :A11R55 - - - - A0PC37 - - :A0PC37 - :AJW532 - - - - A0PQ76 - - :1111P1 - :A21A42 - - - - A2991D - - :A2991D - - - - A31Y83 - - :A11R55 - :A3LY83 - - - - A3685C - - :A7GB86 - :CA9622 - - - - A3MA06 - - :A3MA06 - - - - A53U19 - - :1111RZ - :A53U19 - - - - A5RM87 - - :F11111 - :A5RM87 - δ֪:13QV11 - - - - A97971 - - :A97971 - - - - A9NU97 - - :A9NU91 - :B450LJ - - - - A9YU86 - - :111111 - :A9YU86 - :A0PC37 - - - - AA8516 - - :AA8516 - - - - AED855 - - :AED855 - :A7KZ55 - - - - AL8387 - - :AL8387 - - - - AT1Y68 - - :AT1Y68 - - - - AT3597 - - :AT3597 - - - - AT8781 - - :AT8781 - - - - ATL269 - - :ATL269 - - - - AYN355 - - :AYN355 - :K16893 - - - - F8ZS83 - - :F4ZQ33 - - - - G60009 - - :G60009 - - - - GR0L16 - - :GR0L16 - - - - A00T45 - - :A00T45 - - - - A09T87 - - :5A09T1 - :A09T8 - - - - A1T235 - - :A1T235 - - - - A22T43 - - :A22T43 - - - - A30123 - - :A30123 - - - - A45277 - - :145277 - - - - A80003 - - :A80003 - - - - A80375 - - :A80375 - - - - A85501 - - :A85501 - - - - A85890 - - :A85890 - - - - A87271 - - :A87271 - - - - AH8331 - - :AH833 - - - - AJH155 - - :AJH155 - - - - AK169H - - :AK169H - - - - AL025S - - :A1M95S - - - - AS3165 - - :1M6111 - :AS3165 - - - - AT789S - - :AT789S - - - - ATH859 - - :ATH859 - - - - AUB816 - - :AUB816 - - - - AX688A - - :AX688A - :DAX688 - - - - BBC666 + D71603 - EZM618 - - :EZM618 - - - - F71646 - - :F71646 - - - - M12288 - - :M12288 - - - - M69311 - - :M69311 - - - - P77222 - - :P77222 - - - - Q06417 - - :Q06413 - - - - Q0686ѧ + A26M71 - Q15538 - - :Q15538 - - - - Q18632 - - :Q18632 - - - - Q19917 - - :Q19917 - - - - Q80197 + C01701 - QA2825 - - :0A2825 - - - - A2TS00 - - :A2TS00 - :AK595R - - - - A5J512 - - :A5J512 - - - - A5PJ05 - - :ԥA5PJ95 - - - - A5Q951 - - :A5Q9Q1 - - - - A84266 - - :A84266 - - - - A89311 - - :A8931Z - - - - A94372 - - :A9A372 - - - - AA0825 - - δ֪:A0857 - - - - AAA379 - - :AAA3M9 - - - - AAB457 - - :1LAAB4 - - - - AAC044 - - :AAC044 - :0Y0TA - - - - AAD348 - - :1PAAL3 - - - - AAF230 - - :AAF20 - - - - AB288Y - - :AB288Y - :A883J9 - - - - AF9C00 - - :AF9C00 - - - - AK9331 + E6686V - AL6212 - - :VRAL62 - - - - AV0U41 - - :AV0U41 - - - - B0K999 - - :B0K999 - - - - B2757B - - :B2757B - - - - B2LR57 - - :B2LR57 - - - - B3RS91 - - :B3RS91 - - - - B4051L - - :B4051L - - - - B577QK - - :B577QK - - - - B5PQ23 - - :B5PQ23 - - - - B5W601 - - :7550D - :B5W601 - - - - B7VW40 - - :B7VW40 - - - - B901BF - - :B901BF - - - - B972HL - - :B972HL - - - - BA9H07 - - :BA9H07 - - - - BB867A - - :BB867A - - - - BE24Q7 - - :BE24Q7 - - - - BK33E3 - - :BK33E3 - - - - BP3T05 - - :BP3T05 - - - - BR75Y3 - - :BR75Y3 - - - - BTT255 + L11921 - BW44R6 - - :BW44R6 - - - - BZ756T - - :BZ756T - - - - BZ89M9 - - :BZ89M9 - - - - E9R439 - - :E9R439 - :14Z1LL - - - - OA2160 - - :0A2160 - - - - OA9112 - - :0A9112 - - - - OT9048 - - :0T90A8 - - - - SD050L - - :SL050L - δ֪:80021 - - - - TD1291 - - :ԥYD1291 - - - - VS1866 - - :VS1866 - :VM1V1 - - - - A0CP56 - - :A0CP56 - - - - A6E176 - - :A6F176 - - - - A75976 - - :A759Z6 - - - - A85Z95 - - :EQAZ95 - :A85Z95 - - - - A9YP07 - - :A9YP07 - - - - ADW072 - - :ADK072 - - - - AHOG66 - - :AH0G66 - - - - AL926V - - :AL926V - - - - B551QV - - :B551QV - - - - B577CU - - :B577CU - - - - BGG522 - - :BGG522 - - - - BU5838 - - :BU5838 - - - - CTF181 - - :CTF181 - - - - DLA031 - - :DLA031 - - - - E2Y256 - - :E2Y256 - - - - E75614 - - :E75614 - - - - EB68W9 - - :EB68W9 - - - - ԥK91239 - - :ԥK9Y239 - - - - ԥS33909 - - :ԥS33909 - + L66736 + - A82F36 - - :AB148G - :A82F36 - :AE0855 - + BE7773 + - A88888 - - :A88888 - + A2HQ34 + - B99999 - - :11111Y - :699999 - + A5DP12 + - BMW005 - - :BMW005 - + A82349 + - C33333 - - :C33333 - + A961F3 + - PE9218 - - :PE9218 - + AB2893 + - HB1508 - - :HB1508 - :HB1508 - + BA103N + - AD2N68 - - :AD2N68 - + BDB720 + - AE8F80 - - :MLVVF1 - :AE8F80 - :AF8D1 - + BE609T + - ³A88888 - - :³A88888 - + SK903B + - ³B995EQ - - :³B995EQ - + X30479 + - ³BQG527 + A20Q03 - ³JRW350 - - :³JRW350 - + A36E80 + - ³KK5555 - - :³KK5555 - + A66U71 + - ³LD9016 - - :³LD9016 - + AD6A99 + - ³Y44748 - - :³Y44748 - + AP0966 + - A16341 - - :A16341 - + AQ5B65 + - A1R272 - - :A1R272 - + AVW997 + - AB4444 - - :AB4444 - + ԥU00000 + diff --git a/src/core/chars_identify.cpp b/src/core/chars_identify.cpp index 9445e13..55b9f29 100644 --- a/src/core/chars_identify.cpp +++ b/src/core/chars_identify.cpp @@ -33,13 +33,14 @@ namespace easypr { ann_->predict(featureRows, output); for (int output_index = 0; output_index < rowNum; output_index++) { + Mat output_row = output.row(output_index); int result = -1; float maxVal = -2.f; bool isChinses = isChineseVec[output_index]; if (!isChinses) { result = 0; for (int j = 0; j < kCharactersNumber; j++) { - float val = output.at(j); + float val = output_row.at(j); // std::cout << "j:" << j << "val:" << val << std::endl; if (val > maxVal) { maxVal = val; @@ -50,7 +51,7 @@ namespace easypr { else { result = kCharactersNumber; for (int j = kCharactersNumber; j < kCharsTotalNumber; j++) { - float val = output.at(j); + float val = output_row.at(j); //std::cout << "j:" << j << "val:" << val << std::endl; if (val > maxVal) { maxVal = val; @@ -63,6 +64,63 @@ namespace easypr { } } + + void CharsIdentify::classify(std::vector& charVec){ + size_t charVecSize = charVec.size(); + + Mat featureRows; + for (size_t index = 0; index < charVecSize; index++) { + Mat charInput = charVec[index].getCharacterMat(); + Mat feature = charFeatures(charInput, kPredictSize); + featureRows.push_back(feature); + } + + cv::Mat output(charVecSize, kCharsTotalNumber, CV_32FC1); + ann_->predict(featureRows, output); + + for (size_t output_index = 0; output_index < charVecSize; output_index++) { + CCharacter& character = charVec[output_index]; + Mat output_row = output.row(output_index); + + int result = -1; + float maxVal = -2.f; + std::string label = ""; + + bool isChinses = character.getIsChinese(); + if (!isChinses) { + result = 0; + for (int j = 0; j < kCharactersNumber; j++) { + float val = output_row.at(j); + //std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + label = std::make_pair(kChars[result], kChars[result]).second; + } + else { + result = kCharactersNumber; + for (int j = kCharactersNumber; j < kCharsTotalNumber; j++) { + float val = output_row.at(j); + //std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + const char* key = kChars[result]; + std::string s = key; + std::string province = kv_->get(s); + label = std::make_pair(s, province).second; + } + /*std::cout << "result:" << result << std::endl; + std::cout << "maxVal:" << maxVal << std::endl;*/ + character.setCharacterScore(maxVal); + character.setCharacterStr(label); + } + } + int CharsIdentify::classify(cv::Mat f, float& maxVal, bool isChinses){ int result = -1; diff --git a/src/core/core_func.cpp b/src/core/core_func.cpp index 84d1e89..1928c3c 100644 --- a/src/core/core_func.cpp +++ b/src/core/core_func.cpp @@ -3,7 +3,6 @@ // 这个部分中的函数轻易不要改动 #include "easypr/core/core_func.h" -#include "easypr/core/character.hpp" #include "easypr/core/chars_identify.h" using namespace cv; @@ -688,7 +687,6 @@ Rect GetCenterRect(Mat &in) { Mat charFeatures(Mat in, int sizeData) { //抠取中间区域 - Rect _rect = GetCenterRect(in); Mat tmpIn = CutTheRect(in, _rect); @@ -724,6 +722,9 @@ Mat charFeatures(Mat in, int sizeData) { j++; } } + + //std::cout << out << std::endl; + return out; } @@ -963,6 +964,106 @@ bool verifyRotatedPlateSizes(RotatedRect mr) { return true; } +//! 非极大值抑制 +void NMStoCharacter(std::vector &inVec, std::vector &resultVec, double overlap) { + + std::sort(inVec.begin(), inVec.end()); + + std::vector::iterator it = inVec.begin(); + for (; it != inVec.end(); ++it) { + CCharacter charSrc = *it; + //std::cout << "plateScore:" << plateSrc.getPlateScore() << std::endl; + Rect rectSrc = charSrc.getCharacterPos(); + + std::vector::iterator itc = it + 1; + + for (; itc != inVec.end();) { + CCharacter charComp = *itc; + Rect rectComp = charComp.getCharacterPos(); + Rect rectInter = rectSrc & rectComp; + Rect rectUnion = rectSrc | rectComp; + double r = double(rectInter.area()) / double(rectUnion.area()); + if (r > overlap) { + itc = inVec.erase(itc); + } + else { + ++itc; + } + } + } + + resultVec = inVec; +} + +// judge weather two CCharacter are nearly the same; +bool compareCharRect(const CCharacter& character1, const CCharacter& character2) +{ + Rect rect1 = character1.getCharacterPos(); + Rect rect2 = character2.getCharacterPos(); + + // the character in plate are similar height + float width_1 = float(rect1.width); + float height_1 = float(rect1.height); + + float width_2 = float(rect2.width); + float height_2 = float(rect2.height); + + float height_diff = abs(height_1 - height_2); + double height_diff_ratio = height_diff / min(height_1, height_2); + + if (height_diff_ratio > 0.25) + return false; + + // the character in plate are similar in the y-axis + float y_1 = float(rect1.tl().y); + float y_2 = float(rect2.tl().y); + + float y_diff = abs(y_1 - y_2); + double y_diff_ratio = y_diff / min(height_1, height_2); + + if (y_diff_ratio > 0.5) + return false; + + // the character in plate are near in the x-axis but not very near + float x_margin_left = float(min(rect1.br().x, rect2.br().x)); + float x_margin_right = float(max(rect1.tl().x, rect2.tl().x)); + + float x_margin_diff = abs(x_margin_left - x_margin_right); + double x_margin_diff_ratio = x_margin_diff / max(width_1, width_2); + + if (x_margin_diff_ratio > 3) + return false; + + return true; +} + +//! merge chars to group, using the similarity +void mergeCharToGroup(std::vector vecRect, + std::vector>& charGroupVec) { + + std::vector charVec; + + std::vector labels; + double overlap = 0.9; + NMStoCharacter(vecRect, charVec, overlap); + + int numbers = partition(charVec, labels, &compareCharRect); + for (size_t j = 0; j < size_t(numbers); j++) { + std::vector charGroup; + + for (size_t t = 0; t < charVec.size(); t++) { + int label = labels[t]; + + if (label == j) + charGroup.push_back(charVec[t]); + } + + if (charGroup.size() < 2) + continue; + + charGroupVec.push_back(charGroup); + } +} //! use verify size to first generate char candidates Mat mserCharMatch(const Mat &src, Mat &match, std::vector& out_charRect) { @@ -979,8 +1080,12 @@ Mat mserCharMatch(const Mat &src, Mat &match, std::vector& out_charRect) { Mat result = image.clone(); cvtColor(result, result, COLOR_GRAY2BGR); - int imageArea = image.rows * image.cols; - mser = MSER::create(1, 30, int(0.05 * imageArea)); + const int imageArea = image.rows * image.cols; + const int delta = 1; + const int minArea = 30; + const double maxAreaRatio = 0.05; + + mser = MSER::create(delta, minArea, int(maxAreaRatio * imageArea)); mser->setPass2Only(true); mser->detectRegions(image, all_contours, all_boxes); @@ -988,45 +1093,80 @@ Mat mserCharMatch(const Mat &src, Mat &match, std::vector& out_charRect) { size_t size = all_contours.size(); int char_index = 0; + int char_size = 20; + // verify char size and output to rects; for (size_t index = 0; index < size; index++) { Rect rect = all_boxes[index]; std::vector contour = all_contours[index]; - RotatedRect rrect = minAreaRect(Mat(contour)); if (verifyCharSizes(rect)) { - Mat mserMat = adaptive_image_from_points(contour, rect, rect.size()); + Mat mserMat = adaptive_image_from_points(contour, rect, Size(char_size, char_size)); + Mat charInput = preprocessChar(mserMat, char_size); + + CCharacter charCandidate; + charCandidate.setCharacterPos(rect); + charCandidate.setCharacterMat(charInput); + charCandidate.setIsChinese(false); + charVec.push_back(charCandidate); + } + } - /*Mat region = image(rect); - Mat binaryMat; - threshold(region, binaryMat, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);*/ + CharsIdentify::instance()->classify(charVec); - Mat charInput = preprocessChar(mserMat, 20); - - std::string label = ""; - float maxVal = -2.f; - bool isCharacter = CharsIdentify::instance()->isCharacter(charInput, label, maxVal); - - if (1) { - //match(rect) = min(max(0, int(maxVal * 255)),255); - match(rect) = 255; - - cv::rectangle(result, rect, Scalar(255, 0, 0)); - Point center(rect.tl().x + rect.width / 2, rect.tl().y + rect.height / 2); - - cv::circle(result, center, 3, Scalar(0, 255, 0), 2); - out_charRect.push_back(rect); - //CCharacter character; - //character.setCharacterPos(rect); - //character.setCharacterMat(binary_region); - //character.setCharacterStr(label); - //character.setCharacterScore(maxVal); - //charVec.push_back(character); + std::vector strongSeedVec; + std::vector weakSeedVec; + std::vector littleSeedVec; + + size_t charCan_size = charVec.size(); + for (size_t index = 0; index < charCan_size; index++) { + CCharacter& charCandidate = charVec[index]; + Rect postion = charCandidate.getCharacterPos(); + double score = charCandidate.getCharacterScore(); + if (charCandidate.getIsStrong()) { + strongSeedVec.push_back(charCandidate); + } + else if (charCandidate.getIsWeak()) { + weakSeedVec.push_back(charCandidate); + } + else if (charCandidate.getIsLittle()) { + littleSeedVec.push_back(charCandidate); + } + + } + + //merge chars to group + std::vector> charGroupVec; + mergeCharToGroup(strongSeedVec, charGroupVec); + + //draw the line of the group + for (auto charGroup : charGroupVec) { + std::vector points; + Vec4f line; + int maxarea = 0; + Rect maxrect; + + for (auto character : charGroup) { + Rect charRect = character.getCharacterPos(); + Point center(charRect.tl().x + charRect.width / 2, charRect.tl().y + charRect.height / 2); + points.push_back(center); + + cv::circle(result, center, 3, Scalar(0, 255, 0), 2); + if (charRect.area() > maxarea) { + maxrect = charRect; + maxarea = charRect.area(); } } + + if (points.size() >= 2) { + fitLine(Mat(points), line, CV_DIST_L2, 0, 0.01, 0.01); + float k = line[1] / line[0]; + float step = 10.f * (float)maxrect.width; + cv::line(result, Point2f(line[2] - step, line[3] - k*step), Point2f(line[2] + step, k*step + line[3]), Scalar(255, 255, 255)); + } } - if (0) { + if (1) { imshow("result", result); waitKey(0); destroyWindow("result"); @@ -1035,8 +1175,6 @@ Mat mserCharMatch(const Mat &src, Mat &match, std::vector& out_charRect) { return match; } - - //! use verify size to first generate candidates Mat mserMatch(const Mat &src, Mat &match, const Color r, std::vector& out_plateRect, std::vector& out_charRect) { diff --git a/src/core/plate_locate.cpp b/src/core/plate_locate.cpp index 6b2c525..0406cc8 100644 --- a/src/core/plate_locate.cpp +++ b/src/core/plate_locate.cpp @@ -985,8 +985,7 @@ int CPlateLocate::plateMserLocate(Mat src, vector &candPlates, int index vector plates; Mat src_b; - for (size_t i = 0; i < channelImages.size(); ++i) - { + for (size_t i = 0; i < channelImages.size(); ++i) { Mat channelImage = channelImages[i]; Mat image = scaleImage(channelImage, Size(scale_size, scale_size), scale_ratio); @@ -999,13 +998,11 @@ int CPlateLocate::plateMserLocate(Mat src, vector &candPlates, int index } } - for (size_t i = 0; i < rects_mser.size(); ++i) - { + for (size_t i = 0; i < rects_mser.size(); ++i) { Rect_ outputRect; calcSafeRect(rects_mser[i], src, outputRect); - if (0) - { + if (0) { std::stringstream ss(std::stringstream::in | std::stringstream::out); ss << "resources/image/tmp/plate_" << i << ".jpg"; imwrite(ss.str(), src(outputRect)); @@ -1015,8 +1012,7 @@ int CPlateLocate::plateMserLocate(Mat src, vector &candPlates, int index plate.setPlateLocateType(CMSER); plate.setPlateMat(src(outputRect)); plate.setPlatePos(rects_mser[i]); - - candPlates.push_back(plate); + //candPlates.push_back(plate); } //deskew(src, src_b, rects_mser_blue, plates); diff --git a/src/core/plate_recognize.cpp b/src/core/plate_recognize.cpp index 1a34fcf..becf230 100644 --- a/src/core/plate_recognize.cpp +++ b/src/core/plate_recognize.cpp @@ -110,283 +110,184 @@ namespace easypr { return 0; } - - // compareRectIs - bool compareCharRect(const CCharacter& character1, const CCharacter& character2) - { - Rect rect1 = character1.getCharacterPos(); - Rect rect2 = character2.getCharacterPos(); - - // the character in plate are similar height - float width_1 = float(rect1.width); - float height_1 = float(rect1.height); - - float width_2 = float(rect2.width); - float height_2 = float(rect2.height); - - float height_diff = abs(height_1 - height_2); - double height_diff_ratio = height_diff / min(height_1, height_2); - - if (height_diff_ratio > 0.25) - return false; - - // the character in plate are similar in the y-axis - float y_1 = float(rect1.tl().y); - float y_2 = float(rect2.tl().y); - - float y_diff = abs(y_1 - y_2); - double y_diff_ratio = y_diff / min(height_1, height_2); - - if (y_diff_ratio > 0.5) - return false; - - // the character in plate are near in the x-axis but not very near - float x_margin_left = float(min(rect1.br().x, rect2.br().x)); - float x_margin_right = float(max(rect1.tl().x, rect2.tl().x)); - - float x_margin_diff = abs(x_margin_left - x_margin_right); - double x_margin_diff_ratio = x_margin_diff / max(width_1, width_2); - - if (x_margin_diff_ratio > 3) - return false; - - return true; - } - - //! Ǽֵ - void NMStoCharacter(std::vector &inVec, std::vector &resultVec, double overlap) { - - std::sort(inVec.begin(), inVec.end()); - - std::vector::iterator it = inVec.begin(); - for (; it != inVec.end(); ++it) { - CCharacter charSrc = *it; - //std::cout << "plateScore:" << plateSrc.getPlateScore() << std::endl; - Rect rectSrc = charSrc.getCharacterPos(); - - std::vector::iterator itc = it + 1; - - for (; itc != inVec.end();) { - CCharacter charComp = *itc; - Rect rectComp = charComp.getCharacterPos(); - Rect rectInter = rectSrc & rectComp; - Rect rectUnion = rectSrc | rectComp; - double r = double(rectInter.area()) / double(rectUnion.area()); - if (r > overlap) { - itc = inVec.erase(itc); - } - else { - ++itc; - } - } - } - - resultVec = inVec; - } - - - void MergeCharToPlate(std::vector& vecRect, std::vector>& vecRectPlate){ - std::vector labels; - - std::vector charVec; - - double overlap = 0.5; - NMStoCharacter(vecRect, charVec, overlap); - - int numbers = partition(charVec, labels, &compareCharRect); - for (size_t j = 0; j < size_t(numbers); j++) { - std::vector charRects; - - for (size_t t = 0; t < charVec.size(); t++) { - int label = labels[t]; - - if (label == j) - charRects.push_back(charVec[t]); - } - - if (charRects.size() == 0 || charRects.size() < 5) - continue; - - vecRectPlate.push_back(charRects); - } - } + int CPlateRecognize::plateRecognizeAsText(Mat src, std::vector &licenseVec) { - // Ʒ鼯 - std::vector plateVec; - - std::vector channelImages; - std::vector flags; - - - // only conside blue plate - if (1) { - Mat grayImage; - cvtColor(src, grayImage, COLOR_BGR2GRAY); - channelImages.push_back(grayImage); - flags.push_back(0); - - //Mat singleChannelImage; - //extractChannel(src, singleChannelImage, 2); - //channelImages.push_back(singleChannelImage); - //flags.push_back(0); - } - - std::vector>> all_contours; - std::vector> all_boxes; - - all_contours.resize(channelImages.size()); - all_boxes.resize(channelImages.size()); - - for (int i = 0; i < (int)channelImages.size(); ++i) - { - Ptr mser; - std::vector charVec; - Mat channelImage = channelImages[i]; - - // ͼͳһţȷͼҪ - // ٶ - int scale_size = 1200; - double scale_ratio = 1; - Mat image = scaleImage(channelImage, Size(scale_size, scale_size), scale_ratio); - - Mat result = image; - cvtColor(result, result, COLOR_GRAY2BGR); - - int imageArea = image.rows * image.cols; - - if (3 == flags[i]) { - mser = MSER::create(3, 30, int(0.05 * imageArea)); - } - else { - mser = MSER::create(1, 30, int(0.05 * imageArea)); - } - mser->detectRegions(image, all_contours[i], all_boxes[i]); - - size_t size = all_contours[i].size(); - - for (size_t index = 0; index < size; index++) { - Rect rect = all_boxes[i][index]; - - if (verifyCharSizes(rect)) { - float aspect = float(rect.width) / float(rect.height); - //if (aspect < 0.2) { - // cv::rectangle(result, rect, Scalar(0, 0, 255)); - //} - - Mat region = image(rect); - Mat binary_region; - threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); - - Mat charInput = preprocessChar(binary_region, 20); - std::string label = ""; - float maxVal = -2.f; - bool isCharacter = CharsIdentify::instance()->isCharacter(charInput, label, maxVal); - - if (isCharacter) { - cv::rectangle(result, rect, Scalar(255, 0, 0)); - - CCharacter character; - character.setCharacterPos(rect); - character.setCharacterMat(binary_region); - character.setCharacterStr(label); - character.setCharacterScore(maxVal); - charVec.push_back(character); - } - } - } - - std::vector> vecRectPlate; - MergeCharToPlate(charVec, vecRectPlate); - - for (size_t plate_i = 0; plate_i < vecRectPlate.size(); plate_i++) { - std::vector chars = vecRectPlate[plate_i]; - - std::sort(chars.begin(), chars.end(), - [](const CCharacter& r1, const CCharacter& r2) { return r1.getCharacterPos().x < r2.getCharacterPos().x; }); - - Rect specificRect = chars[0].getCharacterPos(); - Rect chineseRect = GetChineseRect(specificRect); - - cv::rectangle(result, chineseRect, Scalar(0, 255, 0)); - - Mat region = image(chineseRect); - Mat binary_region; - threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); - - Mat charInput = preprocessChar(binary_region, 20); - std::string label = ""; - float maxVal = -2.f; - bool isCharacter = CharsIdentify::instance()->isCharacter(charInput, label, maxVal, true); - - if (isCharacter) { - CCharacter character; - character.setCharacterPos(chineseRect); - character.setCharacterMat(binary_region); - character.setCharacterStr(label); - character.setCharacterScore(maxVal); - chars.push_back(character); - } - - int leftmax = image.rows; - int rightmax = 0; - int topmax = image.cols; - int bottommax = 0; - - std::string plateLicense = ""; - - for (size_t p = 0; p < chars.size(); p++) { - Rect rect = chars[p].getCharacterPos(); - int left = rect.tl().x; - int right = rect.br().x; - - int top = rect.tl().y; - int bottom = rect.br().y; - - if (left < leftmax) leftmax = left; - if (right > rightmax) rightmax = right; - if (top < topmax) topmax = top; - if (bottom > bottommax) bottommax = bottom; - - if (p != chars.size() - 1) - plateLicense.append(chars[p].getCharacterStr()); - } - - Rect grouprect(leftmax, topmax, rightmax - leftmax + 1, bottommax - topmax + 1); - Point2f center(float(leftmax + grouprect.width / 2), float(topmax + grouprect.height / 2)); - - plateLicense = label + plateLicense; - - if (verifyPlateSize(grouprect)) { - - cv::rectangle(result, grouprect, Scalar(0, 0, 255)); - - CPlate plate; - plate.setPlateLocateType(LocateType::CMSER); - plate.setPlatePos(RotatedRect(center, Size2f(float(grouprect.width), float(grouprect.height)), 0)); - plate.setPlateMat(image(grouprect)); - plate.setPlateStr(plateLicense); - licenseVec.push_back(plate); - } - } - - //RotatedRect testrr(Point2f(200, 200), Size2f(375, 100), 1.f); - //Point2f rect_points[4]; - //testrr.points(rect_points); - //for (int j = 0; j < 4; j++) - // line(result, rect_points[j], rect_points[(j + 1) % 4], Scalar(0,0,255), 2, 8); - - - if (1) { - imshow("result", result); - waitKey(0); - } - } - - - //showResult(result); + //// Ʒ鼯 + //std::vector plateVec; + + //std::vector channelImages; + //std::vector flags; + + + //// only conside blue plate + //if (1) { + // Mat grayImage; + // cvtColor(src, grayImage, COLOR_BGR2GRAY); + // channelImages.push_back(grayImage); + // flags.push_back(0); + + // //Mat singleChannelImage; + // //extractChannel(src, singleChannelImage, 2); + // //channelImages.push_back(singleChannelImage); + // //flags.push_back(0); + //} + + //std::vector>> all_contours; + //std::vector> all_boxes; + + //all_contours.resize(channelImages.size()); + //all_boxes.resize(channelImages.size()); + + //for (int i = 0; i < (int)channelImages.size(); ++i) + //{ + // Ptr mser; + // std::vector charVec; + // Mat channelImage = channelImages[i]; + + // // ͼͳһţȷͼҪ + // // ٶ + // int scale_size = 1200; + // double scale_ratio = 1; + // Mat image = scaleImage(channelImage, Size(scale_size, scale_size), scale_ratio); + + // Mat result = image; + // cvtColor(result, result, COLOR_GRAY2BGR); + + // int imageArea = image.rows * image.cols; + + // if (3 == flags[i]) { + // mser = MSER::create(3, 30, int(0.05 * imageArea)); + // } + // else { + // mser = MSER::create(1, 30, int(0.05 * imageArea)); + // } + // mser->detectRegions(image, all_contours[i], all_boxes[i]); + + // size_t size = all_contours[i].size(); + + // for (size_t index = 0; index < size; index++) { + // Rect rect = all_boxes[i][index]; + // + // if (verifyCharSizes(rect)) { + // float aspect = float(rect.width) / float(rect.height); + // //if (aspect < 0.2) { + // // cv::rectangle(result, rect, Scalar(0, 0, 255)); + // //} + + // Mat region = image(rect); + // Mat binary_region; + // threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); + + // Mat charInput = preprocessChar(binary_region, 20); + // std::string label = ""; + // float maxVal = -2.f; + // bool isCharacter = CharsIdentify::instance()->isCharacter(charInput, label, maxVal); + + // if (isCharacter) { + // cv::rectangle(result, rect, Scalar(255, 0, 0)); + + // CCharacter character; + // character.setCharacterPos(rect); + // character.setCharacterMat(binary_region); + // character.setCharacterStr(label); + // character.setCharacterScore(maxVal); + // charVec.push_back(character); + // } + // } + // } + + // std::vector> vecRectPlate; + // MergeCharToPlate(charVec, vecRectPlate); + + // for (size_t plate_i = 0; plate_i < vecRectPlate.size(); plate_i++) { + // std::vector chars = vecRectPlate[plate_i]; + + // std::sort(chars.begin(), chars.end(), + // [](const CCharacter& r1, const CCharacter& r2) { return r1.getCharacterPos().x < r2.getCharacterPos().x; }); + + // Rect specificRect = chars[0].getCharacterPos(); + // Rect chineseRect = GetChineseRect(specificRect); + + // cv::rectangle(result, chineseRect, Scalar(0, 255, 0)); + + // Mat region = image(chineseRect); + // Mat binary_region; + // threshold(region, binary_region, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); + + // Mat charInput = preprocessChar(binary_region, 20); + // std::string label = ""; + // float maxVal = -2.f; + // bool isCharacter = CharsIdentify::instance()->isCharacter(charInput, label, maxVal, true); + + // if (isCharacter) { + // CCharacter character; + // character.setCharacterPos(chineseRect); + // character.setCharacterMat(binary_region); + // character.setCharacterStr(label); + // character.setCharacterScore(maxVal); + // chars.push_back(character); + // } + + // int leftmax = image.rows; + // int rightmax = 0; + // int topmax = image.cols; + // int bottommax = 0; + + // std::string plateLicense = ""; + + // for (size_t p = 0; p < chars.size(); p++) { + // Rect rect = chars[p].getCharacterPos(); + // int left = rect.tl().x; + // int right = rect.br().x; + + // int top = rect.tl().y; + // int bottom = rect.br().y; + + // if (left < leftmax) leftmax = left; + // if (right > rightmax) rightmax = right; + // if (top < topmax) topmax = top; + // if (bottom > bottommax) bottommax = bottom; + + // if (p != chars.size() - 1) + // plateLicense.append(chars[p].getCharacterStr()); + // } + + // Rect grouprect(leftmax, topmax, rightmax - leftmax + 1, bottommax - topmax + 1); + // Point2f center(float(leftmax + grouprect.width / 2), float(topmax + grouprect.height / 2)); + + // plateLicense = label + plateLicense; + + // if (verifyPlateSize(grouprect)) { + // + // cv::rectangle(result, grouprect, Scalar(0, 0, 255)); + + // CPlate plate; + // plate.setPlateLocateType(LocateType::CMSER); + // plate.setPlatePos(RotatedRect(center, Size2f(float(grouprect.width), float(grouprect.height)), 0)); + // plate.setPlateMat(image(grouprect)); + // plate.setPlateStr(plateLicense); + // licenseVec.push_back(plate); + // } + // } + + // //RotatedRect testrr(Point2f(200, 200), Size2f(375, 100), 1.f); + // //Point2f rect_points[4]; + // //testrr.points(rect_points); + // //for (int j = 0; j < 4; j++) + // // line(result, rect_points[j], rect_points[(j + 1) % 4], Scalar(0,0,255), 2, 8); + + + // if (1) { + // imshow("result", result); + // waitKey(0); + // } + //} + + // + ////showResult(result); return 0; } diff --git a/src/core/tin12i3x.v2x b/src/core/tin12i3x.v2x new file mode 100644 index 0000000..894f85e --- /dev/null +++ b/src/core/tin12i3x.v2x @@ -0,0 +1,237 @@ +#include "easypr/core/chars_identify.h" +#include "easypr/core/character.hpp" +#include "easypr/config.h" +#include "easypr/core/core_func.h" + +namespace easypr { + + CharsIdentify* CharsIdentify::instance_ = nullptr; + + CharsIdentify* CharsIdentify::instance() { + if (!instance_) { + instance_ = new CharsIdentify; + } + return instance_; + } + + CharsIdentify::CharsIdentify() { + ann_ = ml::ANN_MLP::load(kDefaultAnnPath); + kv_ = std::shared_ptr(new Kv); + kv_->load("etc/province_mapping"); + } + + void CharsIdentify::LoadModel(std::string path) { + ann_->clear(); + ann_->ml::ANN_MLP::load(path); + } + + void CharsIdentify::classify(cv::Mat featureRows, std::vector& out_maxIndexs, + std::vector& out_maxVals, std::vector isChineseVec){ + int rowNum = featureRows.rows; + + cv::Mat output(rowNum, kCharsTotalNumber, CV_32FC1); + ann_->predict(featureRows, output); + + for (int output_index = 0; output_index < rowNum; output_index++) { + int result = -1; + float maxVal = -2.f; + bool isChinses = isChineseVec[output_index]; + if (!isChinses) { + result = 0; + for (int j = 0; j < kCharactersNumber; j++) { + float val = output.at(j); + // std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + } + else { + result = kCharactersNumber; + for (int j = kCharactersNumber; j < kCharsTotalNumber; j++) { + float val = output.at(j); + //std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + } + out_maxIndexs[output_index] = result; + out_maxVals[output_index] = maxVal; + } + } + + + void CharsIdentify::classify(std::vector& charVec){ + size_t charVec_size = charVec.size(); + + Mat featureRows; + for (size_t index = 0; index < charVec_size; index++) { + Mat charInput = charVec[index].getCharacterMat(); + Mat feature = charFeatures(charInput, kPredictSize); + featureRows.push_back(feature); + } + + int rowNum = featureRows.rows; + cv::Mat output(rowNum, kCharsTotalNumber, CV_32FC1); + ann_->predict(featureRows, output); + + for (int output_index = 0; output_index < rowNum; output_index++) { + CCharacter& character = charVec[output_index]; + int result = -1; + float maxVal = -2.f; + string label = ""; + bool isChinses = character.getIsChinese(); + if (!isChinses) { + result = 0; + for (int j = 0; j < kCharactersNumber; j++) { + float val = output.at(j); + // std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + } + else { + result = kCharactersNumber; + for (int j = kCharactersNumber; j < kCharsTotalNumber; j++) { + float val = output.at(j); + //std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + } + character.setCharacterScore(maxVal); + character.setCharacterStr(result); + } + } + + int CharsIdentify::classify(cv::Mat f, float& maxVal, bool isChinses){ + int result = -1; + + cv::Mat output(1, kCharsTotalNumber, CV_32FC1); + ann_->predict(f, output); + + maxVal = -2.f; + if (!isChinses) { + result = 0; + for (int j = 0; j < kCharactersNumber; j++) { + float val = output.at(j); + // std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + } + else { + result = kCharactersNumber; + for (int j = kCharactersNumber; j < kCharsTotalNumber; j++) { + float val = output.at(j); + //std::cout << "j:" << j << "val:" << val << std::endl; + if (val > maxVal) { + maxVal = val; + result = j; + } + } + } + //std::cout << "maxVal:" << maxVal << std::endl; + return result; + } + + bool CharsIdentify::isCharacter(cv::Mat input, std::string& label, float& maxVal, bool isChinese) { + cv::Mat feature = charFeatures(input, kPredictSize); + auto index = static_cast(classify(feature, maxVal, isChinese)); + + if (isChinese) + std::cout << "maxVal:" << maxVal << std::endl; + + if (maxVal >= 0.9) { + if (index < kCharactersNumber) { + label = std::make_pair(kChars[index], kChars[index]).second; + } + else { + const char* key = kChars[index]; + std::string s = key; + std::string province = kv_->get(s); + label = std::make_pair(s, province).second; + } + return true; + } + else + return false; + } + + /*bool CharsIdentify::charsJudge(std::vector& charVec) { + cv::Mat feature = charFeatures(input, kPredictSize); + auto index = static_cast(classify(feature, maxVal, isChinese)); + + if (isChinese) + std::cout << "maxVal:" << maxVal << std::endl; + + if (maxVal >= 0.9) { + if (index < kCharactersNumber) { + label = std::make_pair(kChars[index], kChars[index]).second; + } + else { + const char* key = kChars[index]; + std::string s = key; + std::string province = kv_->get(s); + label = std::make_pair(s, province).second; + } + return true; + } + else + return false; + }*/ + + + std::pair CharsIdentify::identify(cv::Mat input, bool isChinese) { + cv::Mat feature = charFeatures(input, kPredictSize); + float maxVal = -2; + auto index = static_cast(classify(feature, maxVal, isChinese)); + if (index < kCharactersNumber) { + return std::make_pair(kChars[index], kChars[index]); + } + else { + const char* key = kChars[index]; + std::string s = key; + std::string province = kv_->get(s); + return std::make_pair(s, province); + } + } + + int CharsIdentify::identify(std::vector inputs, std::vector>& outputs, + std::vector isChineseVec) { + Mat featureRows; + size_t input_size = inputs.size(); + for (size_t i = 0; i < input_size; i++) { + Mat input = inputs[i]; + cv::Mat feature = charFeatures(input, kPredictSize); + featureRows.push_back(feature); + } + + std::vector maxIndexs; + std::vector maxVals; + classify(featureRows, maxIndexs, maxVals, isChineseVec); + + for (size_t row_index = 0; row_index < input_size; row_index++) { + int index = maxIndexs[row_index]; + if (index < kCharactersNumber) { + outputs[row_index] = std::make_pair(kChars[index], kChars[index]); + } + else { + const char* key = kChars[index]; + std::string s = key; + std::string province = kv_->get(s); + outputs[row_index] = std::make_pair(s, province); + } + } + return 0; + } +} diff --git a/test/accuracy.hpp b/test/accuracy.hpp index ddd324b..7c1e94f 100644 --- a/test/accuracy.hpp +++ b/test/accuracy.hpp @@ -95,8 +95,8 @@ namespace easypr { // ҪһͼƬжٳ pr.setMaxPlates(4); - pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_SOBEL); - //pr.setDetectType(PR_DETECT_CMSER); + //pr.setDetectType(PR_DETECT_COLOR | PR_DETECT_SOBEL); + pr.setDetectType(PR_DETECT_CMSER); //CPlateDetect pd; //pd.setDetectType(PR_DETECT_CMSER);