From d5cfa6fcce622982624274e4e86a6670b29b634d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sun, 19 Feb 2017 15:17:40 +0800 Subject: [PATCH 01/10] Stash --- paddle/gserver/evaluators/Evaluator.cpp | 19 +++++++++++-- paddle/gserver/evaluators/Evaluator.h | 38 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index ae7508e2bb..4689222e3a 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -449,6 +449,21 @@ double AucEvaluator::calcAuc() const { } } +real AucEvaluator::getValueImpl() const { return calcAuc(); } + +std::string AucEvaluator::getTypeImpl() const { + if (colIdx_ == -1) { + return "last-column-auc"; + } else { + return "auc"; + } +} + +static InitFunction __reg_type_auc__([]() { + Evaluator::registrar_.registerClass("last-column-auc", + [] { return new AucEvaluator(-1); }); +}); + // class RankAucEvaluator REGISTER_EVALUATOR(rankauc, RankAucEvaluator); @@ -873,8 +888,6 @@ Evaluator* Evaluator::create(const EvaluatorConfig& config) { evaluator = new SumEvaluator(); } else if (config.type() == "last-column-sum") { evaluator = new ColumnSumEvaluator(-1); - } else if (config.type() == "last-column-auc") { - evaluator = new AucEvaluator(-1); } else { evaluator = registrar_.createByType(config.type()); } @@ -1253,4 +1266,6 @@ public: }; REGISTER_EVALUATOR(classification_error_printer, ClassificationErrorPrinter); +std::string DummyEvaluator::getTypeImpl() const { return "dummy"; } + } // namespace paddle diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index 5770847309..bf08aa07f0 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -19,6 +19,7 @@ limitations under the License. */ #include "paddle/parameter/Argument.h" #include "paddle/pserver/ParameterClient2.h" #include "paddle/utils/ClassRegistrar.h" +#include "paddle/utils/Error.h" namespace paddle { @@ -117,6 +118,34 @@ public: static ClassRegistrar registrar_; + virtual void getNames(std::vector* names) { + names->clear(); + names->push_back(config_.name()); + } + + virtual real getValue(const std::string& name, + paddle::Error* err = nullptr) const { + if (name != config_.name() && err != nullptr) { + *err = paddle::Error("no such name of evaluator %s", name.c_str()); + return .0f; + } + return this->getValueImpl(); + } + + virtual std::string getType(const std::string& name, + paddle::Error* err = nullptr) const { + if (name != config_.name() && err != nullptr) { + *err = paddle::Error("no such name of evaluator %s", name.c_str()); + return std::string(); + } + return this->getTypeImpl(); + } + +protected: + virtual real getValueImpl() const { return .0f; } + + virtual std::string getTypeImpl() const { return "base"; } + protected: EvaluatorConfig config_; double numSamples_; @@ -135,6 +164,10 @@ public: } virtual void finish() {} virtual void printStats(std::ostream&) const {} + + // Evaluator interface +protected: + std::string getTypeImpl() const; }; /** * @brief evaluate AUC using colIdx-th column as prediction. @@ -191,6 +224,11 @@ private: } double calcAuc() const; + + // Evaluator interface +protected: + real getValueImpl() const; + std::string getTypeImpl() const; }; /** From 04eaf75c5ea049ec88ba0c34f40176dee930c5ed Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sun, 19 Feb 2017 17:06:36 +0800 Subject: [PATCH 02/10] Add getValue to some evaluators. --- paddle/gserver/evaluators/Evaluator.cpp | 149 +++++++++++++----- paddle/gserver/evaluators/Evaluator.h | 44 +++++- .../gradientmachines/NeuralNetwork.cpp | 38 +++++ .../gserver/gradientmachines/NeuralNetwork.h | 4 + paddle/utils/Error.h | 2 + 5 files changed, 198 insertions(+), 39 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index be4f5f558f..b1daa09062 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -538,12 +538,15 @@ double RankAucEvaluator::calcRankAuc(real* outputData, : aucTmp / (clickSum * noClickSum); } +std::string RankAucEvaluator::getTypeImpl() const { return "rankauc"; } + // class PrecisionRecallEvaluator REGISTER_EVALUATOR(precision_recall, PrecisionRecallEvaluator); void PrecisionRecallEvaluator::start() { Evaluator::start(); statsInfo_.clear(); + values_.clear(); } real PrecisionRecallEvaluator::evalImp(std::vector& arguments) { @@ -603,7 +606,9 @@ real PrecisionRecallEvaluator::evalImp(std::vector& arguments) { return 0; } -void PrecisionRecallEvaluator::printStats(std::ostream& os) const { +template +void PrecisionRecallEvaluator::printStatsHelper(T1 labelCallback, + T2 microAvgCallback) const { int label = config_.positive_label(); if (label != -1) { CHECK(label >= 0 && label < (int)statsInfo_.size()) @@ -612,9 +617,7 @@ void PrecisionRecallEvaluator::printStats(std::ostream& os) const { double precision = calcPrecision(statsInfo_[label].TP, statsInfo_[label].FP); double recall = calcRecall(statsInfo_[label].TP, statsInfo_[label].FN); - os << "positive_label=" << label << " precision=" << precision - << " recall=" << recall - << " F1-score=" << calcF1Score(precision, recall); + labelCallback(label, precision, recall, calcF1Score(precision, recall)); return; } @@ -636,21 +639,45 @@ void PrecisionRecallEvaluator::printStats(std::ostream& os) const { macroAvgPrecision /= numLabels; macroAvgRecall /= numLabels; double macroAvgF1Score = calcF1Score(macroAvgPrecision, macroAvgRecall); - os << "macro-average-precision=" << macroAvgPrecision - << " macro-average-recall=" << macroAvgRecall - << " macro-average-F1-score=" << macroAvgF1Score; double microAvgPrecision = calcPrecision(microTotalTP, microTotalFP); double microAvgRecall = calcPrecision(microTotalTP, microTotalFN); double microAvgF1Score = calcF1Score(microAvgPrecision, microAvgRecall); - if (!isMultiBinaryLabel_) { - // precision and recall are equal in this case - os << " micro-average-precision=" << microAvgPrecision; - } else { - os << " micro-average-precision=" << microAvgPrecision - << " micro-average-recall=" << microAvgRecall - << " micro-average-F1-score=" << microAvgF1Score; - } + + microAvgCallback(macroAvgPrecision, + macroAvgRecall, + macroAvgF1Score, + isMultiBinaryLabel_, + microAvgPrecision, + microAvgRecall, + microAvgF1Score); +} + +void PrecisionRecallEvaluator::printStats(std::ostream& os) const { + this->printStatsHelper( + [&os](int label, double precision, double recall, double f1) { + os << "positive_label=" << label << " precision=" << precision + << " recall=" << recall << " F1-score=" << f1; + }, + [&os](double macroAvgPrecision, + double macroAvgRecall, + double macroAvgF1Score, + bool isMultiBinaryLabel, + double microAvgPrecision, + double microAvgRecall, + double microAvgF1Score) { + os << "macro-average-precision=" << macroAvgPrecision + << " macro-average-recall=" << macroAvgRecall + << " macro-average-F1-score=" << macroAvgF1Score; + if (!isMultiBinaryLabel) { + // precision and recall are equal in this case + os << " micro-average-precision=" << microAvgPrecision; + } else { + os << " micro-average-precision=" << microAvgPrecision + << " micro-average-recall=" << microAvgRecall + << " micro-average-F1-score=" << microAvgF1Score; + } + }); } void PrecisionRecallEvaluator::calcStatsInfo(const MatrixPtr& output, @@ -731,6 +758,69 @@ void PrecisionRecallEvaluator::calcStatsInfoMulti(const MatrixPtr& output, } } +void PrecisionRecallEvaluator::storeLocalValues() const { + if (this->values_.size() == 0) { + this->printStatsHelper( + [this](int label, double precision, double recall, double f1) { + values_["positive_label"] = (double)label; + values_["precision"] = precision; + values_["recal"] = recall; + values_["F1-score"] = f1; + }, + [this](double macroAvgPrecision, + double macroAvgRecall, + double macroAvgF1Score, + bool isMultiBinaryLabel, + double microAvgPrecision, + double microAvgRecall, + double microAvgF1Score) { + values_["macro-average-precision"] = macroAvgPrecision; + values_["macro-average-recall"] = macroAvgRecall; + values_["macro-average-F1-score"] = macroAvgF1Score; + if (!isMultiBinaryLabel) { + // precision and recall are equal in this case + values_["micro-average-precision"] = microAvgPrecision; + } else { + values_["micro-average-precision"] = microAvgPrecision; + values_["micro-average-recall"] = microAvgRecall; + values_["micro-average-F1-score"] = microAvgF1Score; + } + }); + } +} + +void PrecisionRecallEvaluator::getNames(std::vector* names) { + this->storeLocalValues(); + names->clear(); + names->reserve(this->values_.size()); + for (auto it = this->values_.begin(); it != this->values_.end(); ++it) { + names->push_back(this->config_.name() + "." + it->first); + } +} + +real PrecisionRecallEvaluator::getValue(const std::string& name, + Error* err) const { + this->storeLocalValues(); + auto it = this->values_.find(name); + if (it != this->values_.end() && err != nullptr) { + *err = Error("No such key %s", name.c_str()); + return .0f; + } + + return it->second; +} + +std::string PrecisionRecallEvaluator::getType(const std::string& name, + Error* err) const { + this->storeLocalValues(); + auto it = this->values_.find(name); + if (it != this->values_.end() && err != nullptr) { + *err = Error("No such key %s", name.c_str()); + return ""; + } + return "precision_recall"; +} + void PrecisionRecallEvaluator::distributeEval(ParameterClient2* client) { size_t size = 4 * statsInfo_.size(); double* buf = new double[size]; @@ -874,6 +964,8 @@ void PnpairEvaluator::calc(std::vector& predictArray) { << " calc total special pair: " << special; } +std::string PnpairEvaluator::getTypeImpl() const { return "pnpair"; } + ClassRegistrar Evaluator::registrar_; Evaluator* Evaluator::create(const EvaluatorConfig& config) { Evaluator* evaluator = registrar_.createByType(config.type()); @@ -901,27 +993,12 @@ public: virtual void eval(const NeuralNetwork& nn) { for (const std::string& name : config_.input_layers()) { - const Argument& argu = nn.getLayer(name)->getOutput(); - if (argu.value) { - std::ostringstream os; - argu.value->print(os); - LOG(INFO) << "layer=" << name << " value matrix:\n" << os.str(); - } - if (argu.ids) { - std::ostringstream os; - argu.ids->print(os, argu.ids->getSize()); - LOG(INFO) << "layer=" << name << " ids vector:\n" << os.str(); - } - if (auto startPos = argu.sequenceStartPositions) { - std::ostringstream os; - startPos->getVector(false)->print(os, startPos->getSize()); - LOG(INFO) << "layer=" << name << " sequence pos vector:\n" << os.str(); - } - if (auto subStartPos = argu.subSequenceStartPositions) { - std::ostringstream os; - subStartPos->getVector(false)->print(os, subStartPos->getSize()); - LOG(INFO) << "layer=" << name << " sub-sequence pos vector:\n" - << os.str(); + std::vector> out; + auto err = nn.getLayerOutputValue(name, &out); + err.check(); + for (auto& each : out) { + LOG(INFO) << "layer=" << name << std::get<0>(each) << ":\n" + << std::get<1>(each); } } } diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index bf08aa07f0..bec3d6e668 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -132,6 +132,20 @@ public: return this->getValueImpl(); } + virtual std::string getValueStr(const std::string& name, + paddle::Error* err = nullptr) const { + paddle::Error localErr; + if (err == nullptr) { + err = &localErr; + } + real result = this->getValue(name, err); + if (!err->isOK()) { + return ""; + } else { + return std::to_string(result); + } + } + virtual std::string getType(const std::string& name, paddle::Error* err = nullptr) const { if (name != config_.name() && err != nullptr) { @@ -142,7 +156,9 @@ public: } protected: - virtual real getValueImpl() const { return .0f; } + virtual real getValueImpl() const { + return numSamples_ != .0 ? totalScore_ / numSamples_ : .0; + } virtual std::string getTypeImpl() const { return "base"; } @@ -261,6 +277,10 @@ private: real* clickData, real* pvData, size_t size); + + // Evaluator interface +protected: + std::string getTypeImpl() const; }; /** * @brief precision, recall and f1 score Evaluator @@ -310,6 +330,9 @@ private: IVectorPtr cpuLabel_; MatrixPtr cpuWeight_; + template + void printStatsHelper(T1 labelCallback, T2 microAvgCallback) const; + void calcStatsInfo(const MatrixPtr& output, const IVectorPtr& label, const MatrixPtr& weight); @@ -341,6 +364,15 @@ private: return 0; } } + + mutable std::unordered_map values_; + + void storeLocalValues() const; + // Evaluator interface +public: + void getNames(std::vector* names); + real getValue(const std::string& name, Error* err) const; + std::string getType(const std::string& name, Error* err) const; }; /* @@ -387,8 +419,7 @@ public: virtual void finish() { calc(predictArray_); } virtual void printStats(std::ostream& os) const { - os << " pos/neg" - << "=" << pairArray_[0] / ((pairArray_[1] <= 0) ? 1.0 : pairArray_[1]); + os << " pos/neg=" << this->getValueImpl(); } virtual void distributeEval(ParameterClient2* client) { @@ -404,6 +435,13 @@ private: IVectorPtr cpuLabel_; IVectorPtr cpuInfo_; MatrixPtr cpuWeight_; + + // Evaluator interface +protected: + real getValueImpl() const { + return pairArray_[0] / ((pairArray_[1] <= 0) ? 1.0 : pairArray_[1]); + } + std::string getTypeImpl() const; }; } // namespace paddle diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index 22051e07ee..bf76830d61 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -405,4 +405,42 @@ NeuralNetwork* NeuralNetwork::newNeuralNetwork(const std::string& name, } } +Error NeuralNetwork::getLayerOutputValue( + const std::string& layerName, + std::vector>* out) const { + auto& layers = this->config_.layers(); + auto it = std::find_if( + layers.begin(), layers.end(), [&layerName](const LayerConfig& conf) { + return conf.name() == layerName; + }); + if (it == layers.end()) { + return Error("Cannot find layer %s", layerName.c_str()); + } + auto& layer = this->getLayer(layerName); + out->reserve(4); + auto& argu = layer->getOutput(); + + if (argu.value) { + std::ostringstream os; + argu.value->print(os); + out->push_back({"value", os.str()}); + } + if (argu.ids) { + std::ostringstream os; + argu.ids->print(os, argu.ids->getSize()); + out->push_back({"ids", os.str()}); + } + if (auto startPos = argu.sequenceStartPositions) { + std::ostringstream os; + startPos->getVector(false)->print(os, startPos->getSize()); + out->push_back({"sequence pos", os.str()}); + } + if (auto subStartPos = argu.subSequenceStartPositions) { + std::ostringstream os; + subStartPos->getVector(false)->print(os, subStartPos->getSize()); + out->push_back({"sub-sequence pos", os.str()}); + } + return Error(); +} + } // namespace paddle diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.h b/paddle/gserver/gradientmachines/NeuralNetwork.h index 25af4abcf8..4d277d103f 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.h +++ b/paddle/gserver/gradientmachines/NeuralNetwork.h @@ -128,6 +128,10 @@ public: static NeuralNetwork* newNeuralNetwork(const std::string& name = "", NeuralNetwork* rootNetwork = nullptr); + inline Error __must_check getLayerOutputValue( + const std::string& layerName, + std::vector>* out) const; + protected: /** * The constructor of NeuralNetwork. diff --git a/paddle/utils/Error.h b/paddle/utils/Error.h index 2b4fbef4e0..1ae202890f 100644 --- a/paddle/utils/Error.h +++ b/paddle/utils/Error.h @@ -116,6 +116,8 @@ public: */ operator bool() const { return msg_ == nullptr; } + bool isOK() const { return *this; } + /** * @brief check this status by glog. * @note It is a temp method used during cleaning Paddle code. It will be From 5b4e7d5cc7efe48d78036621318c6649294071d1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sun, 19 Feb 2017 18:09:06 +0800 Subject: [PATCH 03/10] complete value printer --- paddle/gserver/evaluators/Evaluator.cpp | 90 ++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 97b25b04c4..7ea4ed973c 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -13,9 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/gserver/evaluators/Evaluator.h" -#include "paddle/utils/Stat.h" - #include "paddle/gserver/gradientmachines/NeuralNetwork.h" +#include "paddle/utils/Stat.h" +#include "paddle/utils/StringUtil.h" DECLARE_int32(trainer_id); @@ -801,7 +801,9 @@ void PrecisionRecallEvaluator::getNames(std::vector* names) { real PrecisionRecallEvaluator::getValue(const std::string& name, Error* err) const { this->storeLocalValues(); - auto it = this->values_.find(name); + std::vector buffers; + paddle::str::split(name, '.', &buffers); + auto it = this->values_.find(buffers[buffers.size() - 1]); if (it != this->values_.end() && err != nullptr) { *err = Error("No such key %s", name.c_str()); return .0f; @@ -812,10 +814,12 @@ real PrecisionRecallEvaluator::getValue(const std::string& name, std::string PrecisionRecallEvaluator::getType(const std::string& name, Error* err) const { - this->storeLocalValues(); - auto it = this->values_.find(name); - if (it != this->values_.end() && err != nullptr) { - *err = Error("No such key %s", name.c_str()); + Error localErr; + if (err == nullptr) { + err = &localErr; + } + this->getValue(name, err); + if (!err->isOK()) { return ""; } return "precision_recall"; @@ -989,12 +993,12 @@ static InitFunction __reg_type_auc_sum__([]() { */ class ValuePrinter : public Evaluator { public: - ValuePrinter() {} - virtual void eval(const NeuralNetwork& nn) { + layerOutputs_.clear(); for (const std::string& name : config_.input_layers()) { auto& argu = nn.getLayer(name)->getOutput(); - std::unordered_map out; + layerOutputs_[name] = std::unordered_map(); + auto& out = layerOutputs_[name]; argu.getValueString(&out); for (auto field : {"value", "id", "sequence pos", "sub-sequence pos"}) { auto it = out.find(field); @@ -1008,6 +1012,72 @@ public: virtual void updateSamplesNum(const std::vector& arguments) {} virtual real evalImp(std::vector& arguments) { return 0; } + +private: + std::unordered_map> + layerOutputs_; + + // Evaluator interface +public: + void getNames(std::vector* names) { + for (auto layerIt = layerOutputs_.begin(); layerIt != layerOutputs_.end(); + ++layerIt) { + for (auto it = layerIt->second.begin(); it != layerIt->second.end(); + ++it) { + names->push_back(config_.name() + "." + layerIt->first + "." + + it->second); + } + } + } + + real getValue(const std::string& name, Error* err) const { + (void)(name); + if (err != nullptr) { + *err = Error( + "ValuePrinter do not support getValue, use getValueString instead."); + } + return .0f; + } + std::string getValueStr(const std::string& name, Error* err) const { + std::vector buffer; + str::split(name, '.', &buffer); + if (buffer.size() < 2) { + if (err != nullptr) { + *err = Error("No such key %s", name.c_str()); + } + return ""; + } + auto fieldName = buffer[buffer.size() - 1]; + auto layerName = buffer[buffer.size() - 2]; + auto layerIt = layerOutputs_.find(layerName); + if (layerIt == layerOutputs_.end()) { + if (err != nullptr) { + *err = Error("No such layer %s", layerName.c_str()); + } + return ""; + } + + auto fieldIt = layerIt->second.find(fieldName); + if (fieldIt == layerIt->second.end()) { + if (err != nullptr) { + *err = Error("No such value field %s", fieldName.c_str()); + } + return ""; + } + + return fieldIt->second; + } + std::string getType(const std::string& name, Error* err) const { + Error localErr; + if (err == nullptr) { + err = &localErr; + } + this->getValueStr(name, err); + if (!err->isOK()) { + return ""; + } + return "value_printer"; + } }; REGISTER_EVALUATOR(value_printer, ValuePrinter); /** From 6deb33d9dcd3db9203b98de0f6fb0d0e83cc2176 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sun, 19 Feb 2017 20:21:13 +0800 Subject: [PATCH 04/10] Complete combined evaluator --- paddle/gserver/evaluators/Evaluator.cpp | 32 ++++++++++--- paddle/gserver/evaluators/Evaluator.h | 20 ++++++++- .../gradientmachines/NeuralNetwork.cpp | 45 ++++++++++++++++++- 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index cde47f5720..29b5284fe5 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -102,6 +102,10 @@ public: virtual void distributeEval(ParameterClient2* client) { mergeResultsOfAllClients(client); } + + // Evaluator interface +protected: + std::string getTypeImpl() const { return "classification_error"; } }; /** @@ -140,6 +144,10 @@ public: virtual void distributeEval(ParameterClient2* client) { mergeResultsOfAllClients(client); } + + // Evaluator interface +protected: + std::string getTypeImpl() const { return "seq_classification_error"; } }; REGISTER_EVALUATOR(seq_classification_error, SequenceClassificationErrorEvaluator); @@ -230,6 +238,10 @@ public: private: IVectorPtr cpuLabel_; MatrixPtr cpuWeight_; + + // Evaluator interface +protected: + std::string getTypeImpl() const { return "sum"; } }; /** * @brief column sum Evaluator @@ -337,10 +349,18 @@ public: } private: - ColumnSumEvaluator() {} int32_t colIdx_; size_t colNum_; MatrixPtr sum_; /* cpu matrix */ + + // Evaluator interface +protected: + std::string getTypeImpl() const { + if (colIdx_ == -1) + return "last-column-sum"; + else + return "column-sum"; + } }; void AucEvaluator::start() { @@ -791,7 +811,6 @@ void PrecisionRecallEvaluator::storeLocalValues() const { void PrecisionRecallEvaluator::getNames(std::vector* names) { this->storeLocalValues(); - names->clear(); names->reserve(this->values_.size()); for (auto it = this->values_.begin(); it != this->values_.end(); ++it) { names->push_back(this->config_.name() + "." + it->first); @@ -1080,12 +1099,13 @@ public: } }; REGISTER_EVALUATOR(value_printer, ValuePrinter); + /** * @brief print gradient of each layer. * * The config file api is gradient_printer_evaluator. */ -class GradientPrinter : public Evaluator { +class GradientPrinter : public NotGetableEvaluator { public: virtual void eval(const NeuralNetwork& nn) { for (const std::string& name : config_.input_layers()) { @@ -1108,7 +1128,7 @@ REGISTER_EVALUATOR(gradient_printer, GradientPrinter); * * The config file api is maxid_printer_evaluator. */ -class MaxIdPrinter : public Evaluator { +class MaxIdPrinter : public NotGetableEvaluator { private: IVectorPtr maxIds_; MatrixPtr maxValues_; @@ -1150,7 +1170,7 @@ REGISTER_EVALUATOR(max_id_printer, MaxIdPrinter); * * The config file api is maxframe_printer_evaluator. */ -class MaxFramePrinter : public Evaluator { +class MaxFramePrinter : public NotGetableEvaluator { private: IVectorPtr maxIds_; MatrixPtr maxValues_; @@ -1237,7 +1257,7 @@ REGISTER_EVALUATOR(max_frame_printer, MaxFramePrinter); * The config file api is seqtext_printer_evaluator. * */ -class SequenceTextPrinter : public Evaluator { +class SequenceTextPrinter : public NotGetableEvaluator { private: /// dict_file, which contains a list of tokens std::vector dict_; diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index bec3d6e668..d5b2a63e35 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -119,7 +119,6 @@ public: static ClassRegistrar registrar_; virtual void getNames(std::vector* names) { - names->clear(); names->push_back(config_.name()); } @@ -168,6 +167,25 @@ protected: double totalScore_; }; +class NotGetableEvaluator : public Evaluator { + // Evaluator interface +public: + void getNames(std::vector* names) {} + + real getValue(const std::string& name, Error* err) const { + if (err != nullptr) { + *err = Error("Not implemented"); + } + return .0f; + } + std::string getType(const std::string& name, Error* err) const { + if (err != nullptr) { + *err = Error("Not implemented"); + } + return ""; + } +}; + class DummyEvaluator : public Evaluator { public: DummyEvaluator() {} diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index 22051e07ee..277fb6d8db 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -306,7 +306,6 @@ void NeuralNetwork::onPassEnd() { class CombinedEvaluator : public Evaluator { public: - CombinedEvaluator() {} void addEvaluator(std::unique_ptr&& evaluator) { evaluators_.emplace_back(std::move(evaluator)); } @@ -346,6 +345,50 @@ public: protected: std::vector> evaluators_; + + // Evaluator interface +public: + void getNames(std::vector* names) { + for (auto& eval : evaluators_) { + eval->getNames(names); + } + } + + real getValue(const std::string& name, Error* err) const { + return this->getMethodHelper( + name, err, [&name, err](const std::unique_ptr& eval) { + return eval->getValue(name, err); + }); + } + std::string getValueStr(const std::string& name, Error* err) const { + return this->getMethodHelper( + name, err, [&name, err](const std::unique_ptr& eval) { + return eval->getValueStr(name, err); + }); + } + std::string getType(const std::string& name, Error* err) const { + return this->getMethodHelper( + name, err, [&name, err](const std::unique_ptr& eval) { + return eval->getType(name, err); + }); + } + +private: + template + T getMethodHelper(const std::string& name, + Error* err, + const std::function&)>& + callback) const { + for (auto& eval : evaluators_) { + std::vector names; + eval->getNames(&names); + if (std::find(names.begin(), names.end(), name) != names.end()) { + return callback(eval); + } + } + if (err != nullptr) *err = Error("No such key %s", name.c_str()); + return T(); + } }; Evaluator* NeuralNetwork::makeEvaluator() const { From 1100366803c10aa9cbd7265616d286e1b38681c0 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sun, 19 Feb 2017 20:43:04 +0800 Subject: [PATCH 05/10] Add unittests & fix some bugs. --- paddle/gserver/evaluators/CTCErrorEvaluator.cpp | 2 +- paddle/gserver/evaluators/Evaluator.cpp | 6 ++++-- paddle/gserver/tests/test_Evaluator.cpp | 12 ++++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp index 05aa6c012a..132119015f 100644 --- a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp +++ b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp @@ -20,7 +20,7 @@ namespace paddle { /** * calculate sequence-to-sequence edit distance */ -class CTCErrorEvaluator : public Evaluator { +class CTCErrorEvaluator : public NotGetableEvaluator { private: MatrixPtr outActivations_; int numTimes_, numClasses_, numSequences_, blank_; diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 29b5284fe5..42a877954b 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -823,8 +823,10 @@ real PrecisionRecallEvaluator::getValue(const std::string& name, std::vector buffers; paddle::str::split(name, '.', &buffers); auto it = this->values_.find(buffers[buffers.size() - 1]); - if (it != this->values_.end() && err != nullptr) { - *err = Error("No such key %s", name.c_str()); + if (it == this->values_.end()) { // not found + if (err != nullptr) { + *err = Error("No such key %s", name.c_str()); + } return .0f; } diff --git a/paddle/gserver/tests/test_Evaluator.cpp b/paddle/gserver/tests/test_Evaluator.cpp index 8165eb8269..100cf07809 100644 --- a/paddle/gserver/tests/test_Evaluator.cpp +++ b/paddle/gserver/tests/test_Evaluator.cpp @@ -110,6 +110,18 @@ void testEvaluator(TestConfig testConf, testEvaluator->finish(); LOG(INFO) << *testEvaluator; + std::vector names; + testEvaluator->getNames(&names); + paddle::Error err; + for (auto& name : names) { + auto value = testEvaluator->getValueStr(name, &err); + ASSERT_TRUE(err.isOK()); + LOG(INFO) << name << " " << value; + auto tp = testEvaluator->getType(name, &err); + ASSERT_TRUE(err.isOK()); + ASSERT_EQ(testConf.evaluatorConfig.type(), tp); + } + double totalScore2 = 0.0; if (testConf.testAccumulate) { testEvaluator->start(); From bb751b7f2c274879fdc91cb5f84fb1d60b608faf Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 20 Feb 2017 15:20:14 +0800 Subject: [PATCH 06/10] Add documentation for codes. * also remove unused getValueStr --- paddle/gserver/evaluators/Evaluator.h | 71 ++++++++++++++----- .../gradientmachines/NeuralNetwork.cpp | 17 +++-- paddle/gserver/tests/test_Evaluator.cpp | 2 +- 3 files changed, 67 insertions(+), 23 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index d5b2a63e35..a5694a088c 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -118,33 +118,55 @@ public: static ClassRegistrar registrar_; + /** + * @brief getNames will return all field names of current evaluator. + * + * The format of name is `evaluator_name.evaluator_fields`. If the evaluator + * has multiple field, the name could be `evaluator_name.field1`. For example + * the PrecisionRecallEvaluator contains `precision`, `recall` fields. The get + * names will return `precision_recall_evaluator.precision`, + * `precision_recall.recal`, etc. + * + * Also, if current Evaluator is a combined evaluator. getNames will return + * all names of all evaluators inside the combined evaluator. + * + * @param names [out]: the field names of current evaluator. + * @note Never clear the names parameter inside getNames. + */ virtual void getNames(std::vector* names) { names->push_back(config_.name()); } + /** + * @brief getValue will return the current evaluate value of one field. + * + * @param name: The field name of current evaluator. + * @param err [out]: The error state. nullptr means don't care. + * + * @return The evaluate value(metric). + */ virtual real getValue(const std::string& name, paddle::Error* err = nullptr) const { - if (name != config_.name() && err != nullptr) { - *err = paddle::Error("no such name of evaluator %s", name.c_str()); + if (name != config_.name()) { + if (err != nullptr) { + *err = paddle::Error("no such name of evaluator %s", name.c_str()); + } return .0f; } return this->getValueImpl(); } - virtual std::string getValueStr(const std::string& name, - paddle::Error* err = nullptr) const { - paddle::Error localErr; - if (err == nullptr) { - err = &localErr; - } - real result = this->getValue(name, err); - if (!err->isOK()) { - return ""; - } else { - return std::to_string(result); - } - } - + /** + * @brief getType will return the evaluator type by field name. + * + * Evaluate Type is the current type of evaluator in string. Such as 'auc', + * 'precision_recall'. In combined evaluator, different name may get different + * evaluate type because it could be evaluated by different evaluator inside. + * + * @param name: The field name of current Evaluator. + * @param err: The error state. nullptr means don't care. + * @return the evaluator type string. + */ virtual std::string getType(const std::string& name, paddle::Error* err = nullptr) const { if (name != config_.name() && err != nullptr) { @@ -155,10 +177,22 @@ public: } protected: + /** + * @brief getValueImpl The simplest way to define getValue result. If this + * evaluator doesn't contain multiple fields, and do not throw any error, just + * implemented this method to get the evaluate result(metric). + * @return Evaluate result(metric). + */ virtual real getValueImpl() const { return numSamples_ != .0 ? totalScore_ / numSamples_ : .0; } + /** + * @brief getTypeImpl The simplest way to define getType result. If this + * evaluator doesn't combine many evaluators, the get type should only return + * itself type. + * @return Evaluator type. + */ virtual std::string getTypeImpl() const { return "base"; } protected: @@ -167,6 +201,11 @@ protected: double totalScore_; }; +/** + * @brief The NotGetableEvaluator class is the base class of evaluator that + * cannot get value in runtime. The most NotGetableEvaluator is Printer + * Evaluator, which is only used to debug network configuration. + */ class NotGetableEvaluator : public Evaluator { // Evaluator interface public: diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index 277fb6d8db..4d2bdf0dc9 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -348,24 +348,29 @@ protected: // Evaluator interface public: + /** + * @brief getNames will return all inside evaluators' names. + * @param names [out]: return names. + */ void getNames(std::vector* names) { for (auto& eval : evaluators_) { eval->getNames(names); } } + /** + * @brief getValue could get all inside evaluators' value. + */ real getValue(const std::string& name, Error* err) const { return this->getMethodHelper( name, err, [&name, err](const std::unique_ptr& eval) { return eval->getValue(name, err); }); } - std::string getValueStr(const std::string& name, Error* err) const { - return this->getMethodHelper( - name, err, [&name, err](const std::unique_ptr& eval) { - return eval->getValueStr(name, err); - }); - } + + /** + * @brief getType could get all inside evaluators' type. + */ std::string getType(const std::string& name, Error* err) const { return this->getMethodHelper( name, err, [&name, err](const std::unique_ptr& eval) { diff --git a/paddle/gserver/tests/test_Evaluator.cpp b/paddle/gserver/tests/test_Evaluator.cpp index 100cf07809..07f486b1f4 100644 --- a/paddle/gserver/tests/test_Evaluator.cpp +++ b/paddle/gserver/tests/test_Evaluator.cpp @@ -114,7 +114,7 @@ void testEvaluator(TestConfig testConf, testEvaluator->getNames(&names); paddle::Error err; for (auto& name : names) { - auto value = testEvaluator->getValueStr(name, &err); + auto value = testEvaluator->getValue(name, &err); ASSERT_TRUE(err.isOK()); LOG(INFO) << name << " " << value; auto tp = testEvaluator->getType(name, &err); From b1ab8b56cfbad21ba1b70d010392031e72cf5c71 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 23 Feb 2017 15:03:34 +0800 Subject: [PATCH 07/10] Use plain C++ 03 to implement getStatsInfo. --- paddle/gserver/evaluators/Evaluator.cpp | 196 ++++++++++++------------ paddle/gserver/evaluators/Evaluator.h | 13 +- paddle/utils/Error.h | 28 ++-- paddle/utils/tests/test_Error.cpp | 8 +- 4 files changed, 129 insertions(+), 116 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 89f9543801..8fce8df8a3 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -626,78 +626,34 @@ real PrecisionRecallEvaluator::evalImp(std::vector& arguments) { return 0; } -template -void PrecisionRecallEvaluator::printStatsHelper(T1 labelCallback, - T2 microAvgCallback) const { - int label = config_.positive_label(); - if (label != -1) { - CHECK(label >= 0 && label < (int)statsInfo_.size()) - << "positive_label [" << label << "] should be in range [0, " - << statsInfo_.size() << ")"; - double precision = - calcPrecision(statsInfo_[label].TP, statsInfo_[label].FP); - double recall = calcRecall(statsInfo_[label].TP, statsInfo_[label].FN); - labelCallback(label, precision, recall, calcF1Score(precision, recall)); - return; - } - - // micro average method: precision = (TP1+TP2)/(TP1+FP1+TP2+FP2) - // macro average method: precision = (precision1+precision2)/2 - double microTotalTP = 0; - double microTotalFP = 0; - double microTotalFN = 0; - double macroAvgPrecision = 0; - double macroAvgRecall = 0; - size_t numLabels = statsInfo_.size(); - for (size_t i = 0; i < numLabels; ++i) { - microTotalTP += statsInfo_[i].TP; - microTotalFP += statsInfo_[i].FP; - microTotalFN += statsInfo_[i].FN; - macroAvgPrecision += calcPrecision(statsInfo_[i].TP, statsInfo_[i].FP); - macroAvgRecall += calcRecall(statsInfo_[i].TP, statsInfo_[i].FN); - } - macroAvgPrecision /= numLabels; - macroAvgRecall /= numLabels; - double macroAvgF1Score = calcF1Score(macroAvgPrecision, macroAvgRecall); - - double microAvgPrecision = calcPrecision(microTotalTP, microTotalFP); - double microAvgRecall = calcPrecision(microTotalTP, microTotalFN); - double microAvgF1Score = calcF1Score(microAvgPrecision, microAvgRecall); - - microAvgCallback(macroAvgPrecision, - macroAvgRecall, - macroAvgF1Score, - isMultiBinaryLabel_, - microAvgPrecision, - microAvgRecall, - microAvgF1Score); -} - void PrecisionRecallEvaluator::printStats(std::ostream& os) const { - this->printStatsHelper( - [&os](int label, double precision, double recall, double f1) { - os << "positive_label=" << label << " precision=" << precision - << " recall=" << recall << " F1-score=" << f1; - }, - [&os](double macroAvgPrecision, - double macroAvgRecall, - double macroAvgF1Score, - bool isMultiBinaryLabel, - double microAvgPrecision, - double microAvgRecall, - double microAvgF1Score) { - os << "macro-average-precision=" << macroAvgPrecision - << " macro-average-recall=" << macroAvgRecall - << " macro-average-F1-score=" << macroAvgF1Score; - if (!isMultiBinaryLabel) { - // precision and recall are equal in this case - os << " micro-average-precision=" << microAvgPrecision; - } else { - os << " micro-average-precision=" << microAvgPrecision - << " micro-average-recall=" << microAvgRecall - << " micro-average-F1-score=" << microAvgF1Score; - } - }); + double precision, recall, f1, macroAvgPrecision, macroAvgRecall, + macroAvgF1Score, microAvgPrecision, microAvgRecall, microAvgF1Score; + bool containMacroMicroInfo = getStatsInfo(&precision, + &recall, + &f1, + ¯oAvgPrecision, + ¯oAvgRecall, + ¯oAvgF1Score, + µAvgPrecision, + µAvgRecall, + µAvgF1Score); + os << "positive_label=" << config_.positive_label() + << " precision=" << precision << " recall=" << recall + << " F1-score=" << f1; + if (containMacroMicroInfo) { + os << "macro-average-precision=" << macroAvgPrecision + << " macro-average-recall=" << macroAvgRecall + << " macro-average-F1-score=" << macroAvgF1Score; + if (!isMultiBinaryLabel_) { + // precision and recall are equal in this case + os << " micro-average-precision=" << microAvgPrecision; + } else { + os << " micro-average-precision=" << microAvgPrecision + << " micro-average-recall=" << microAvgRecall + << " micro-average-F1-score=" << microAvgF1Score; + } + }; } void PrecisionRecallEvaluator::calcStatsInfo(const MatrixPtr& output, @@ -780,32 +736,33 @@ void PrecisionRecallEvaluator::calcStatsInfoMulti(const MatrixPtr& output, void PrecisionRecallEvaluator::storeLocalValues() const { if (this->values_.size() == 0) { - this->printStatsHelper( - [this](int label, double precision, double recall, double f1) { - values_["positive_label"] = (double)label; - values_["precision"] = precision; - values_["recal"] = recall; - values_["F1-score"] = f1; - }, - [this](double macroAvgPrecision, - double macroAvgRecall, - double macroAvgF1Score, - bool isMultiBinaryLabel, - double microAvgPrecision, - double microAvgRecall, - double microAvgF1Score) { - values_["macro-average-precision"] = macroAvgPrecision; - values_["macro-average-recall"] = macroAvgRecall; - values_["macro-average-F1-score"] = macroAvgF1Score; - if (!isMultiBinaryLabel) { - // precision and recall are equal in this case - values_["micro-average-precision"] = microAvgPrecision; - } else { - values_["micro-average-precision"] = microAvgPrecision; - values_["micro-average-recall"] = microAvgRecall; - values_["micro-average-F1-score"] = microAvgF1Score; - } - }); + double precision, recall, f1, macroAvgPrecision, macroAvgRecall, + macroAvgF1Score, microAvgPrecision, microAvgRecall, microAvgF1Score; + bool containMacroMicroInfo = getStatsInfo(&precision, + &recall, + &f1, + ¯oAvgPrecision, + ¯oAvgRecall, + ¯oAvgF1Score, + µAvgPrecision, + µAvgRecall, + µAvgF1Score); + values_["precision"] = precision; + values_["recal"] = recall; + values_["F1-score"] = f1; + if (containMacroMicroInfo) { + values_["macro-average-precision"] = macroAvgPrecision; + values_["macro-average-recall"] = macroAvgRecall; + values_["macro-average-F1-score"] = macroAvgF1Score; + if (!isMultiBinaryLabel_) { + // precision and recall are equal in this case + values_["micro-average-precision"] = microAvgPrecision; + } else { + values_["micro-average-precision"] = microAvgPrecision; + values_["micro-average-recall"] = microAvgRecall; + values_["micro-average-F1-score"] = microAvgF1Score; + } + } } } @@ -865,6 +822,51 @@ void PrecisionRecallEvaluator::distributeEval(ParameterClient2* client) { delete[] buf; } +bool PrecisionRecallEvaluator::getStatsInfo(double* precision, + double* recall, + double* f1, + double* macroAvgPrecision, + double* macroAvgRecall, + double* macroAvgF1Score, + double* microAvgPrecision, + double* microAvgRecall, + double* microAvgF1Score) const { + int label = config_.positive_label(); + if (label != -1) { + CHECK(label >= 0 && label < (int)statsInfo_.size()) + << "positive_label [" << label << "] should be in range [0, " + << statsInfo_.size() << ")"; + *precision = calcPrecision(statsInfo_[label].TP, statsInfo_[label].FP); + *recall = calcRecall(statsInfo_[label].TP, statsInfo_[label].FN); + *f1 = calcF1Score(*precision, *recall); + return false; + } + + // micro average method: precision = (TP1+TP2)/(TP1+FP1+TP2+FP2) + // macro average method: precision = (precision1+precision2)/2 + double microTotalTP = 0; + double microTotalFP = 0; + double microTotalFN = 0; + *macroAvgPrecision = 0; + *macroAvgRecall = 0; + size_t numLabels = statsInfo_.size(); + for (size_t i = 0; i < numLabels; ++i) { + microTotalTP += statsInfo_[i].TP; + microTotalFP += statsInfo_[i].FP; + microTotalFN += statsInfo_[i].FN; + *macroAvgPrecision += calcPrecision(statsInfo_[i].TP, statsInfo_[i].FP); + *macroAvgRecall += calcRecall(statsInfo_[i].TP, statsInfo_[i].FN); + } + *macroAvgPrecision /= numLabels; + *macroAvgRecall /= numLabels; + *macroAvgF1Score = calcF1Score(*macroAvgPrecision, *macroAvgRecall); + + *microAvgPrecision = calcPrecision(microTotalTP, microTotalFP); + *microAvgRecall = calcPrecision(microTotalTP, microTotalFN); + *microAvgF1Score = calcF1Score(*microAvgPrecision, *microAvgRecall); + return true; +} + REGISTER_EVALUATOR(pnpair, PnpairEvaluator); void PnpairEvaluator::start() { Evaluator::start(); diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index a5694a088c..eb19e6f4dd 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -125,7 +125,7 @@ public: * has multiple field, the name could be `evaluator_name.field1`. For example * the PrecisionRecallEvaluator contains `precision`, `recall` fields. The get * names will return `precision_recall_evaluator.precision`, - * `precision_recall.recal`, etc. + * `precision_recall_evaluator.recal`, etc. * * Also, if current Evaluator is a combined evaluator. getNames will return * all names of all evaluators inside the combined evaluator. @@ -387,8 +387,15 @@ private: IVectorPtr cpuLabel_; MatrixPtr cpuWeight_; - template - void printStatsHelper(T1 labelCallback, T2 microAvgCallback) const; + bool getStatsInfo(double* precision, + double* recall, + double* f1, + double* macroAvgPrecision, + double* macroAvgRecall, + double* macroAvgF1Score, + double* microAvgPrecision, + double* microAvgRecall, + double* microAvgF1Score) const; void calcStatsInfo(const MatrixPtr& output, const IVectorPtr& label, diff --git a/paddle/utils/Error.h b/paddle/utils/Error.h index 1ae202890f..cda1b5c37d 100644 --- a/paddle/utils/Error.h +++ b/paddle/utils/Error.h @@ -37,10 +37,10 @@ namespace paddle { * * Error __must_check bar() { * // do something. - * Status s = foo(); // invoke other method return status. - * if (!s) return s; + * Error err = foo(); // invoke other method return status. + * if (err) return err; * // do something else. - * return Status(); + * return Error(); * } * @endcode{cpp} * @@ -53,8 +53,8 @@ namespace paddle { * * int foo(Error* error) { * // Do something. - * Error s = bar(); - * if (!s) { + * Error err = bar(); + * if (err) { * *error = s; * return 0; * } @@ -68,10 +68,10 @@ namespace paddle { * } * * Error foobar() { - * Error s; + * Error err; * // do something. - * foo(&s); - * if (!s) return s; + * foo(&err); + * if (err) return err; * } * @endcode{cpp} * @@ -112,18 +112,22 @@ public: } /** - * @brief operator bool, return True if there is no error. + * @brief operator bool, return True if there is something error. */ - operator bool() const { return msg_ == nullptr; } + operator bool() const { return !this->isOK(); } - bool isOK() const { return *this; } + /** + * @brief isOK return True if there is no error. + * @return True if no error. + */ + bool isOK() const { return msg_ == nullptr; } /** * @brief check this status by glog. * @note It is a temp method used during cleaning Paddle code. It will be * removed later. */ - void check() const { CHECK(*this) << msg(); } + void check() const { CHECK(this->isOK()) << msg(); } private: std::shared_ptr msg_; diff --git a/paddle/utils/tests/test_Error.cpp b/paddle/utils/tests/test_Error.cpp index 85156466e2..fdf326b17a 100644 --- a/paddle/utils/tests/test_Error.cpp +++ b/paddle/utils/tests/test_Error.cpp @@ -18,17 +18,17 @@ limitations under the License. */ TEST(Error, testAll) { paddle::Error error; - ASSERT_TRUE(error); - error = paddle::Error("I'm the error"); ASSERT_FALSE(error); + error = paddle::Error("I'm the error"); + ASSERT_TRUE(error); ASSERT_STREQ("I'm the error", error.msg()); error = paddle::Error("error2"); - ASSERT_FALSE(error); + ASSERT_TRUE(error); ASSERT_STREQ("error2", error.msg()); int i = 3; auto error3 = paddle::Error("error%d", i); - ASSERT_FALSE(error3); + ASSERT_TRUE(error3); ASSERT_STREQ("error3", error3.msg()); } From b6ac64a36bf260780d18dc6722f6ccdd0c96aba0 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 23 Feb 2017 15:06:30 +0800 Subject: [PATCH 08/10] Remove unnecessary error default value. --- paddle/gserver/evaluators/Evaluator.cpp | 8 +------ paddle/gserver/evaluators/Evaluator.h | 21 +++++++------------ .../gradientmachines/NeuralNetwork.cpp | 2 +- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 8fce8df8a3..6c1c2f62be 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -781,9 +781,7 @@ real PrecisionRecallEvaluator::getValue(const std::string& name, paddle::str::split(name, '.', &buffers); auto it = this->values_.find(buffers[buffers.size() - 1]); if (it == this->values_.end()) { // not found - if (err != nullptr) { - *err = Error("No such key %s", name.c_str()); - } + *err = Error("No such key %s", name.c_str()); return .0f; } @@ -792,10 +790,6 @@ real PrecisionRecallEvaluator::getValue(const std::string& name, std::string PrecisionRecallEvaluator::getType(const std::string& name, Error* err) const { - Error localErr; - if (err == nullptr) { - err = &localErr; - } this->getValue(name, err); if (!err->isOK()) { return ""; diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index eb19e6f4dd..6a122704ea 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -141,16 +141,13 @@ public: * @brief getValue will return the current evaluate value of one field. * * @param name: The field name of current evaluator. - * @param err [out]: The error state. nullptr means don't care. + * @param err [out]: The error state. * * @return The evaluate value(metric). */ - virtual real getValue(const std::string& name, - paddle::Error* err = nullptr) const { + virtual real getValue(const std::string& name, paddle::Error* err) const { if (name != config_.name()) { - if (err != nullptr) { - *err = paddle::Error("no such name of evaluator %s", name.c_str()); - } + *err = paddle::Error("no such name of evaluator %s", name.c_str()); return .0f; } return this->getValueImpl(); @@ -168,8 +165,8 @@ public: * @return the evaluator type string. */ virtual std::string getType(const std::string& name, - paddle::Error* err = nullptr) const { - if (name != config_.name() && err != nullptr) { + paddle::Error* err) const { + if (name != config_.name()) { *err = paddle::Error("no such name of evaluator %s", name.c_str()); return std::string(); } @@ -212,15 +209,11 @@ public: void getNames(std::vector* names) {} real getValue(const std::string& name, Error* err) const { - if (err != nullptr) { - *err = Error("Not implemented"); - } + *err = Error("Not implemented"); return .0f; } std::string getType(const std::string& name, Error* err) const { - if (err != nullptr) { - *err = Error("Not implemented"); - } + *err = Error("Not implemented"); return ""; } }; diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index 4d2bdf0dc9..273a9111c3 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -391,7 +391,7 @@ private: return callback(eval); } } - if (err != nullptr) *err = Error("No such key %s", name.c_str()); + *err = Error("No such key %s", name.c_str()); return T(); } }; From 2bb7fd817718794eb3fdef87207bf4a7596c606c Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 23 Feb 2017 15:09:09 +0800 Subject: [PATCH 09/10] Remove padde namespace in some code. --- paddle/gserver/evaluators/Evaluator.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index 6a122704ea..c4110ec1c0 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -145,9 +145,9 @@ public: * * @return The evaluate value(metric). */ - virtual real getValue(const std::string& name, paddle::Error* err) const { + virtual real getValue(const std::string& name, Error* err) const { if (name != config_.name()) { - *err = paddle::Error("no such name of evaluator %s", name.c_str()); + *err = Error("no such name of evaluator %s", name.c_str()); return .0f; } return this->getValueImpl(); @@ -164,10 +164,9 @@ public: * @param err: The error state. nullptr means don't care. * @return the evaluator type string. */ - virtual std::string getType(const std::string& name, - paddle::Error* err) const { + virtual std::string getType(const std::string& name, Error* err) const { if (name != config_.name()) { - *err = paddle::Error("no such name of evaluator %s", name.c_str()); + *err = Error("no such name of evaluator %s", name.c_str()); return std::string(); } return this->getTypeImpl(); From 9087e3a9bb9aac10236b513c5c9a52af1590b3a7 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 23 Feb 2017 16:18:19 +0800 Subject: [PATCH 10/10] Follow comments to use struct get return value. --- paddle/gserver/evaluators/Evaluator.cpp | 108 ++++++++++-------------- paddle/gserver/evaluators/Evaluator.h | 22 +++-- 2 files changed, 56 insertions(+), 74 deletions(-) diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 5911a9ec59..9db6d252d9 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -647,33 +647,24 @@ real PrecisionRecallEvaluator::evalImp(std::vector& arguments) { } void PrecisionRecallEvaluator::printStats(std::ostream& os) const { - double precision, recall, f1, macroAvgPrecision, macroAvgRecall, - macroAvgF1Score, microAvgPrecision, microAvgRecall, microAvgF1Score; - bool containMacroMicroInfo = getStatsInfo(&precision, - &recall, - &f1, - ¯oAvgPrecision, - ¯oAvgRecall, - ¯oAvgF1Score, - µAvgPrecision, - µAvgRecall, - µAvgF1Score); + PrintStatsInfo info; + bool containMacroMicroInfo = getStatsInfo(&info); os << "positive_label=" << config_.positive_label() - << " precision=" << precision << " recall=" << recall - << " F1-score=" << f1; + << " precision=" << info.precision << " recall=" << info.recall + << " F1-score=" << info.f1; if (containMacroMicroInfo) { - os << "macro-average-precision=" << macroAvgPrecision - << " macro-average-recall=" << macroAvgRecall - << " macro-average-F1-score=" << macroAvgF1Score; + os << "macro-average-precision=" << info.macroAvgPrecision + << " macro-average-recall=" << info.macroAvgRecall + << " macro-average-F1-score=" << info.macroAvgF1Score; if (!isMultiBinaryLabel_) { // precision and recall are equal in this case - os << " micro-average-precision=" << microAvgPrecision; + os << " micro-average-precision=" << info.microAvgPrecision; } else { - os << " micro-average-precision=" << microAvgPrecision - << " micro-average-recall=" << microAvgRecall - << " micro-average-F1-score=" << microAvgF1Score; + os << " micro-average-precision=" << info.microAvgPrecision + << " micro-average-recall=" << info.microAvgRecall + << " micro-average-F1-score=" << info.microAvgF1Score; } - }; + } } void PrecisionRecallEvaluator::calcStatsInfo(const MatrixPtr& output, @@ -756,31 +747,22 @@ void PrecisionRecallEvaluator::calcStatsInfoMulti(const MatrixPtr& output, void PrecisionRecallEvaluator::storeLocalValues() const { if (this->values_.size() == 0) { - double precision, recall, f1, macroAvgPrecision, macroAvgRecall, - macroAvgF1Score, microAvgPrecision, microAvgRecall, microAvgF1Score; - bool containMacroMicroInfo = getStatsInfo(&precision, - &recall, - &f1, - ¯oAvgPrecision, - ¯oAvgRecall, - ¯oAvgF1Score, - µAvgPrecision, - µAvgRecall, - µAvgF1Score); - values_["precision"] = precision; - values_["recal"] = recall; - values_["F1-score"] = f1; + PrintStatsInfo info; + bool containMacroMicroInfo = getStatsInfo(&info); + values_["precision"] = info.precision; + values_["recal"] = info.recall; + values_["F1-score"] = info.f1; if (containMacroMicroInfo) { - values_["macro-average-precision"] = macroAvgPrecision; - values_["macro-average-recall"] = macroAvgRecall; - values_["macro-average-F1-score"] = macroAvgF1Score; + values_["macro-average-precision"] = info.macroAvgPrecision; + values_["macro-average-recall"] = info.macroAvgRecall; + values_["macro-average-F1-score"] = info.macroAvgF1Score; if (!isMultiBinaryLabel_) { // precision and recall are equal in this case - values_["micro-average-precision"] = microAvgPrecision; + values_["micro-average-precision"] = info.microAvgPrecision; } else { - values_["micro-average-precision"] = microAvgPrecision; - values_["micro-average-recall"] = microAvgRecall; - values_["micro-average-F1-score"] = microAvgF1Score; + values_["micro-average-precision"] = info.microAvgPrecision; + values_["micro-average-recall"] = info.microAvgRecall; + values_["micro-average-F1-score"] = info.microAvgF1Score; } } } @@ -836,23 +818,16 @@ void PrecisionRecallEvaluator::distributeEval(ParameterClient2* client) { delete[] buf; } -bool PrecisionRecallEvaluator::getStatsInfo(double* precision, - double* recall, - double* f1, - double* macroAvgPrecision, - double* macroAvgRecall, - double* macroAvgF1Score, - double* microAvgPrecision, - double* microAvgRecall, - double* microAvgF1Score) const { +bool PrecisionRecallEvaluator::getStatsInfo( + PrecisionRecallEvaluator::PrintStatsInfo* info) const { int label = config_.positive_label(); if (label != -1) { CHECK(label >= 0 && label < (int)statsInfo_.size()) << "positive_label [" << label << "] should be in range [0, " << statsInfo_.size() << ")"; - *precision = calcPrecision(statsInfo_[label].TP, statsInfo_[label].FP); - *recall = calcRecall(statsInfo_[label].TP, statsInfo_[label].FN); - *f1 = calcF1Score(*precision, *recall); + info->precision = calcPrecision(statsInfo_[label].TP, statsInfo_[label].FP); + info->recall = calcRecall(statsInfo_[label].TP, statsInfo_[label].FN); + info->f1 = calcF1Score(info->precision, info->recall); return false; } @@ -861,23 +836,26 @@ bool PrecisionRecallEvaluator::getStatsInfo(double* precision, double microTotalTP = 0; double microTotalFP = 0; double microTotalFN = 0; - *macroAvgPrecision = 0; - *macroAvgRecall = 0; + info->macroAvgPrecision = 0; + info->macroAvgRecall = 0; size_t numLabels = statsInfo_.size(); for (size_t i = 0; i < numLabels; ++i) { microTotalTP += statsInfo_[i].TP; microTotalFP += statsInfo_[i].FP; microTotalFN += statsInfo_[i].FN; - *macroAvgPrecision += calcPrecision(statsInfo_[i].TP, statsInfo_[i].FP); - *macroAvgRecall += calcRecall(statsInfo_[i].TP, statsInfo_[i].FN); + info->macroAvgPrecision += + calcPrecision(statsInfo_[i].TP, statsInfo_[i].FP); + info->macroAvgRecall += calcRecall(statsInfo_[i].TP, statsInfo_[i].FN); } - *macroAvgPrecision /= numLabels; - *macroAvgRecall /= numLabels; - *macroAvgF1Score = calcF1Score(*macroAvgPrecision, *macroAvgRecall); - - *microAvgPrecision = calcPrecision(microTotalTP, microTotalFP); - *microAvgRecall = calcPrecision(microTotalTP, microTotalFN); - *microAvgF1Score = calcF1Score(*microAvgPrecision, *microAvgRecall); + info->macroAvgPrecision /= numLabels; + info->macroAvgRecall /= numLabels; + info->macroAvgF1Score = + calcF1Score(info->macroAvgPrecision, info->macroAvgRecall); + + info->microAvgPrecision = calcPrecision(microTotalTP, microTotalFP); + info->microAvgRecall = calcPrecision(microTotalTP, microTotalFN); + info->microAvgF1Score = + calcF1Score(info->microAvgPrecision, info->microAvgRecall); return true; } diff --git a/paddle/gserver/evaluators/Evaluator.h b/paddle/gserver/evaluators/Evaluator.h index c4110ec1c0..b114500e2b 100644 --- a/paddle/gserver/evaluators/Evaluator.h +++ b/paddle/gserver/evaluators/Evaluator.h @@ -379,15 +379,19 @@ private: IVectorPtr cpuLabel_; MatrixPtr cpuWeight_; - bool getStatsInfo(double* precision, - double* recall, - double* f1, - double* macroAvgPrecision, - double* macroAvgRecall, - double* macroAvgF1Score, - double* microAvgPrecision, - double* microAvgRecall, - double* microAvgF1Score) const; + struct PrintStatsInfo { + double precision; + double recall; + double f1; + double macroAvgPrecision; + double macroAvgRecall; + double macroAvgF1Score; + double microAvgPrecision; + double microAvgRecall; + double microAvgF1Score; + }; + + bool getStatsInfo(PrintStatsInfo* info) const; void calcStatsInfo(const MatrixPtr& output, const IVectorPtr& label,