m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add getValue to some evaluators.

Browse Source
avx_docs
Yu Yang 8 years ago
parent 39feacb02a
commit 04eaf75c5e
5 changed files with 198 additions and 39 deletions
Show Stats Download Patch File Download Diff File

135
paddle/gserver/evaluators/Evaluator.cpp
Unescape View File

@ -538,12 +538,15 @@ double RankAucEvaluator::calcRankAuc(real* outputData,
: aucTmp / (clickSum * noClickSum);
}
std::string RankAucEvaluator::getTypeImpl() const { return "rankauc"; }
// class PrecisionRecallEvaluator
REGISTER_EVALUATOR(precision_recall, PrecisionRecallEvaluator);
void PrecisionRecallEvaluator::start() {
Evaluator::start();
statsInfo_.clear();
values_.clear();
}
real PrecisionRecallEvaluator::evalImp(std::vector<Argument>& arguments) {
@ -603,7 +606,9 @@ real PrecisionRecallEvaluator::evalImp(std::vector<Argument>& arguments) {
return 0;
}
void PrecisionRecallEvaluator::printStats(std::ostream& os) const {
template <typename T1, typename T2>
void PrecisionRecallEvaluator::printStatsHelper(T1 labelCallback,
T2 microAvgCallback) const {
int label = config_.positive_label();
if (label != -1) {
CHECK(label >= 0 && label < (int)statsInfo_.size())
@ -612,9 +617,7 @@ void PrecisionRecallEvaluator::printStats(std::ostream& os) const {
double precision =
calcPrecision(statsInfo_[label].TP, statsInfo_[label].FP);
double recall = calcRecall(statsInfo_[label].TP, statsInfo_[label].FN);
os << "positive_label=" << label << " precision=" << precision
<< " recall=" << recall
<< " F1-score=" << calcF1Score(precision, recall);
labelCallback(label, precision, recall, calcF1Score(precision, recall));
return;
}
@ -636,14 +639,37 @@ void PrecisionRecallEvaluator::printStats(std::ostream& os) const {
macroAvgPrecision /= numLabels;
macroAvgRecall /= numLabels;
double macroAvgF1Score = calcF1Score(macroAvgPrecision, macroAvgRecall);
os << "macro-average-precision=" << macroAvgPrecision
<< " macro-average-recall=" << macroAvgRecall
<< " macro-average-F1-score=" << macroAvgF1Score;
double microAvgPrecision = calcPrecision(microTotalTP, microTotalFP);
double microAvgRecall = calcPrecision(microTotalTP, microTotalFN);
double microAvgF1Score = calcF1Score(microAvgPrecision, microAvgRecall);
if (!isMultiBinaryLabel_) {
microAvgCallback(macroAvgPrecision,
macroAvgRecall,
macroAvgF1Score,
isMultiBinaryLabel_,
microAvgPrecision,
microAvgRecall,
microAvgF1Score);
}
void PrecisionRecallEvaluator::printStats(std::ostream& os) const {
this->printStatsHelper(
[&os](int label, double precision, double recall, double f1) {
os << "positive_label=" << label << " precision=" << precision
<< " recall=" << recall << " F1-score=" << f1;
},
[&os](double macroAvgPrecision,
double macroAvgRecall,
double macroAvgF1Score,
bool isMultiBinaryLabel,
double microAvgPrecision,
double microAvgRecall,
double microAvgF1Score) {
os << "macro-average-precision=" << macroAvgPrecision
<< " macro-average-recall=" << macroAvgRecall
<< " macro-average-F1-score=" << macroAvgF1Score;
if (!isMultiBinaryLabel) {
// precision and recall are equal in this case
os << " micro-average-precision=" << microAvgPrecision;
} else {
@ -651,6 +677,7 @@ void PrecisionRecallEvaluator::printStats(std::ostream& os) const {
<< " micro-average-recall=" << microAvgRecall
<< " micro-average-F1-score=" << microAvgF1Score;
}
});
}
void PrecisionRecallEvaluator::calcStatsInfo(const MatrixPtr& output,
@ -731,6 +758,69 @@ void PrecisionRecallEvaluator::calcStatsInfoMulti(const MatrixPtr& output,
}
}
void PrecisionRecallEvaluator::storeLocalValues() const {
if (this->values_.size() == 0) {
this->printStatsHelper(
[this](int label, double precision, double recall, double f1) {
values_["positive_label"] = (double)label;
values_["precision"] = precision;
values_["recal"] = recall;
values_["F1-score"] = f1;
},
[this](double macroAvgPrecision,
double macroAvgRecall,
double macroAvgF1Score,
bool isMultiBinaryLabel,
double microAvgPrecision,
double microAvgRecall,
double microAvgF1Score) {
values_["macro-average-precision"] = macroAvgPrecision;
values_["macro-average-recall"] = macroAvgRecall;
values_["macro-average-F1-score"] = macroAvgF1Score;
if (!isMultiBinaryLabel) {
// precision and recall are equal in this case
values_["micro-average-precision"] = microAvgPrecision;
} else {
values_["micro-average-precision"] = microAvgPrecision;
values_["micro-average-recall"] = microAvgRecall;
values_["micro-average-F1-score"] = microAvgF1Score;
}
});
}
}
void PrecisionRecallEvaluator::getNames(std::vector<std::string>* names) {
this->storeLocalValues();
names->clear();
names->reserve(this->values_.size());
for (auto it = this->values_.begin(); it != this->values_.end(); ++it) {
names->push_back(this->config_.name() + "." + it->first);
}
}
real PrecisionRecallEvaluator::getValue(const std::string& name,
Error* err) const {
this->storeLocalValues();
auto it = this->values_.find(name);
if (it != this->values_.end() && err != nullptr) {
*err = Error("No such key %s", name.c_str());
return .0f;
}
return it->second;
}
std::string PrecisionRecallEvaluator::getType(const std::string& name,
Error* err) const {
this->storeLocalValues();
auto it = this->values_.find(name);
if (it != this->values_.end() && err != nullptr) {
*err = Error("No such key %s", name.c_str());
return "";
}
return "precision_recall";
}
void PrecisionRecallEvaluator::distributeEval(ParameterClient2* client) {
size_t size = 4 * statsInfo_.size();
double* buf = new double[size];
@ -874,6 +964,8 @@ void PnpairEvaluator::calc(std::vector<PredictionResult>& predictArray) {
<< " calc total special pair: " << special;
}
std::string PnpairEvaluator::getTypeImpl() const { return "pnpair"; }
ClassRegistrar<Evaluator> Evaluator::registrar_;
Evaluator* Evaluator::create(const EvaluatorConfig& config) {
Evaluator* evaluator = registrar_.createByType(config.type());
@ -901,27 +993,12 @@ public:
virtual void eval(const NeuralNetwork& nn) {
for (const std::string& name : config_.input_layers()) {
const Argument& argu = nn.getLayer(name)->getOutput();
if (argu.value) {
std::ostringstream os;
argu.value->print(os);
LOG(INFO) << "layer=" << name << " value matrix:\n" << os.str();
}
if (argu.ids) {
std::ostringstream os;
argu.ids->print(os, argu.ids->getSize());
LOG(INFO) << "layer=" << name << " ids vector:\n" << os.str();
}
if (auto startPos = argu.sequenceStartPositions) {
std::ostringstream os;
startPos->getVector(false)->print(os, startPos->getSize());
LOG(INFO) << "layer=" << name << " sequence pos vector:\n" << os.str();
}
if (auto subStartPos = argu.subSequenceStartPositions) {
std::ostringstream os;
subStartPos->getVector(false)->print(os, subStartPos->getSize());
LOG(INFO) << "layer=" << name << " sub-sequence pos vector:\n"
<< os.str();
std::vector<std::tuple<std::string, std::string>> out;
auto err = nn.getLayerOutputValue(name, &out);
err.check();
for (auto& each : out) {
LOG(INFO) << "layer=" << name << std::get<0>(each) << ":\n"
<< std::get<1>(each);
}
}
}

44
paddle/gserver/evaluators/Evaluator.h
Unescape View File

@ -132,6 +132,20 @@ public:
return this->getValueImpl();
}
virtual std::string getValueStr(const std::string& name,
paddle::Error* err = nullptr) const {
paddle::Error localErr;
if (err == nullptr) {
err = &localErr;
}
real result = this->getValue(name, err);
if (!err->isOK()) {
return "";
} else {
return std::to_string(result);
}
}
virtual std::string getType(const std::string& name,
paddle::Error* err = nullptr) const {
if (name != config_.name() && err != nullptr) {
@ -142,7 +156,9 @@ public:
}
protected:
virtual real getValueImpl() const { return .0f; }
virtual real getValueImpl() const {
return numSamples_ != .0 ? totalScore_ / numSamples_ : .0;
}
virtual std::string getTypeImpl() const { return "base"; }
@ -261,6 +277,10 @@ private:
real* clickData,
real* pvData,
size_t size);
// Evaluator interface
protected:
std::string getTypeImpl() const;
};
/**
* @brief precision, recall and f1 score Evaluator
@ -310,6 +330,9 @@ private:
IVectorPtr cpuLabel_;
MatrixPtr cpuWeight_;
template <typename T1, typename T2>
void printStatsHelper(T1 labelCallback, T2 microAvgCallback) const;
void calcStatsInfo(const MatrixPtr& output,
const IVectorPtr& label,
const MatrixPtr& weight);
@ -341,6 +364,15 @@ private:
return 0;
}
}
mutable std::unordered_map<std::string, real> values_;
void storeLocalValues() const;
// Evaluator interface
public:
void getNames(std::vector<std::string>* names);
real getValue(const std::string& name, Error* err) const;
std::string getType(const std::string& name, Error* err) const;
};
/*
@ -387,8 +419,7 @@ public:
virtual void finish() { calc(predictArray_); }
virtual void printStats(std::ostream& os) const {
os << " pos/neg"
<< "=" << pairArray_[0] / ((pairArray_[1] <= 0) ? 1.0 : pairArray_[1]);
os << " pos/neg=" << this->getValueImpl();
}
virtual void distributeEval(ParameterClient2* client) {
@ -404,6 +435,13 @@ private:
IVectorPtr cpuLabel_;
IVectorPtr cpuInfo_;
MatrixPtr cpuWeight_;
// Evaluator interface
protected:
real getValueImpl() const {
return pairArray_[0] / ((pairArray_[1] <= 0) ? 1.0 : pairArray_[1]);
}
std::string getTypeImpl() const;
};
} // namespace paddle

38
paddle/gserver/gradientmachines/NeuralNetwork.cpp
Unescape View File

@ -405,4 +405,42 @@ NeuralNetwork* NeuralNetwork::newNeuralNetwork(const std::string& name,
}
}
Error NeuralNetwork::getLayerOutputValue(
const std::string& layerName,
std::vector<std::tuple<std::string, std::string>>* out) const {
auto& layers = this->config_.layers();
auto it = std::find_if(
layers.begin(), layers.end(), [&layerName](const LayerConfig& conf) {
return conf.name() == layerName;
});
if (it == layers.end()) {
return Error("Cannot find layer %s", layerName.c_str());
}
auto& layer = this->getLayer(layerName);
out->reserve(4);
auto& argu = layer->getOutput();
if (argu.value) {
std::ostringstream os;
argu.value->print(os);
out->push_back({"value", os.str()});
}
if (argu.ids) {
std::ostringstream os;
argu.ids->print(os, argu.ids->getSize());
out->push_back({"ids", os.str()});
}
if (auto startPos = argu.sequenceStartPositions) {
std::ostringstream os;
startPos->getVector(false)->print(os, startPos->getSize());
out->push_back({"sequence pos", os.str()});
}
if (auto subStartPos = argu.subSequenceStartPositions) {
std::ostringstream os;
subStartPos->getVector(false)->print(os, subStartPos->getSize());
out->push_back({"sub-sequence pos", os.str()});
}
return Error();
}
} // namespace paddle

4
paddle/gserver/gradientmachines/NeuralNetwork.h
Unescape View File

@ -128,6 +128,10 @@ public:
static NeuralNetwork* newNeuralNetwork(const std::string& name = "",
NeuralNetwork* rootNetwork = nullptr);
inline Error __must_check getLayerOutputValue(
const std::string& layerName,
std::vector<std::tuple<std::string, std::string>>* out) const;
protected:
/**
* The constructor of NeuralNetwork.

2
paddle/utils/Error.h
Unescape View File

@ -116,6 +116,8 @@ public:
*/
operator bool() const { return msg_ == nullptr; }
bool isOK() const { return *this; }
/**
* @brief check this status by glog.
* @note It is a temp method used during cleaning Paddle code. It will be

Write Preview
Loading…
Cancel
Save