Yu Yang
8 years ago
parent
5d7fabd32c
commit
2ac8e6a518
@ -1,209 +0,0 @@
|
|||||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License. */
|
|
||||||
|
|
||||||
#include <fstream>
|
|
||||||
#include "paddle/utils/Logging.h"
|
|
||||||
|
|
||||||
#include "ParallelParameter.h"
|
|
||||||
|
|
||||||
namespace paddle {
|
|
||||||
|
|
||||||
UpdateFunction paramUpdateFunctions[UPDATE_TYPE_NUM] = {
|
|
||||||
nullptr, // &ParallelParameter::singleUpdate, /* single thread */
|
|
||||||
nullptr, // &ParallelParameter::controlUpdate, /* controller thread */
|
|
||||||
&ParallelParameter::majorUpdate, /* major thread */
|
|
||||||
&ParallelParameter::minorUpdate, /* minor thread */
|
|
||||||
|
|
||||||
nullptr, /* master */
|
|
||||||
&ParallelParameter::slaveUpdate, /* slave */
|
|
||||||
};
|
|
||||||
ParallelParameterPtr ParallelParameter::create(TrainerRole role,
|
|
||||||
ParameterPtr localParam,
|
|
||||||
int asyncCount) {
|
|
||||||
ParallelParameterPtr ptr = nullptr;
|
|
||||||
switch (role) {
|
|
||||||
case TRAINER_ROLE_CONTROL:
|
|
||||||
case TRAINER_ROLE_MAJOR:
|
|
||||||
case TRAINER_ROLE_MINOR:
|
|
||||||
ptr = std::make_shared<SyncParameter>(role, localParam);
|
|
||||||
break;
|
|
||||||
case TRAINER_ROLE_MASTER:
|
|
||||||
case TRAINER_ROLE_SLAVE:
|
|
||||||
ptr = std::make_shared<AsyncParameter>(role, asyncCount, localParam);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
LOG(FATAL) << "unknown role " << role << "\n";
|
|
||||||
}
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
void ParallelParameter::syncUpdate(TrainerRole role, real learnRate) {
|
|
||||||
if (paramUpdateFunctions[role]) {
|
|
||||||
(this->*paramUpdateFunctions[role])(learnRate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::attachControlParam(ParallelParameterPtr controler) {
|
|
||||||
controlParam_ = controler;
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::attachMajorParam(ParallelParameterPtr partner) {
|
|
||||||
majorPartners_.push_back(partner);
|
|
||||||
if (role_ == TRAINER_ROLE_CONTROL) {
|
|
||||||
localParam_->setSharedCount(majorPartners_.size());
|
|
||||||
}
|
|
||||||
// partnerParam_ = partner;
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::attachMinorParam(ParallelParameterPtr partner,
|
|
||||||
int deviceId) {
|
|
||||||
minorPartners_.push_back(partner);
|
|
||||||
minorDeviceIds_.push_back(deviceId);
|
|
||||||
// partnerParam_ = partner;
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::waitAllMajorGradReady() {
|
|
||||||
for (size_t i = 0; i < majorPartners_.size(); i++) {
|
|
||||||
majorPartners_[i]->waitGradReady();
|
|
||||||
partnerParam_ = majorPartners_[i]->getLocalParameter();
|
|
||||||
VectorPtr localGrad = localParam_->getBuf(PARAMETER_GRADIENT);
|
|
||||||
VectorPtr patnrGrad = partnerParam_->getBuf(PARAMETER_GRADIENT);
|
|
||||||
if (FLAGS_use_gpu) hl_set_device(minorDeviceIds_[i]);
|
|
||||||
localGrad->add(*patnrGrad);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::synchronizeParamter() {
|
|
||||||
valueSem_->wait();
|
|
||||||
if (role_ == TRAINER_ROLE_MINOR) {
|
|
||||||
/* copy the value from controller */
|
|
||||||
VectorPtr cntrlVec =
|
|
||||||
(controlParam_->getLocalParameter())->getBuf(PARAMETER_VALUE);
|
|
||||||
VectorPtr localVec = localParam_->getBuf(PARAMETER_VALUE);
|
|
||||||
localVec->copyFrom(*cntrlVec);
|
|
||||||
|
|
||||||
/* dispatch the value to major */
|
|
||||||
for (size_t i = 0; i < majorPartners_.size(); i++) {
|
|
||||||
VectorPtr majorVec =
|
|
||||||
(majorPartners_[i]->getLocalParameter())->getBuf(PARAMETER_VALUE);
|
|
||||||
majorVec->copyFrom(*localVec);
|
|
||||||
majorPartners_[i]->postValueReady();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::singleUpdate(real learnRate) {
|
|
||||||
CHECK(role_ == TRAINER_ROLE_SINGLE);
|
|
||||||
localParam_->updateWithGradient(learnRate);
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::controlUpdate(const UpdateCallback &callBack) {
|
|
||||||
CHECK(role_ == TRAINER_ROLE_CONTROL);
|
|
||||||
CHECK(gradSem_ != NULL && valueSem_ != NULL);
|
|
||||||
CHECK(majorPartners_.size());
|
|
||||||
|
|
||||||
/* update */
|
|
||||||
if (callBack) {
|
|
||||||
callBack(localParam_.get());
|
|
||||||
localParam_->clearGradient();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (size_t i = 0; i < minorPartners_.size(); i++) {
|
|
||||||
minorPartners_[i]->postValueReady();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::majorUpdate(real learnRate) {
|
|
||||||
(void)learnRate;
|
|
||||||
CHECK(role_ == TRAINER_ROLE_MAJOR);
|
|
||||||
CHECK(gradSem_ != NULL && valueSem_ != NULL);
|
|
||||||
CHECK(minorPartners_.size() && controlParam_);
|
|
||||||
|
|
||||||
/* wait the minor-Gradient is ready */
|
|
||||||
for (size_t i = 0; i < minorPartners_.size(); i++) {
|
|
||||||
minorPartners_[i]->waitGradReady();
|
|
||||||
partnerParam_ = minorPartners_[i]->getLocalParameter();
|
|
||||||
VectorPtr localGrad = localParam_->getBuf(PARAMETER_GRADIENT);
|
|
||||||
VectorPtr minorGrad = partnerParam_->getBuf(PARAMETER_GRADIENT);
|
|
||||||
localGrad->add(*minorGrad);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* notice the controller that the gradient is ready */
|
|
||||||
gradSem_->post();
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncParameter::minorUpdate(real learnRate) {
|
|
||||||
(void)learnRate;
|
|
||||||
CHECK(role_ == TRAINER_ROLE_MINOR);
|
|
||||||
CHECK(gradSem_ != NULL && valueSem_ != NULL);
|
|
||||||
|
|
||||||
// notice the major that the gradient is ready
|
|
||||||
gradSem_->post();
|
|
||||||
}
|
|
||||||
|
|
||||||
AsyncParameter::AsyncParameter(TrainerRole role,
|
|
||||||
int asyncCount,
|
|
||||||
ParameterPtr localParam)
|
|
||||||
: ParallelParameter(role, localParam) {
|
|
||||||
asyncCount_ = asyncCount;
|
|
||||||
accumCounter_ = 0;
|
|
||||||
gradientAccum_ = Vector::create(localParam->getSize(), localParam->useGpu());
|
|
||||||
gradientAccum_->zeroMem();
|
|
||||||
}
|
|
||||||
|
|
||||||
void AsyncParameter::slaveUpdate(real learnRate) {
|
|
||||||
/* increase the accumCounter_ */
|
|
||||||
accumCounter_++;
|
|
||||||
|
|
||||||
/* accumulate the gradient to the buffer */
|
|
||||||
VectorPtr grad = localParam_->getBuf(PARAMETER_GRADIENT);
|
|
||||||
gradientAccum_->add(*grad);
|
|
||||||
|
|
||||||
/* if need to be synchronized with the master */
|
|
||||||
if (accumCounter_ == asyncCount_) {
|
|
||||||
gradSem_->post();
|
|
||||||
// accumCounter_ = 0; NOTICE: the upper-function need to reset the counter
|
|
||||||
} else { // self update
|
|
||||||
localParam_->updateWithGradient(learnRate);
|
|
||||||
}
|
|
||||||
localParam_->clearGradient();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AsyncParameter::masterUpdate(ParallelParameterPtr slaveParam,
|
|
||||||
const UpdateCallback &callback) {
|
|
||||||
CHECK(slaveParam && callback);
|
|
||||||
|
|
||||||
/* wait the slave is ready */
|
|
||||||
if (!slaveParam->timeWaitGradReady(5)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
AsyncParameter *asyncParam = dynamic_cast<AsyncParameter *>(slaveParam.get());
|
|
||||||
|
|
||||||
/* get the accum-gradient to update local parameter */
|
|
||||||
VectorPtr slaveVec = asyncParam->getAccum();
|
|
||||||
localParam_->getBuf(PARAMETER_GRADIENT)->copyFrom(*slaveVec);
|
|
||||||
callback(localParam_.get());
|
|
||||||
// slaveVec->zeroMem();
|
|
||||||
|
|
||||||
/* copy the newest parameter-value to the slave */
|
|
||||||
slaveVec = (slaveParam->getLocalParameter())->getBuf(PARAMETER_VALUE);
|
|
||||||
slaveVec->copyFrom(*(localParam_->getBuf(PARAMETER_VALUE)));
|
|
||||||
|
|
||||||
/* release the semphore */
|
|
||||||
slaveParam->postValueReady();
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace paddle
|
|
||||||
@ -1,244 +0,0 @@
|
|||||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License. */
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <stdint.h>
|
|
||||||
|
|
||||||
#include <sys/time.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <iostream>
|
|
||||||
#include <string>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "hl_gpu.h"
|
|
||||||
#include "paddle/math/Vector.h"
|
|
||||||
#include "paddle/parameter/Parameter.h"
|
|
||||||
#include "paddle/parameter/ParameterUpdateFunctions.h"
|
|
||||||
#include "paddle/utils/Common.h"
|
|
||||||
#include "paddle/utils/Flags.h"
|
|
||||||
#include "paddle/utils/Locks.h"
|
|
||||||
|
|
||||||
#include "ParameterConfig.pb.h"
|
|
||||||
|
|
||||||
namespace paddle {
|
|
||||||
|
|
||||||
class ParallelParameter;
|
|
||||||
class SyncParameter;
|
|
||||||
class AsyncParameter;
|
|
||||||
|
|
||||||
typedef std::shared_ptr<ParallelParameter> ParallelParameterPtr;
|
|
||||||
|
|
||||||
const int UPDATE_TYPE_NUM = 32;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* TrainRole denotes the role of current training, different roles have
|
|
||||||
* different jobs.
|
|
||||||
*
|
|
||||||
* control, major, minor are three kinds of role to support mutiple GPUs
|
|
||||||
* parallel SGD training. SM on GPU card has two groups, each group
|
|
||||||
* consist of a major and a minor.
|
|
||||||
*
|
|
||||||
* @param single single GPU card single thread training.
|
|
||||||
*
|
|
||||||
*
|
|
||||||
* @param control current parameter updates via control role,
|
|
||||||
* not participate in real training. control role is
|
|
||||||
* responsible for merging all major's gradient and
|
|
||||||
* update parameter value.
|
|
||||||
*
|
|
||||||
* @param major major role paticipates in real training, when local
|
|
||||||
* gradient is ready, merge its corresponding minor's
|
|
||||||
* gradient and notify controller: this group's gradient
|
|
||||||
* is already ready.
|
|
||||||
*
|
|
||||||
* @param minor minor role participates in real training, when local
|
|
||||||
* gradient is ready, only notify its corresponding major.
|
|
||||||
* In order to maximum apportion jobs, after controller
|
|
||||||
* updates the paramemter value, each group's minior
|
|
||||||
* reponses to dispatch the latest model into local and
|
|
||||||
* major.
|
|
||||||
*/
|
|
||||||
enum TrainerRole {
|
|
||||||
TRAINER_ROLE_SINGLE,
|
|
||||||
TRAINER_ROLE_CONTROL,
|
|
||||||
TRAINER_ROLE_MAJOR,
|
|
||||||
TRAINER_ROLE_MINOR,
|
|
||||||
TRAINER_ROLE_MASTER,
|
|
||||||
TRAINER_ROLE_SLAVE
|
|
||||||
};
|
|
||||||
typedef void (ParallelParameter::*UpdateFunction)(real learnRate);
|
|
||||||
|
|
||||||
class ParallelParameter {
|
|
||||||
public:
|
|
||||||
static ParallelParameterPtr create(TrainerRole role,
|
|
||||||
ParameterPtr localParam,
|
|
||||||
int asyncCount = 1);
|
|
||||||
|
|
||||||
ParallelParameter(TrainerRole role, ParameterPtr localParam) {
|
|
||||||
role_ = role;
|
|
||||||
gradSem_.reset(new Semaphore(0));
|
|
||||||
valueSem_.reset(new Semaphore(0));
|
|
||||||
localParam_ = localParam;
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual ~ParallelParameter() {}
|
|
||||||
|
|
||||||
ParameterPtr getLocalParameter() { return localParam_; }
|
|
||||||
bool timeWaitGradReady(int sec) {
|
|
||||||
struct timespec ts;
|
|
||||||
ts.tv_nsec = 0;
|
|
||||||
ts.tv_sec = time(NULL) + sec;
|
|
||||||
return gradSem_->timeWait(&ts);
|
|
||||||
}
|
|
||||||
void waitGradReady() { gradSem_->wait(); }
|
|
||||||
void postValueReady() { valueSem_->post(); }
|
|
||||||
|
|
||||||
void syncUpdate(TrainerRole role, real learnRate);
|
|
||||||
|
|
||||||
virtual void synchronizeParamter() = 0;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* for synchronous
|
|
||||||
*/
|
|
||||||
virtual void singleUpdate(real learnRate) { (void)learnRate; }
|
|
||||||
|
|
||||||
virtual void controlUpdate(const UpdateCallback& callback) { (void)callback; }
|
|
||||||
|
|
||||||
virtual void majorUpdate(real learnRate) { (void)learnRate; }
|
|
||||||
|
|
||||||
virtual void minorUpdate(real learnRate) { (void)learnRate; }
|
|
||||||
|
|
||||||
/**
|
|
||||||
* for asynchronous
|
|
||||||
*/
|
|
||||||
virtual void slaveUpdate(real learnRate) { (void)learnRate; }
|
|
||||||
|
|
||||||
protected:
|
|
||||||
TrainerRole role_;
|
|
||||||
ParameterPtr localParam_;
|
|
||||||
std::unique_ptr<Semaphore>
|
|
||||||
gradSem_; /// wether the local parameter-gradient is ready
|
|
||||||
std::unique_ptr<Semaphore>
|
|
||||||
valueSem_; /// wether the local parameter-value is updated
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* this class is designed for multi-threading training.
|
|
||||||
*
|
|
||||||
* "Synchronous" means multiple GPUs calculate 1/4 mini-Batch,
|
|
||||||
* but will get only one gradient
|
|
||||||
*/
|
|
||||||
class SyncParameter : public ParallelParameter {
|
|
||||||
public:
|
|
||||||
SyncParameter(TrainerRole role, ParameterPtr localParam)
|
|
||||||
: ParallelParameter(role, localParam) {
|
|
||||||
controlParam_ = nullptr;
|
|
||||||
majorPartners_.clear();
|
|
||||||
minorPartners_.clear();
|
|
||||||
}
|
|
||||||
~SyncParameter() {
|
|
||||||
majorPartners_.clear();
|
|
||||||
minorPartners_.clear();
|
|
||||||
}
|
|
||||||
void attachControlParam(ParallelParameterPtr controler);
|
|
||||||
|
|
||||||
void attachMajorParam(ParallelParameterPtr partner);
|
|
||||||
|
|
||||||
void attachMinorParam(ParallelParameterPtr partner, int deviceId);
|
|
||||||
|
|
||||||
void waitAllMajorGradReady();
|
|
||||||
|
|
||||||
void synchronizeParamter();
|
|
||||||
|
|
||||||
void singleUpdate(real learnRate);
|
|
||||||
|
|
||||||
void controlUpdate(const UpdateCallback& callback);
|
|
||||||
|
|
||||||
void majorUpdate(real learnRate);
|
|
||||||
|
|
||||||
void minorUpdate(real learnRate);
|
|
||||||
|
|
||||||
std::vector<ParallelParameterPtr>& getMajorPartners() {
|
|
||||||
return majorPartners_;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<ParallelParameterPtr>& getMinorPartners() {
|
|
||||||
return minorPartners_;
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
// The following variables are used in a multithreaded training situation
|
|
||||||
// partnerParam_ is local-parameter's partner
|
|
||||||
// controlParam_ is the controller-thread 's parameter
|
|
||||||
ParameterPtr partnerParam_;
|
|
||||||
std::vector<ParallelParameterPtr> majorPartners_;
|
|
||||||
std::vector<ParallelParameterPtr> minorPartners_;
|
|
||||||
std::vector<int> minorDeviceIds_;
|
|
||||||
ParallelParameterPtr controlParam_;
|
|
||||||
};
|
|
||||||
|
|
||||||
class AsyncParameter : public ParallelParameter {
|
|
||||||
public:
|
|
||||||
AsyncParameter(TrainerRole role, int asyncCount, ParameterPtr localParam);
|
|
||||||
|
|
||||||
void clearCounter() { accumCounter_ = 0; }
|
|
||||||
|
|
||||||
VectorPtr getAccum() { return gradientAccum_; }
|
|
||||||
|
|
||||||
void synchronizeParamter() {
|
|
||||||
if (accumCounter_ == asyncCount_) {
|
|
||||||
valueSem_->wait();
|
|
||||||
clearCounter();
|
|
||||||
gradientAccum_->zeroMem();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* When asynchronous training, update strategy including slave and master.
|
|
||||||
*
|
|
||||||
* slave: If in range asyncCount, adopting self-update method.
|
|
||||||
* If beyond asyncCount, waiting for master to update.
|
|
||||||
*/
|
|
||||||
void slaveUpdate(real learnRate);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* When asynchronous training, update strategy including slave and master.
|
|
||||||
*
|
|
||||||
* master: it only polls slaves, do not training data.
|
|
||||||
* If slave's gradient is ready, fetch it.
|
|
||||||
* Update master's parameter, then copy it into
|
|
||||||
* corresponding slave.
|
|
||||||
*/
|
|
||||||
bool masterUpdate(ParallelParameterPtr slaveParam,
|
|
||||||
const UpdateCallback& callback);
|
|
||||||
|
|
||||||
private:
|
|
||||||
/**
|
|
||||||
* When asynchronous training, every aysnc trainer needs to
|
|
||||||
* accumulate a number of batch gradient.
|
|
||||||
*
|
|
||||||
* gradientAccum_ is used to save the sum of gradients.
|
|
||||||
*/
|
|
||||||
VectorPtr gradientAccum_;
|
|
||||||
|
|
||||||
/// Asynchronous count.
|
|
||||||
int asyncCount_;
|
|
||||||
/// Accumulate counter of current gradients.
|
|
||||||
int accumCounter_;
|
|
||||||
};
|
|
||||||
|
|
||||||
typedef std::map<std::string, ParallelParameterPtr> ParallelParameterMap;
|
|
||||||
|
|
||||||
} // namespace paddle
|
|
||||||
Loading…
Reference in new issue