Fix/sync barrier (#25016)
* fix sync barrier with barrier monitor, test=developrevert-24981-add_device_attr_for_regulization
tangwei12
5 years ago
committed by
GitHub
parent
8db66fc3f6
commit
be6a315fbd
@ -0,0 +1,166 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/operators/distributed/barrier_monitor.h"
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
#include <functional>
|
||||
#include <future> // NOLINT
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <thread> // NOLINT
|
||||
|
||||
#include <ThreadPool.h>
|
||||
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace distributed {
|
||||
bool BarrierMonitor::IncreaseBarrier(const int worker_id,
|
||||
const std::string &barrier) {
|
||||
release_ = false;
|
||||
|
||||
if (barrier == BATCH_BARRIER_MESSAGE) {
|
||||
VLOG(4) << "BarrierMonitor send queue recv trainer: " << worker_id;
|
||||
send_barrier_queue->Push(worker_id);
|
||||
} else if (barrier == FETCH_BARRIER_MESSAGE) {
|
||||
VLOG(4) << "BarrierMonitor recv queue recv trainer: " << worker_id;
|
||||
recv_barrier_queue->Push(worker_id);
|
||||
} else {
|
||||
PADDLE_THROW(platform::errors::Unavailable(
|
||||
"unknown Message status %s, only "
|
||||
"BATCH_BARRIER_MESSAGE/FETCH_BARRIER_MESSAGE",
|
||||
barrier));
|
||||
}
|
||||
return Wait();
|
||||
}
|
||||
|
||||
void BarrierMonitor::DecreaseWorker() {
|
||||
std::unique_lock<std::mutex> lck(mutex_);
|
||||
workers_--;
|
||||
VLOG(1) << "decrement worker num to " << workers_;
|
||||
}
|
||||
|
||||
void BarrierMonitor::Reset(int workers, BarrierType type) {
|
||||
std::unique_lock<std::mutex> lk(server_mutex_);
|
||||
|
||||
workers_ = workers;
|
||||
barrier_type = type;
|
||||
|
||||
send_barrier_queue->Clear();
|
||||
recv_barrier_queue->Clear();
|
||||
VLOG(2) << "reset monitor workers: " << workers_ << " type: " << barrier_type;
|
||||
}
|
||||
|
||||
void BarrierMonitor::Monitor() {
|
||||
while (!IsReady() && running_) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
|
||||
VLOG(3) << "sync at first time, wait all trainer ready";
|
||||
}
|
||||
|
||||
while (running_) {
|
||||
int timer = 0;
|
||||
|
||||
if (IsReady()) {
|
||||
Swap(true);
|
||||
} else {
|
||||
VLOG(4) << "running timer: " << timer << " barrier: " << barrier_type
|
||||
<< " sendQ:" << send_barrier_queue->Size()
|
||||
<< " recvQ: " << recv_barrier_queue->Size();
|
||||
|
||||
timer++;
|
||||
if (max_wait_ms == -1 || timer < max_wait_ms) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
} else {
|
||||
VLOG(1) << "time out of " << max_wait_ms
|
||||
<< ", need barreir: " << barrier_type << " retry";
|
||||
Swap(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool BarrierMonitor::IsReady() {
|
||||
if (barrier_type == BarrierType::kSendBarrier) {
|
||||
return static_cast<int>(send_barrier_queue->Size()) == workers_;
|
||||
} else {
|
||||
return static_cast<int>(recv_barrier_queue->Size()) == workers_;
|
||||
}
|
||||
}
|
||||
|
||||
void BarrierMonitor::Swap(bool is_valid) {
|
||||
std::unique_lock<std::mutex> lck(mutex_);
|
||||
|
||||
valid_ = is_valid;
|
||||
release_ = true;
|
||||
|
||||
if (barrier_type == BarrierType::kSendBarrier) {
|
||||
barrier_type = BarrierType::kRecvBarrier;
|
||||
send_barrier_queue->Clear();
|
||||
VLOG(4) << "barrier monitor server clean up queue and barrier";
|
||||
ServerWeakup();
|
||||
VLOG(4) << "barrier monitor server weak up sync to do";
|
||||
WaitServerWeakup();
|
||||
VLOG(4) << "barrier monitor server weak up sync done";
|
||||
|
||||
} else {
|
||||
barrier_type = BarrierType::kSendBarrier;
|
||||
recv_barrier_queue->Clear();
|
||||
VLOG(4) << "barrier monitor server switch to send barrier";
|
||||
}
|
||||
|
||||
worker_cv_.notify_all();
|
||||
}
|
||||
|
||||
void BarrierMonitor::Stop() {
|
||||
valid_ = true;
|
||||
release_ = true;
|
||||
running_ = false;
|
||||
|
||||
barrier_type = BarrierType::kRecvBarrier;
|
||||
send_barrier_queue->Clear();
|
||||
recv_barrier_queue->Clear();
|
||||
|
||||
worker_cv_.notify_all();
|
||||
server_cv_.notify_all();
|
||||
|
||||
if (monitor_thread_) monitor_thread_->join();
|
||||
monitor_thread_ = nullptr;
|
||||
}
|
||||
|
||||
bool BarrierMonitor::Wait() {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
worker_cv_.wait(lk, [this] { return (release_); });
|
||||
return valid_;
|
||||
}
|
||||
|
||||
void BarrierMonitor::WaitServerWeakup() {
|
||||
std::unique_lock<std::mutex> lk(server_mutex_);
|
||||
server_cv_.wait(lk);
|
||||
}
|
||||
|
||||
void BarrierMonitor::ServerWeakup() { server_cv_.notify_all(); }
|
||||
|
||||
std::once_flag BarrierMonitor::init_flag_;
|
||||
std::unique_ptr<BarrierMonitor> BarrierMonitor::monitor_(nullptr);
|
||||
|
||||
} // namespace distributed
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,186 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
#include <chrono> // NOLINT
|
||||
#include <deque>
|
||||
#include <functional>
|
||||
#include <future> // NOLINT
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <thread> // NOLINT
|
||||
|
||||
#include <ThreadPool.h>
|
||||
|
||||
#include "paddle/fluid/operators/distributed/rpc_server.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace distributed {
|
||||
|
||||
enum BarrierType { kSendBarrier, kRecvBarrier };
|
||||
|
||||
constexpr int64_t kMaxWaitMS = 120000;
|
||||
|
||||
template <typename T>
|
||||
class BlockingQueueForBarrier {
|
||||
public:
|
||||
explicit BlockingQueueForBarrier(size_t capacity) : capacity_(capacity) {
|
||||
PADDLE_ENFORCE_GT(capacity_, 0,
|
||||
platform::errors::InvalidArgument(
|
||||
"The capacity must be greater than 0."));
|
||||
}
|
||||
|
||||
bool Push(const T &elem) {
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
worker_cv_.wait(lock, [&] { return queue_.size() < capacity_; });
|
||||
queue_.push_back(elem);
|
||||
}
|
||||
worker_cv_.notify_one();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Push(T &&elem) {
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
worker_cv_.wait(lock, [&] { return queue_.size() < capacity_; });
|
||||
queue_.emplace_back(std::move(elem));
|
||||
}
|
||||
worker_cv_.notify_one();
|
||||
return true;
|
||||
}
|
||||
|
||||
T Pop() {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
worker_cv_.wait(lock, [=] { return !queue_.empty(); });
|
||||
T rc(std::move(queue_.front()));
|
||||
queue_.pop_front();
|
||||
worker_cv_.notify_one();
|
||||
return rc;
|
||||
}
|
||||
|
||||
size_t Cap() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return capacity_;
|
||||
}
|
||||
|
||||
size_t Size() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return queue_.size();
|
||||
}
|
||||
|
||||
void Clear() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
std::deque<T>().swap(queue_);
|
||||
}
|
||||
|
||||
private:
|
||||
const size_t capacity_;
|
||||
std::deque<T> queue_;
|
||||
|
||||
mutable std::mutex mutex_;
|
||||
std::condition_variable worker_cv_;
|
||||
};
|
||||
|
||||
class BarrierMonitor {
|
||||
public:
|
||||
explicit BarrierMonitor(int workers)
|
||||
: BarrierMonitor(workers, BarrierType::kRecvBarrier, kMaxWaitMS) {}
|
||||
|
||||
explicit BarrierMonitor(int workers, BarrierType type, int64_t max_wait_times)
|
||||
: workers_(workers), barrier_type(type), max_wait_ms(max_wait_times) {
|
||||
PADDLE_ENFORCE_GT(workers, 0, platform::errors::InvalidArgument(
|
||||
"trainers must have one or more"));
|
||||
|
||||
send_barrier_queue =
|
||||
std::make_shared<BlockingQueueForBarrier<int>>(workers);
|
||||
recv_barrier_queue =
|
||||
std::make_shared<BlockingQueueForBarrier<int>>(workers);
|
||||
|
||||
running_ = true;
|
||||
monitor_thread_.reset(
|
||||
new std::thread(std::bind(&BarrierMonitor::Monitor, this)));
|
||||
}
|
||||
|
||||
static BarrierMonitor *Init(int workers) {
|
||||
InitImpl(workers);
|
||||
return GetInstance();
|
||||
}
|
||||
|
||||
static BarrierMonitor *GetInstance() { return monitor_.get(); }
|
||||
|
||||
bool IncreaseBarrier(const int worker_id, const std::string &barrier);
|
||||
|
||||
void DecreaseWorker();
|
||||
|
||||
int GetWorkerNum() { return workers_; }
|
||||
|
||||
void Monitor();
|
||||
|
||||
void Swap(bool is_valid);
|
||||
|
||||
void Stop();
|
||||
|
||||
bool IsReady();
|
||||
|
||||
bool Wait();
|
||||
|
||||
void WaitServerWeakup();
|
||||
|
||||
void ServerWeakup();
|
||||
|
||||
void WorkerWeakup();
|
||||
|
||||
void Reset(int workers, BarrierType type);
|
||||
|
||||
private:
|
||||
// Init is called by GetInstance.
|
||||
static void InitImpl(int workers) {
|
||||
monitor_.reset(new BarrierMonitor(workers));
|
||||
}
|
||||
|
||||
static std::once_flag init_flag_;
|
||||
static std::unique_ptr<BarrierMonitor> monitor_;
|
||||
|
||||
int workers_;
|
||||
bool running_ = false;
|
||||
bool valid_ = false;
|
||||
bool release_ = false;
|
||||
|
||||
std::condition_variable worker_cv_;
|
||||
std::condition_variable server_cv_;
|
||||
|
||||
std::mutex server_mutex_;
|
||||
std::mutex mutex_;
|
||||
|
||||
BarrierType barrier_type;
|
||||
int64_t max_wait_ms;
|
||||
std::unique_ptr<std::thread> monitor_thread_{nullptr};
|
||||
std::shared_ptr<BlockingQueueForBarrier<int>> send_barrier_queue;
|
||||
std::shared_ptr<BlockingQueueForBarrier<int>> recv_barrier_queue;
|
||||
};
|
||||
|
||||
} // namespace distributed
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue