Trainer heartbeat for async mode (#19600)
Heartbeat for distributed async training.fix-python-transpose
tangwei12
6 years ago
committed by
GitHub
parent
76ba55e891
commit
b5a410466c
@ -0,0 +1,97 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/operators/distributed/heart_beat_monitor.h"
|
||||
#include <chrono> // NOLINT
|
||||
#include <ctime>
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace distributed {
|
||||
|
||||
DEFINE_int32(worker_update_interval_secs, 900,
|
||||
" the longest time interval between the worker update variables");
|
||||
|
||||
inline int GetCurrentUS() {
|
||||
// current date/time based on current system
|
||||
time_t t = std::time(0);
|
||||
int now = static_cast<int>(t);
|
||||
return now;
|
||||
}
|
||||
|
||||
void HeartBeatMonitor::Update(const int worker_id, std::string be_monitored_var,
|
||||
WorkerStatus status) {
|
||||
if (status == UNINITED) {
|
||||
LOG(WARNING) << "HeartBeatMonitor receive UNINITED status can not be used "
|
||||
"in Update, something error";
|
||||
}
|
||||
|
||||
if (!is_chief_) {
|
||||
return;
|
||||
}
|
||||
|
||||
if ((be_monitored_var == be_monitored_var_ && status == RUNNING) ||
|
||||
status == COMPLETED) {
|
||||
auto timestamp = GetCurrentUS();
|
||||
UnderMonitoredWorker& worker = worker_status_map_.at(worker_id);
|
||||
|
||||
if (worker.status != COMPLETED) {
|
||||
worker.status = status;
|
||||
}
|
||||
worker.timestamp = timestamp;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void HeartBeatMonitor::LostWorkerMonitor() {
|
||||
VLOG(1) << "worker heartbeat monitor start at No.0 parameter server";
|
||||
while (running_) {
|
||||
for (int id = 0; id < workers_; ++id) {
|
||||
auto& worker = worker_status_map_.at(id);
|
||||
|
||||
if (worker.status == UNINITED) {
|
||||
VLOG(4) << "worker " << worker.id << " is under UNINITED";
|
||||
continue;
|
||||
}
|
||||
if (worker.status == COMPLETED) {
|
||||
VLOG(4) << "worker " << worker.id << " is under COMPLETED";
|
||||
continue;
|
||||
}
|
||||
|
||||
auto timestamp = GetCurrentUS();
|
||||
|
||||
VLOG(4) << "worker " << worker.id << " status is " << worker.status
|
||||
<< " timestamp is " << worker.timestamp << " the interval is "
|
||||
<< timestamp - worker.timestamp;
|
||||
|
||||
if (timestamp - worker.timestamp >= FLAGS_worker_update_interval_secs) {
|
||||
PADDLE_THROW(
|
||||
"the latest update of worker %d is %d secs ago, we doubt the "
|
||||
"the worker is not alive and this may have a bad effect on the "
|
||||
"fitting result, please check",
|
||||
worker.id, FLAGS_worker_update_interval_secs);
|
||||
}
|
||||
}
|
||||
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(30 * 1000));
|
||||
}
|
||||
VLOG(1) << "worker heartbeat monitor stopped, thread exit";
|
||||
}
|
||||
|
||||
std::once_flag HeartBeatMonitor::init_flag_;
|
||||
std::unique_ptr<HeartBeatMonitor> HeartBeatMonitor::monitor_(nullptr);
|
||||
|
||||
} // namespace distributed
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,136 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
#include <functional>
|
||||
#include <future> // NOLINT
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <thread> // NOLINT
|
||||
|
||||
#include <ThreadPool.h>
|
||||
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace distributed {
|
||||
|
||||
enum WorkerStatus { UNINITED = 0, RUNNING, COMPLETED };
|
||||
|
||||
struct UnderMonitoredWorker {
|
||||
int id;
|
||||
WorkerStatus status;
|
||||
int timestamp;
|
||||
|
||||
UnderMonitoredWorker() {}
|
||||
|
||||
explicit UnderMonitoredWorker(int worker_id) {
|
||||
this->id = worker_id;
|
||||
this->status = UNINITED;
|
||||
this->timestamp = 0;
|
||||
}
|
||||
};
|
||||
|
||||
class HeartBeatMonitor {
|
||||
public:
|
||||
explicit HeartBeatMonitor(int workers, bool is_chief,
|
||||
std::string be_monitored_var)
|
||||
: workers_(workers),
|
||||
is_chief_(is_chief),
|
||||
be_monitored_var_(be_monitored_var),
|
||||
running_(true) {
|
||||
PADDLE_ENFORCE_GT(workers, 0, "trainers must have one or more");
|
||||
|
||||
for (auto worker_id = 0; worker_id < workers; worker_id++) {
|
||||
UnderMonitoredWorker worker(worker_id);
|
||||
worker_status_map_[worker_id] = std::move(worker);
|
||||
}
|
||||
|
||||
// we define the No.0 pserver is the first parameter server
|
||||
// only No.0 will check the heartbeat of all trainers
|
||||
if (is_chief) {
|
||||
monitor_thread_.reset(new std::thread(
|
||||
std::bind(&HeartBeatMonitor::LostWorkerMonitor, this)));
|
||||
}
|
||||
}
|
||||
|
||||
~HeartBeatMonitor() {
|
||||
running_ = false;
|
||||
if (monitor_thread_) monitor_thread_->join();
|
||||
}
|
||||
|
||||
static void Init(int workers, bool is_chief, std::string be_monitored_var) {
|
||||
std::call_once(init_flag_, &HeartBeatMonitor::InitImpl, workers, is_chief,
|
||||
be_monitored_var);
|
||||
}
|
||||
|
||||
static HeartBeatMonitor* GetInstance() {
|
||||
if (monitor_ == nullptr) {
|
||||
PADDLE_THROW(
|
||||
"HeartBeatMonitor is not inited, call "
|
||||
"HeartBeatMonitor::Init first");
|
||||
}
|
||||
return monitor_.get();
|
||||
}
|
||||
|
||||
void Stop() {
|
||||
running_ = false;
|
||||
if (!monitor_) {
|
||||
VLOG(0) << "HeartBeatMonitor is not inited, do nothing";
|
||||
} else {
|
||||
if (monitor_thread_) {
|
||||
monitor_thread_->join();
|
||||
monitor_thread_.reset(nullptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Update(const int worker_id, std::string be_monitored_var,
|
||||
WorkerStatus status);
|
||||
|
||||
void LostWorkerMonitor();
|
||||
|
||||
private:
|
||||
// Init is called by GetInstance.
|
||||
static void InitImpl(int workers, bool is_chief,
|
||||
std::string be_monitored_var) {
|
||||
if (monitor_ == nullptr) {
|
||||
monitor_.reset(new HeartBeatMonitor(workers, is_chief, be_monitored_var));
|
||||
}
|
||||
}
|
||||
|
||||
static std::once_flag init_flag_;
|
||||
static std::unique_ptr<HeartBeatMonitor> monitor_;
|
||||
|
||||
int workers_;
|
||||
bool is_chief_;
|
||||
std::string be_monitored_var_;
|
||||
std::unordered_map<int, UnderMonitoredWorker> worker_status_map_;
|
||||
std::unique_ptr<std::thread> monitor_thread_{nullptr};
|
||||
std::mutex mutex_;
|
||||
bool running_ = false;
|
||||
};
|
||||
|
||||
} // namespace distributed
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,57 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/operators/distributed/heart_beat_monitor.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <thread> // NOLINT
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace distributed {
|
||||
|
||||
void run(HeartBeatMonitor* monitor) { monitor->LostWorkerMonitor(); }
|
||||
|
||||
TEST(HeartBeatMonitor, All) {
|
||||
int trainers = 10;
|
||||
int pserver_id = 0;
|
||||
std::string var = "nce_w@GRAD.block0";
|
||||
std::string var2 = "nce_w@GRAD.block2";
|
||||
|
||||
HeartBeatMonitor::Init(trainers, pserver_id == 0, var);
|
||||
|
||||
auto* monitor = HeartBeatMonitor::GetInstance();
|
||||
|
||||
std::vector<int> ids{1, 3, 5, 7};
|
||||
|
||||
for (auto& id : ids) {
|
||||
monitor->Update(id, var, RUNNING);
|
||||
}
|
||||
|
||||
monitor->Update(9, var2, RUNNING);
|
||||
monitor->Update(2, var, COMPLETED);
|
||||
|
||||
std::thread t(run, monitor);
|
||||
t.detach();
|
||||
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(45 * 1000));
|
||||
|
||||
monitor->Stop();
|
||||
}
|
||||
|
||||
} // namespace distributed
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
Loading…
Reference in new issue