Revert/barrier for sync (#25417)
* add retry for prefetch
* Revert "Fix/sync barrier (#25016)"
This reverts commit be6a315fbd.
* reopen dist UT, test=develop
* remove fl UT, test=develop
fix_copy_if_different
tangwei12
5 years ago
committed by
GitHub
parent
dd9d146ec3
commit
4b3778a3ee
@ -1,166 +0,0 @@
|
|||||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
#include "paddle/fluid/operators/distributed/barrier_monitor.h"
|
|
||||||
#include <gflags/gflags.h>
|
|
||||||
|
|
||||||
#include <functional>
|
|
||||||
#include <future> // NOLINT
|
|
||||||
#include <memory>
|
|
||||||
#include <string>
|
|
||||||
#include <unordered_map>
|
|
||||||
#include <unordered_set>
|
|
||||||
#include <utility>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include <thread> // NOLINT
|
|
||||||
|
|
||||||
#include <ThreadPool.h>
|
|
||||||
|
|
||||||
#include "paddle/fluid/platform/enforce.h"
|
|
||||||
|
|
||||||
namespace paddle {
|
|
||||||
namespace operators {
|
|
||||||
namespace distributed {
|
|
||||||
bool BarrierMonitor::IncreaseBarrier(const int worker_id,
|
|
||||||
const std::string &barrier) {
|
|
||||||
release_ = false;
|
|
||||||
|
|
||||||
if (barrier == BATCH_BARRIER_MESSAGE) {
|
|
||||||
VLOG(4) << "BarrierMonitor send queue recv trainer: " << worker_id;
|
|
||||||
send_barrier_queue->Push(worker_id);
|
|
||||||
} else if (barrier == FETCH_BARRIER_MESSAGE) {
|
|
||||||
VLOG(4) << "BarrierMonitor recv queue recv trainer: " << worker_id;
|
|
||||||
recv_barrier_queue->Push(worker_id);
|
|
||||||
} else {
|
|
||||||
PADDLE_THROW(platform::errors::Unavailable(
|
|
||||||
"unknown Message status %s, only "
|
|
||||||
"BATCH_BARRIER_MESSAGE/FETCH_BARRIER_MESSAGE",
|
|
||||||
barrier));
|
|
||||||
}
|
|
||||||
return Wait();
|
|
||||||
}
|
|
||||||
|
|
||||||
void BarrierMonitor::DecreaseWorker() {
|
|
||||||
std::unique_lock<std::mutex> lck(mutex_);
|
|
||||||
workers_--;
|
|
||||||
VLOG(1) << "decrement worker num to " << workers_;
|
|
||||||
}
|
|
||||||
|
|
||||||
void BarrierMonitor::Reset(int workers, BarrierType type) {
|
|
||||||
std::unique_lock<std::mutex> lk(server_mutex_);
|
|
||||||
|
|
||||||
workers_ = workers;
|
|
||||||
barrier_type = type;
|
|
||||||
|
|
||||||
send_barrier_queue->Clear();
|
|
||||||
recv_barrier_queue->Clear();
|
|
||||||
VLOG(2) << "reset monitor workers: " << workers_ << " type: " << barrier_type;
|
|
||||||
}
|
|
||||||
|
|
||||||
void BarrierMonitor::Monitor() {
|
|
||||||
while (!IsReady() && running_) {
|
|
||||||
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
|
|
||||||
VLOG(3) << "sync at first time, wait all trainer ready";
|
|
||||||
}
|
|
||||||
|
|
||||||
while (running_) {
|
|
||||||
int timer = 0;
|
|
||||||
|
|
||||||
if (IsReady()) {
|
|
||||||
Swap(true);
|
|
||||||
} else {
|
|
||||||
VLOG(4) << "running timer: " << timer << " barrier: " << barrier_type
|
|
||||||
<< " sendQ:" << send_barrier_queue->Size()
|
|
||||||
<< " recvQ: " << recv_barrier_queue->Size();
|
|
||||||
|
|
||||||
timer++;
|
|
||||||
if (max_wait_ms == -1 || timer < max_wait_ms) {
|
|
||||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
|
||||||
} else {
|
|
||||||
VLOG(1) << "time out of " << max_wait_ms
|
|
||||||
<< ", need barreir: " << barrier_type << " retry";
|
|
||||||
Swap(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bool BarrierMonitor::IsReady() {
|
|
||||||
if (barrier_type == BarrierType::kSendBarrier) {
|
|
||||||
return static_cast<int>(send_barrier_queue->Size()) == workers_;
|
|
||||||
} else {
|
|
||||||
return static_cast<int>(recv_barrier_queue->Size()) == workers_;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void BarrierMonitor::Swap(bool is_valid) {
|
|
||||||
std::unique_lock<std::mutex> lck(mutex_);
|
|
||||||
|
|
||||||
valid_ = is_valid;
|
|
||||||
release_ = true;
|
|
||||||
|
|
||||||
if (barrier_type == BarrierType::kSendBarrier) {
|
|
||||||
barrier_type = BarrierType::kRecvBarrier;
|
|
||||||
send_barrier_queue->Clear();
|
|
||||||
VLOG(4) << "barrier monitor server clean up queue and barrier";
|
|
||||||
ServerWeakup();
|
|
||||||
VLOG(4) << "barrier monitor server weak up sync to do";
|
|
||||||
WaitServerWeakup();
|
|
||||||
VLOG(4) << "barrier monitor server weak up sync done";
|
|
||||||
|
|
||||||
} else {
|
|
||||||
barrier_type = BarrierType::kSendBarrier;
|
|
||||||
recv_barrier_queue->Clear();
|
|
||||||
VLOG(4) << "barrier monitor server switch to send barrier";
|
|
||||||
}
|
|
||||||
|
|
||||||
worker_cv_.notify_all();
|
|
||||||
}
|
|
||||||
|
|
||||||
void BarrierMonitor::Stop() {
|
|
||||||
valid_ = true;
|
|
||||||
release_ = true;
|
|
||||||
running_ = false;
|
|
||||||
|
|
||||||
barrier_type = BarrierType::kRecvBarrier;
|
|
||||||
send_barrier_queue->Clear();
|
|
||||||
recv_barrier_queue->Clear();
|
|
||||||
|
|
||||||
worker_cv_.notify_all();
|
|
||||||
server_cv_.notify_all();
|
|
||||||
|
|
||||||
if (monitor_thread_) monitor_thread_->join();
|
|
||||||
monitor_thread_ = nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool BarrierMonitor::Wait() {
|
|
||||||
std::unique_lock<std::mutex> lk(mutex_);
|
|
||||||
worker_cv_.wait(lk, [this] { return (release_); });
|
|
||||||
return valid_;
|
|
||||||
}
|
|
||||||
|
|
||||||
void BarrierMonitor::WaitServerWeakup() {
|
|
||||||
std::unique_lock<std::mutex> lk(server_mutex_);
|
|
||||||
server_cv_.wait(lk);
|
|
||||||
}
|
|
||||||
|
|
||||||
void BarrierMonitor::ServerWeakup() { server_cv_.notify_all(); }
|
|
||||||
|
|
||||||
std::once_flag BarrierMonitor::init_flag_;
|
|
||||||
std::unique_ptr<BarrierMonitor> BarrierMonitor::monitor_(nullptr);
|
|
||||||
|
|
||||||
} // namespace distributed
|
|
||||||
} // namespace operators
|
|
||||||
} // namespace paddle
|
|
||||||
@ -1,186 +0,0 @@
|
|||||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <gflags/gflags.h>
|
|
||||||
|
|
||||||
#include <chrono> // NOLINT
|
|
||||||
#include <deque>
|
|
||||||
#include <functional>
|
|
||||||
#include <future> // NOLINT
|
|
||||||
#include <memory>
|
|
||||||
#include <string>
|
|
||||||
#include <unordered_map>
|
|
||||||
#include <unordered_set>
|
|
||||||
#include <utility>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include <thread> // NOLINT
|
|
||||||
|
|
||||||
#include <ThreadPool.h>
|
|
||||||
|
|
||||||
#include "paddle/fluid/operators/distributed/rpc_server.h"
|
|
||||||
#include "paddle/fluid/platform/enforce.h"
|
|
||||||
|
|
||||||
namespace paddle {
|
|
||||||
namespace operators {
|
|
||||||
namespace distributed {
|
|
||||||
|
|
||||||
enum BarrierType { kSendBarrier, kRecvBarrier };
|
|
||||||
|
|
||||||
constexpr int64_t kMaxWaitMS = 120000;
|
|
||||||
|
|
||||||
template <typename T>
|
|
||||||
class BlockingQueueForBarrier {
|
|
||||||
public:
|
|
||||||
explicit BlockingQueueForBarrier(size_t capacity) : capacity_(capacity) {
|
|
||||||
PADDLE_ENFORCE_GT(capacity_, 0,
|
|
||||||
platform::errors::InvalidArgument(
|
|
||||||
"The capacity must be greater than 0."));
|
|
||||||
}
|
|
||||||
|
|
||||||
bool Push(const T &elem) {
|
|
||||||
{
|
|
||||||
std::unique_lock<std::mutex> lock(mutex_);
|
|
||||||
worker_cv_.wait(lock, [&] { return queue_.size() < capacity_; });
|
|
||||||
queue_.push_back(elem);
|
|
||||||
}
|
|
||||||
worker_cv_.notify_one();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool Push(T &&elem) {
|
|
||||||
{
|
|
||||||
std::unique_lock<std::mutex> lock(mutex_);
|
|
||||||
worker_cv_.wait(lock, [&] { return queue_.size() < capacity_; });
|
|
||||||
queue_.emplace_back(std::move(elem));
|
|
||||||
}
|
|
||||||
worker_cv_.notify_one();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
T Pop() {
|
|
||||||
std::unique_lock<std::mutex> lock(mutex_);
|
|
||||||
worker_cv_.wait(lock, [=] { return !queue_.empty(); });
|
|
||||||
T rc(std::move(queue_.front()));
|
|
||||||
queue_.pop_front();
|
|
||||||
worker_cv_.notify_one();
|
|
||||||
return rc;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t Cap() const {
|
|
||||||
std::lock_guard<std::mutex> lock(mutex_);
|
|
||||||
return capacity_;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t Size() const {
|
|
||||||
std::lock_guard<std::mutex> lock(mutex_);
|
|
||||||
return queue_.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
void Clear() {
|
|
||||||
std::lock_guard<std::mutex> lock(mutex_);
|
|
||||||
std::deque<T>().swap(queue_);
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
const size_t capacity_;
|
|
||||||
std::deque<T> queue_;
|
|
||||||
|
|
||||||
mutable std::mutex mutex_;
|
|
||||||
std::condition_variable worker_cv_;
|
|
||||||
};
|
|
||||||
|
|
||||||
class BarrierMonitor {
|
|
||||||
public:
|
|
||||||
explicit BarrierMonitor(int workers)
|
|
||||||
: BarrierMonitor(workers, BarrierType::kRecvBarrier, kMaxWaitMS) {}
|
|
||||||
|
|
||||||
explicit BarrierMonitor(int workers, BarrierType type, int64_t max_wait_times)
|
|
||||||
: workers_(workers), barrier_type(type), max_wait_ms(max_wait_times) {
|
|
||||||
PADDLE_ENFORCE_GT(workers, 0, platform::errors::InvalidArgument(
|
|
||||||
"trainers must have one or more"));
|
|
||||||
|
|
||||||
send_barrier_queue =
|
|
||||||
std::make_shared<BlockingQueueForBarrier<int>>(workers);
|
|
||||||
recv_barrier_queue =
|
|
||||||
std::make_shared<BlockingQueueForBarrier<int>>(workers);
|
|
||||||
|
|
||||||
running_ = true;
|
|
||||||
monitor_thread_.reset(
|
|
||||||
new std::thread(std::bind(&BarrierMonitor::Monitor, this)));
|
|
||||||
}
|
|
||||||
|
|
||||||
static BarrierMonitor *Init(int workers) {
|
|
||||||
InitImpl(workers);
|
|
||||||
return GetInstance();
|
|
||||||
}
|
|
||||||
|
|
||||||
static BarrierMonitor *GetInstance() { return monitor_.get(); }
|
|
||||||
|
|
||||||
bool IncreaseBarrier(const int worker_id, const std::string &barrier);
|
|
||||||
|
|
||||||
void DecreaseWorker();
|
|
||||||
|
|
||||||
int GetWorkerNum() { return workers_; }
|
|
||||||
|
|
||||||
void Monitor();
|
|
||||||
|
|
||||||
void Swap(bool is_valid);
|
|
||||||
|
|
||||||
void Stop();
|
|
||||||
|
|
||||||
bool IsReady();
|
|
||||||
|
|
||||||
bool Wait();
|
|
||||||
|
|
||||||
void WaitServerWeakup();
|
|
||||||
|
|
||||||
void ServerWeakup();
|
|
||||||
|
|
||||||
void WorkerWeakup();
|
|
||||||
|
|
||||||
void Reset(int workers, BarrierType type);
|
|
||||||
|
|
||||||
private:
|
|
||||||
// Init is called by GetInstance.
|
|
||||||
static void InitImpl(int workers) {
|
|
||||||
monitor_.reset(new BarrierMonitor(workers));
|
|
||||||
}
|
|
||||||
|
|
||||||
static std::once_flag init_flag_;
|
|
||||||
static std::unique_ptr<BarrierMonitor> monitor_;
|
|
||||||
|
|
||||||
int workers_;
|
|
||||||
bool running_ = false;
|
|
||||||
bool valid_ = false;
|
|
||||||
bool release_ = false;
|
|
||||||
|
|
||||||
std::condition_variable worker_cv_;
|
|
||||||
std::condition_variable server_cv_;
|
|
||||||
|
|
||||||
std::mutex server_mutex_;
|
|
||||||
std::mutex mutex_;
|
|
||||||
|
|
||||||
BarrierType barrier_type;
|
|
||||||
int64_t max_wait_ms;
|
|
||||||
std::unique_ptr<std::thread> monitor_thread_{nullptr};
|
|
||||||
std::shared_ptr<BlockingQueueForBarrier<int>> send_barrier_queue;
|
|
||||||
std::shared_ptr<BlockingQueueForBarrier<int>> recv_barrier_queue;
|
|
||||||
};
|
|
||||||
|
|
||||||
} // namespace distributed
|
|
||||||
} // namespace operators
|
|
||||||
} // namespace paddle
|
|
||||||
@ -1,219 +0,0 @@
|
|||||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
"""test f1 listen and serv_op."""
|
|
||||||
|
|
||||||
from __future__ import print_function
|
|
||||||
|
|
||||||
import paddle
|
|
||||||
import paddle.fluid as fluid
|
|
||||||
from paddle.fluid import Program
|
|
||||||
import os
|
|
||||||
import signal
|
|
||||||
import subprocess
|
|
||||||
import time
|
|
||||||
import unittest
|
|
||||||
from multiprocessing import Process
|
|
||||||
from op_test import OpTest
|
|
||||||
import numpy
|
|
||||||
import urllib
|
|
||||||
import sys
|
|
||||||
from dist_test_utils import *
|
|
||||||
|
|
||||||
cache_path = os.path.expanduser('~/.cache/paddle/dataset')
|
|
||||||
|
|
||||||
|
|
||||||
def run_trainer(use_cuda, sync_mode, ip, port, trainers, trainer_id):
|
|
||||||
'''
|
|
||||||
This function is run trainer.
|
|
||||||
Args:
|
|
||||||
use_cuda (bool): whether use cuda.
|
|
||||||
sync_mode (nouse): specify sync mode.
|
|
||||||
ip (string): the ip address.
|
|
||||||
port (string): the port for listening.
|
|
||||||
trainers (int): the count of trainer.
|
|
||||||
trainer_id (int): the id of trainer.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
None
|
|
||||||
'''
|
|
||||||
x = fluid.layers.data(name='x', shape=[1], dtype='float32')
|
|
||||||
y_predict = fluid.layers.fc(input=x, size=1, act=None)
|
|
||||||
y = fluid.layers.data(name='y', shape=[1], dtype='float32')
|
|
||||||
# loss function
|
|
||||||
cost = fluid.layers.square_error_cost(input=y_predict, label=y)
|
|
||||||
avg_cost = fluid.layers.mean(cost)
|
|
||||||
# optimizer
|
|
||||||
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
|
|
||||||
sgd_optimizer.minimize(avg_cost)
|
|
||||||
with open("{}/trainer_recv_program.dms".format(cache_path), "rb") as f:
|
|
||||||
trainer_recv_program_desc_str = f.read()
|
|
||||||
with open("{}/trainer_main_program.dms".format(cache_path), "rb") as f:
|
|
||||||
trainer_main_program_desc_str = f.read()
|
|
||||||
with open("{}/trainer_send_program.dms".format(cache_path), "rb") as f:
|
|
||||||
trainer_send_program_desc_str = f.read()
|
|
||||||
recv_program = Program.parse_from_string(trainer_recv_program_desc_str)
|
|
||||||
main_program = Program.parse_from_string(trainer_main_program_desc_str)
|
|
||||||
send_program = Program.parse_from_string(trainer_send_program_desc_str)
|
|
||||||
|
|
||||||
trainer_startup_program = fluid.default_startup_program()
|
|
||||||
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
|
|
||||||
exe = fluid.Executor(place)
|
|
||||||
|
|
||||||
exe.run(trainer_startup_program)
|
|
||||||
for i in range(5):
|
|
||||||
exe.run(recv_program)
|
|
||||||
exe.run(fluid.default_main_program(),
|
|
||||||
feed={
|
|
||||||
"x": numpy.array([1, 2]).astype('float32').reshape(2, 1),
|
|
||||||
"y": numpy.array([2, 3]).astype('float32').reshape(2, 1)
|
|
||||||
})
|
|
||||||
exe.run(send_program)
|
|
||||||
|
|
||||||
|
|
||||||
def run_pserver(use_cuda, sync_mode, ip, port, trainers, trainer_id):
|
|
||||||
'''
|
|
||||||
This function is run trainer.
|
|
||||||
Args:
|
|
||||||
use_cuda (bool): whether use cuda.
|
|
||||||
sync_mode (nouse): specify sync mode.
|
|
||||||
ip (string): the ip address.
|
|
||||||
port (string): the port for listening.
|
|
||||||
trainers (int): the count of trainer.
|
|
||||||
trainer_id (int): the id of trainer.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
None
|
|
||||||
'''
|
|
||||||
remove_ps_flag(os.getpid())
|
|
||||||
x = fluid.layers.data(name='x', shape=[1], dtype='float32')
|
|
||||||
y_predict = fluid.layers.fc(input=x, size=1, act=None)
|
|
||||||
y = fluid.layers.data(name='y', shape=[1], dtype='float32')
|
|
||||||
# loss function
|
|
||||||
cost = fluid.layers.square_error_cost(input=y_predict, label=y)
|
|
||||||
avg_cost = fluid.layers.mean(cost)
|
|
||||||
# optimizer
|
|
||||||
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
|
|
||||||
sgd_optimizer.minimize(avg_cost)
|
|
||||||
with open("{}/pserver_startup_program.dms".format(cache_path), "rb") as f:
|
|
||||||
pserver_startup_program_desc_str = f.read()
|
|
||||||
with open("{}/pserver_main_program.dms".format(cache_path), "rb") as f:
|
|
||||||
pserver_main_program_desc_str = f.read()
|
|
||||||
|
|
||||||
startup_program = Program.parse_from_string(
|
|
||||||
pserver_startup_program_desc_str)
|
|
||||||
main_program = Program.parse_from_string(pserver_main_program_desc_str)
|
|
||||||
|
|
||||||
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
|
|
||||||
exe = fluid.Executor(place)
|
|
||||||
exe.run(startup_program)
|
|
||||||
exe.run(main_program)
|
|
||||||
|
|
||||||
|
|
||||||
class TestFlListenAndServOp(unittest.TestCase):
|
|
||||||
"""This class is Test Fl Listen And ServOp."""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
"""This function si set Up."""
|
|
||||||
self.ps_timeout = 5
|
|
||||||
self.ip = "127.0.0.1"
|
|
||||||
self.port = "6000"
|
|
||||||
self.trainers = 2
|
|
||||||
self.trainer_id = 0
|
|
||||||
|
|
||||||
def _start_pserver(self, use_cuda, sync_mode, pserver_func):
|
|
||||||
"""This function is start pserver."""
|
|
||||||
p = Process(
|
|
||||||
target=pserver_func,
|
|
||||||
args=(use_cuda, sync_mode, self.ip, self.port, self.trainers,
|
|
||||||
self.trainer_id))
|
|
||||||
p.daemon = True
|
|
||||||
p.start()
|
|
||||||
return p
|
|
||||||
|
|
||||||
def _start_trainer0(self, use_cuda, sync_mode, pserver_func):
|
|
||||||
"""This function is start trainer0."""
|
|
||||||
p = Process(
|
|
||||||
target=pserver_func,
|
|
||||||
args=(use_cuda, sync_mode, self.ip, self.port, self.trainers, 0))
|
|
||||||
p.daemon = True
|
|
||||||
p.start()
|
|
||||||
return p
|
|
||||||
|
|
||||||
def _start_trainer1(self, use_cuda, sync_mode, pserver_func):
|
|
||||||
"""This function is start trainer1."""
|
|
||||||
p = Process(
|
|
||||||
target=pserver_func,
|
|
||||||
args=(use_cuda, sync_mode, self.ip, self.port, self.trainers, 1))
|
|
||||||
p.daemon = True
|
|
||||||
p.start()
|
|
||||||
return p
|
|
||||||
|
|
||||||
def _wait_ps_ready(self, pid):
|
|
||||||
"""This function is wait ps ready."""
|
|
||||||
start_left_time = self.ps_timeout
|
|
||||||
sleep_time = 0.5
|
|
||||||
while True:
|
|
||||||
assert start_left_time >= 0, "wait ps ready failed"
|
|
||||||
time.sleep(sleep_time)
|
|
||||||
try:
|
|
||||||
os.stat("/tmp/paddle.%d.port" % pid)
|
|
||||||
return
|
|
||||||
except os.error:
|
|
||||||
start_left_time -= sleep_time
|
|
||||||
|
|
||||||
def test_rpc_interfaces(self):
|
|
||||||
"""TODO(Yancey1989): need to make sure the rpc interface correctly."""
|
|
||||||
# TODO(Yancey1989): need to make sure the rpc interface correctly.
|
|
||||||
pass
|
|
||||||
|
|
||||||
def test_handle_signal_in_serv_op(self):
|
|
||||||
"""run pserver on CPU in sync mode."""
|
|
||||||
# run pserver on CPU in sync mode
|
|
||||||
if sys.platform == 'win32' or sys.platform == 'sys.platform':
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
print(sys.platform)
|
|
||||||
file_list = [
|
|
||||||
'pserver_startup_program.dms', 'pserver_main_program.dms',
|
|
||||||
'trainer_recv_program.dms', 'trainer_main_program.dms',
|
|
||||||
'trainer_send_program.dms'
|
|
||||||
]
|
|
||||||
if not os.path.exists(cache_path):
|
|
||||||
os.makedirs(cache_path)
|
|
||||||
prefix = 'wget --no-check-certificate https://paddlefl.bj.bcebos.com/test_fl_listen_and_serv/'
|
|
||||||
for f in file_list:
|
|
||||||
if not os.path.exists('{}/{}'.format(cache_path, f)):
|
|
||||||
cmd = "wget --no-check-certificate https://paddlefl.bj.bcebos.com/test_fl_listen_and_serv/{} -P {}/".format(
|
|
||||||
f, cache_path)
|
|
||||||
os.system(cmd)
|
|
||||||
p1 = self._start_pserver(False, True, run_pserver)
|
|
||||||
self._wait_ps_ready(p1.pid)
|
|
||||||
time.sleep(5)
|
|
||||||
t1 = self._start_trainer0(False, True, run_trainer)
|
|
||||||
time.sleep(2)
|
|
||||||
t2 = self._start_trainer1(False, True, run_trainer)
|
|
||||||
# raise SIGTERM to pserver
|
|
||||||
time.sleep(2)
|
|
||||||
cmd_del = "rm trainer*dms* pserver*dms*"
|
|
||||||
os.system(cmd_del)
|
|
||||||
os.kill(p1.pid, signal.SIGINT)
|
|
||||||
p1.join()
|
|
||||||
os.kill(t1.pid, signal.SIGINT)
|
|
||||||
t1.join()
|
|
||||||
os.kill(t2.pid, signal.SIGINT)
|
|
||||||
t2.join()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
||||||
Loading…
Reference in new issue