【paddle.fleet】fleet_util move to paddle.fleet (#25805)
* test=develop,test=document_fix, remove the out args * fleet_util move to paddle.fleet Co-authored-by: WuHaobo <wuhaobo1994@gmail.com> Co-authored-by: tangwei12 <tangwei12@baidu.com>revert-24895-update_cub
123malin
5 years ago
committed by
GitHub
parent
0d4ce6ac5d
commit
2191a08317
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,18 @@
|
|||||||
|
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from .fs import *
|
||||||
|
from .http_server import KVHandler, KVHTTPServer, KVServer
|
||||||
|
|
||||||
|
__all__ = ['KVHandler', 'KVHTTPServer', 'KVServer'] + fs.__all__
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,195 @@
|
|||||||
|
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""Http Server."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import six
|
||||||
|
# NOTE: HTTPServer has a different name in python2 and python3
|
||||||
|
if six.PY2:
|
||||||
|
from BaseHTTPServer import HTTPServer
|
||||||
|
import SimpleHTTPServer
|
||||||
|
else:
|
||||||
|
from http.server import HTTPServer
|
||||||
|
import http.server as SimpleHTTPServer
|
||||||
|
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
import socket
|
||||||
|
|
||||||
|
|
||||||
|
def get_logger(name, level, fmt):
|
||||||
|
logger = logging.getLogger(name)
|
||||||
|
logger.setLevel(level)
|
||||||
|
handler = logging.FileHandler('http.log', mode='w')
|
||||||
|
formatter = logging.Formatter(fmt=fmt)
|
||||||
|
handler.setFormatter(formatter)
|
||||||
|
logger.addHandler(handler)
|
||||||
|
return logger
|
||||||
|
|
||||||
|
|
||||||
|
_http_server_logger = get_logger(
|
||||||
|
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s')
|
||||||
|
|
||||||
|
|
||||||
|
class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
|
||||||
|
"""
|
||||||
|
kv handler class for kv http server,
|
||||||
|
it defines the way to get/set kv in server.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
"""
|
||||||
|
get method for kv handler, get value according to key.
|
||||||
|
"""
|
||||||
|
log_str = "GET " + self.address_string() + self.path
|
||||||
|
paths = self.path.split('/')
|
||||||
|
if len(paths) < 3:
|
||||||
|
print('len of request path must be 3: ' + self.path)
|
||||||
|
self.send_status_code(400)
|
||||||
|
return
|
||||||
|
_, scope, key = paths
|
||||||
|
with self.server.kv_lock:
|
||||||
|
value = self.server.kv.get(scope, {}).get(key)
|
||||||
|
if value is None:
|
||||||
|
log_str += ' , key not found: ' + key
|
||||||
|
self.send_status_code(404)
|
||||||
|
else:
|
||||||
|
log_str += ' , key found: ' + key
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Length", str(len(value)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(value)
|
||||||
|
_http_server_logger.info(log_str)
|
||||||
|
|
||||||
|
def do_PUT(self):
|
||||||
|
"""
|
||||||
|
put method for kv handler, set value according to key.
|
||||||
|
"""
|
||||||
|
log_str = "PUT " + self.address_string() + self.path
|
||||||
|
paths = self.path.split('/')
|
||||||
|
if len(paths) < 3:
|
||||||
|
print('len of request path must be 3: ' + self.path)
|
||||||
|
self.send_status_code(400)
|
||||||
|
return
|
||||||
|
_, scope, key = paths
|
||||||
|
content_length = int(self.headers['Content-Length'])
|
||||||
|
try:
|
||||||
|
value = self.rfile.read(content_length)
|
||||||
|
except:
|
||||||
|
print("receive error invalid request")
|
||||||
|
self.send_status_code(404)
|
||||||
|
return
|
||||||
|
with self.server.kv_lock:
|
||||||
|
if self.server.kv.get(scope) is None:
|
||||||
|
self.server.kv[scope] = {}
|
||||||
|
self.server.kv[scope][key] = value
|
||||||
|
self.send_status_code(200)
|
||||||
|
_http_server_logger.info(log_str)
|
||||||
|
|
||||||
|
def do_DELETE(self):
|
||||||
|
"""
|
||||||
|
delete method for kv handler, set value according to key.
|
||||||
|
"""
|
||||||
|
log_str = "DELETE " + self.address_string() + self.path
|
||||||
|
paths = self.path.split('/')
|
||||||
|
if len(paths) < 3:
|
||||||
|
print('len of request path must be 3: ' + self.path)
|
||||||
|
self.send_status_code(400)
|
||||||
|
return
|
||||||
|
_, scope, key = paths
|
||||||
|
with self.server.delete_kv_lock:
|
||||||
|
if self.server.delete_kv.get(scope) is None:
|
||||||
|
self.server.delete_kv[scope] = []
|
||||||
|
self.server.delete_kv[scope].append(key)
|
||||||
|
self.send_status_code(200)
|
||||||
|
_http_server_logger.info(log_str)
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
"""
|
||||||
|
ignore all logging messages in kv handler.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
def send_status_code(self, code):
|
||||||
|
"""
|
||||||
|
send status code back to client.
|
||||||
|
"""
|
||||||
|
self.send_response(code)
|
||||||
|
self.send_header("Content-Length", 0)
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
|
||||||
|
class KVHTTPServer(HTTPServer, object):
|
||||||
|
"""
|
||||||
|
it is a http server storing kv pairs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, port, handler):
|
||||||
|
"""Init."""
|
||||||
|
super(KVHTTPServer, self).__init__(('', port), handler)
|
||||||
|
self.delete_kv_lock = threading.Lock()
|
||||||
|
self.delete_kv = {}
|
||||||
|
self.kv_lock = threading.Lock()
|
||||||
|
self.kv = {}
|
||||||
|
|
||||||
|
def get_deleted_size(self, key):
|
||||||
|
"""
|
||||||
|
get deleted size in key.
|
||||||
|
"""
|
||||||
|
ret = 0
|
||||||
|
with self.delete_kv_lock:
|
||||||
|
ret = self.delete_kv.get(key, 0)
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
class KVServer:
|
||||||
|
"""
|
||||||
|
it is a server storing kv pairs, has a http server inside.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, port, size={}):
|
||||||
|
"""Init."""
|
||||||
|
self.http_server = KVHTTPServer(port, KVHandler)
|
||||||
|
self.listen_thread = None
|
||||||
|
self.size = {}
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
"""
|
||||||
|
start server until user calls stop to let it quit.
|
||||||
|
"""
|
||||||
|
self.listen_thread = threading.Thread(
|
||||||
|
target=lambda: self.http_server.serve_forever())
|
||||||
|
self.listen_thread.start()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""
|
||||||
|
stop server and clear its resources.
|
||||||
|
"""
|
||||||
|
self.http_server.shutdown()
|
||||||
|
self.listen_thread.join()
|
||||||
|
self.http_server.server_close()
|
||||||
|
|
||||||
|
def shoud_stop(self):
|
||||||
|
"""
|
||||||
|
return whether the server should stop.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
ret(bool): whether the server should stop
|
||||||
|
"""
|
||||||
|
for key in self.size:
|
||||||
|
s = self.http_server.get_deleted_size(key)
|
||||||
|
if s != self.size.get(key, 0):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
@ -0,0 +1,171 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""Test cloud role maker."""
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
import paddle.fleet.base.role_maker as role_maker
|
||||||
|
|
||||||
|
|
||||||
|
class TestRoleMakerBase(unittest.TestCase):
|
||||||
|
"""
|
||||||
|
Test cases for RoleMakerBase
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_rolemaker_base(self):
|
||||||
|
role = role_maker.RoleMakerBase()
|
||||||
|
self.assertRaises(Exception, role.is_worker)
|
||||||
|
self.assertRaises(Exception, role.is_server)
|
||||||
|
self.assertRaises(Exception, role.is_first_worker)
|
||||||
|
self.assertRaises(Exception, role.worker_num)
|
||||||
|
self.assertRaises(Exception, role.server_num)
|
||||||
|
self.assertRaises(Exception, role.worker_index)
|
||||||
|
self.assertRaises(Exception, role.server_index)
|
||||||
|
self.assertRaises(Exception, role.role_id)
|
||||||
|
|
||||||
|
trainer_endpoints = role.get_trainer_endpoints()
|
||||||
|
self.assertTrue(len(trainer_endpoints) == 0)
|
||||||
|
pserver_endpoints = role.get_pserver_endpoints()
|
||||||
|
self.assertTrue(len(pserver_endpoints) == 0)
|
||||||
|
|
||||||
|
print(role.to_string())
|
||||||
|
self.assertTrue(role._all_gather(role._node_type_comm, 1) is None)
|
||||||
|
self.assertTrue(role._all_reduce(role._node_type_comm, 1) is None)
|
||||||
|
role._barrier(role._node_type_comm)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCloudRoleMaker(unittest.TestCase):
|
||||||
|
"""
|
||||||
|
Test cases for PaddleCloudRoleMaker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
"""Set up, set envs."""
|
||||||
|
os.environ["PADDLE_TRAINERS_NUM"] = "2"
|
||||||
|
os.environ[
|
||||||
|
"PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001"
|
||||||
|
os.environ[
|
||||||
|
"PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.2:36001"
|
||||||
|
os.environ["POD_IP"] = "127.0.0.1"
|
||||||
|
|
||||||
|
def test_tr_rolemaker(self):
|
||||||
|
"""Test tr rolenamer."""
|
||||||
|
os.environ["TRAINING_ROLE"] = "TRAINER"
|
||||||
|
os.environ["PADDLE_TRAINER_ID"] = "0"
|
||||||
|
|
||||||
|
try:
|
||||||
|
import netifaces
|
||||||
|
except:
|
||||||
|
print("warning: no netifaces, skip test_tr_rolemaker")
|
||||||
|
return
|
||||||
|
|
||||||
|
ro = role_maker.PaddleCloudRoleMaker(
|
||||||
|
is_collective=False, init_gloo=False)
|
||||||
|
self.assertTrue(ro.is_worker())
|
||||||
|
self.assertFalse(ro.is_server())
|
||||||
|
self.assertEqual(ro.worker_num(), 2)
|
||||||
|
self.assertTrue(ro.is_first_worker())
|
||||||
|
worker_endpoints = ro.get_trainer_endpoints()
|
||||||
|
self.assertEqual(worker_endpoints[0], '127.0.0.1:36001')
|
||||||
|
self.assertEqual(ro.role_id(), 0)
|
||||||
|
|
||||||
|
def test_tr_rolemaker_collective(self):
|
||||||
|
ro = role_maker.PaddleCloudRoleMaker(is_collective=True)
|
||||||
|
self.assertEqual(ro.worker_num(), 2)
|
||||||
|
|
||||||
|
def test_ps_rolemaker(self):
|
||||||
|
"""Test ps rolemaker."""
|
||||||
|
os.environ["TRAINING_ROLE"] = "PSERVER"
|
||||||
|
os.environ["POD_IP"] = "127.0.0.1"
|
||||||
|
os.environ["PADDLE_PORT"] = "36001"
|
||||||
|
|
||||||
|
try:
|
||||||
|
import netifaces
|
||||||
|
except:
|
||||||
|
print("warning: no netifaces, skip test_ps_rolemaker")
|
||||||
|
return
|
||||||
|
|
||||||
|
ro = role_maker.PaddleCloudRoleMaker(
|
||||||
|
is_collective=False, init_gloo=False)
|
||||||
|
self.assertEqual(ro.server_index(), 0)
|
||||||
|
self.assertFalse(ro.is_worker())
|
||||||
|
self.assertTrue(ro.is_server())
|
||||||
|
self.assertEqual(ro.server_num(), 2)
|
||||||
|
pserver_endpoints = ro.get_pserver_endpoints()
|
||||||
|
self.assertEqual(pserver_endpoints[0], '127.0.0.1:36001')
|
||||||
|
self.assertTrue(ro._all_gather(ro._all_comm, 1) is None)
|
||||||
|
self.assertTrue(ro._all_reduce(ro._all_comm, 1) is None)
|
||||||
|
|
||||||
|
def test_traing_role(self):
|
||||||
|
"""Test training role."""
|
||||||
|
os.environ["TRAINING_ROLE"] = "TEST"
|
||||||
|
try:
|
||||||
|
import netifaces
|
||||||
|
except:
|
||||||
|
print("warning: no netifaces, skip test_training_role")
|
||||||
|
return
|
||||||
|
|
||||||
|
ro = role_maker.PaddleCloudRoleMaker(is_collective=False)
|
||||||
|
self.assertRaises(ValueError, ro.generate_role)
|
||||||
|
|
||||||
|
|
||||||
|
class TestUserDefinedRoleMaker(unittest.TestCase):
|
||||||
|
"""
|
||||||
|
Test cases for UserDefinedRoleMaker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_ps_rolemaker(self):
|
||||||
|
try:
|
||||||
|
import netifaces
|
||||||
|
except:
|
||||||
|
print("warning: no netifaces, skip test_ps_rolemaker")
|
||||||
|
return
|
||||||
|
|
||||||
|
ro = role_maker.UserDefinedRoleMaker(
|
||||||
|
is_collective=False,
|
||||||
|
init_gloo=False,
|
||||||
|
server_endpoints="127.0.0.1:36001,127.0.0.1:36001",
|
||||||
|
role=role_maker.Role.SERVER,
|
||||||
|
current_id=0,
|
||||||
|
worker_num=2)
|
||||||
|
self.assertEqual(ro.server_num(), 2)
|
||||||
|
ro.generate_role()
|
||||||
|
self.assertTrue(ro.is_server())
|
||||||
|
self.assertEqual(ro.role_id(), 0)
|
||||||
|
|
||||||
|
def test_tr_rolemaker(self):
|
||||||
|
try:
|
||||||
|
import netifaces
|
||||||
|
except:
|
||||||
|
print("warning: no netifaces, skip test_tr_rolemaker")
|
||||||
|
return
|
||||||
|
|
||||||
|
ro = role_maker.UserDefinedRoleMaker(
|
||||||
|
is_collective=False,
|
||||||
|
init_gloo=False,
|
||||||
|
server_endpoints="127.0.0.1:36001,127.0.0.1:36001",
|
||||||
|
role=role_maker.Role.WORKER,
|
||||||
|
current_id=0,
|
||||||
|
worker_num=2)
|
||||||
|
self.assertIn("127.0.0.1:36001", ro.get_pserver_endpoints())
|
||||||
|
self.assertTrue(ro.is_worker())
|
||||||
|
self.assertEqual(ro.role_id(), 0)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue