Merge pull request #14873 from colourful-tree/develop
add pslib(pserver) to paddle, an industrial scale high performance parameter server libraryfor_weibo
colourful-tree
6 years ago
committed by
GitHub
commit
44ad2f4479
@ -0,0 +1,78 @@
|
|||||||
|
# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
IF(NOT ${WITH_LIBMCT})
|
||||||
|
return()
|
||||||
|
ENDIF(NOT ${WITH_LIBMCT})
|
||||||
|
|
||||||
|
IF(WIN32 OR APPLE)
|
||||||
|
MESSAGE(WARNING
|
||||||
|
"Windows or Mac is not supported with LIBMCT in Paddle yet."
|
||||||
|
"Force WITH_LIBMCT=OFF")
|
||||||
|
SET(WITH_LIBMCT OFF CACHE STRING "Disable LIBMCT package in Windows and MacOS" FORCE)
|
||||||
|
return()
|
||||||
|
ENDIF()
|
||||||
|
|
||||||
|
INCLUDE(ExternalProject)
|
||||||
|
|
||||||
|
SET(LIBMCT_PROJECT "extern_libmct")
|
||||||
|
IF((NOT DEFINED LIBMCT_VER) OR (NOT DEFINED LIBMCT_URL))
|
||||||
|
MESSAGE(STATUS "use pre defined download url")
|
||||||
|
SET(LIBMCT_VER "0.1.0" CACHE STRING "" FORCE)
|
||||||
|
SET(LIBMCT_NAME "libmct" CACHE STRING "" FORCE)
|
||||||
|
SET(LIBMCT_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${LIBMCT_VER}/${LIBMCT_NAME}.tar.gz" CACHE STRING "" FORCE)
|
||||||
|
ENDIF()
|
||||||
|
MESSAGE(STATUS "LIBMCT_NAME: ${LIBMCT_NAME}, LIBMCT_URL: ${LIBMCT_URL}")
|
||||||
|
SET(LIBMCT_SOURCE_DIR "${THIRD_PARTY_PATH}/libmct")
|
||||||
|
SET(LIBMCT_DOWNLOAD_DIR "${LIBMCT_SOURCE_DIR}/src/${LIBMCT_PROJECT}")
|
||||||
|
SET(LIBMCT_DST_DIR "libmct")
|
||||||
|
SET(LIBMCT_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
|
||||||
|
SET(LIBMCT_INSTALL_DIR ${LIBMCT_INSTALL_ROOT}/${LIBMCT_DST_DIR})
|
||||||
|
SET(LIBMCT_ROOT ${LIBMCT_INSTALL_DIR})
|
||||||
|
SET(LIBMCT_INC_DIR ${LIBMCT_ROOT}/include)
|
||||||
|
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${LIBMCT_ROOT}/lib")
|
||||||
|
|
||||||
|
INCLUDE_DIRECTORIES(${LIBMCT_INC_DIR})
|
||||||
|
|
||||||
|
FILE(WRITE ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
|
||||||
|
"PROJECT(LIBMCT)\n"
|
||||||
|
"cmake_minimum_required(VERSION 3.0)\n"
|
||||||
|
"install(DIRECTORY ${LIBMCT_NAME}/include ${LIBMCT_NAME}/lib \n"
|
||||||
|
" DESTINATION ${LIBMCT_DST_DIR})\n")
|
||||||
|
|
||||||
|
ExternalProject_Add(
|
||||||
|
${LIBMCT_PROJECT}
|
||||||
|
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||||
|
PREFIX ${LIBMCT_SOURCE_DIR}
|
||||||
|
DOWNLOAD_DIR ${LIBMCT_DOWNLOAD_DIR}
|
||||||
|
DOWNLOAD_COMMAND wget --no-check-certificate ${LIBMCT_URL} -c -q -O ${LIBMCT_NAME}.tar.gz
|
||||||
|
&& tar zxvf ${LIBMCT_NAME}.tar.gz
|
||||||
|
DOWNLOAD_NO_PROGRESS 1
|
||||||
|
UPDATE_COMMAND ""
|
||||||
|
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBMCT_INSTALL_ROOT}
|
||||||
|
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LIBMCT_INSTALL_ROOT}
|
||||||
|
)
|
||||||
|
|
||||||
|
if (${CMAKE_VERSION} VERSION_LESS "3.3.0" OR NOT WIN32)
|
||||||
|
set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/boost_dummy.c)
|
||||||
|
file(WRITE ${dummyfile} "const char *dummy = \"${dummyfile}\";")
|
||||||
|
add_library(libmct STATIC ${dummyfile})
|
||||||
|
else()
|
||||||
|
add_library(libmct INTERFACE)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
#ADD_LIBRARY(libmct SHARED IMPORTED GLOBAL)
|
||||||
|
ADD_DEPENDENCIES(libmct ${LIBMCT_PROJECT})
|
||||||
|
LIST(APPEND external_project_dependencies libmct)
|
||||||
|
|
||||||
@ -0,0 +1,77 @@
|
|||||||
|
# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
IF(NOT ${WITH_PSLIB})
|
||||||
|
return()
|
||||||
|
ENDIF(NOT ${WITH_PSLIB})
|
||||||
|
|
||||||
|
IF(WIN32 OR APPLE)
|
||||||
|
MESSAGE(WARNING
|
||||||
|
"Windows or Mac is not supported with PSLIB in Paddle yet."
|
||||||
|
"Force WITH_PSLIB=OFF")
|
||||||
|
SET(WITH_PSLIB OFF CACHE STRING "Disable PSLIB package in Windows and MacOS" FORCE)
|
||||||
|
return()
|
||||||
|
ENDIF()
|
||||||
|
|
||||||
|
INCLUDE(ExternalProject)
|
||||||
|
|
||||||
|
SET(PSLIB_PROJECT "extern_pslib")
|
||||||
|
IF((NOT DEFINED PSLIB_VER) OR (NOT DEFINED PSLIB_URL))
|
||||||
|
MESSAGE(STATUS "use pre defined download url")
|
||||||
|
SET(PSLIB_VER "0.1.0" CACHE STRING "" FORCE)
|
||||||
|
SET(PSLIB_NAME "pslib" CACHE STRING "" FORCE)
|
||||||
|
SET(PSLIB_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${PSLIB_VER}/${PSLIB_NAME}.tar.gz" CACHE STRING "" FORCE)
|
||||||
|
ENDIF()
|
||||||
|
MESSAGE(STATUS "PSLIB_NAME: ${PSLIB_NAME}, PSLIB_URL: ${PSLIB_URL}")
|
||||||
|
SET(PSLIB_SOURCE_DIR "${THIRD_PARTY_PATH}/pslib")
|
||||||
|
SET(PSLIB_DOWNLOAD_DIR "${PSLIB_SOURCE_DIR}/src/${PSLIB_PROJECT}")
|
||||||
|
SET(PSLIB_DST_DIR "pslib")
|
||||||
|
SET(PSLIB_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
|
||||||
|
SET(PSLIB_INSTALL_DIR ${PSLIB_INSTALL_ROOT}/${PSLIB_DST_DIR})
|
||||||
|
SET(PSLIB_ROOT ${PSLIB_INSTALL_DIR})
|
||||||
|
SET(PSLIB_INC_DIR ${PSLIB_ROOT}/include)
|
||||||
|
SET(PSLIB_LIB_DIR ${PSLIB_ROOT}/lib)
|
||||||
|
SET(PSLIB_LIB ${PSLIB_LIB_DIR}/libps.so)
|
||||||
|
SET(PSLIB_IOMP_LIB ${PSLIB_LIB_DIR}/libiomp5.so) #todo what is this
|
||||||
|
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${PSLIB_ROOT}/lib")
|
||||||
|
|
||||||
|
INCLUDE_DIRECTORIES(${PSLIB_INC_DIR})
|
||||||
|
|
||||||
|
FILE(WRITE ${PSLIB_DOWNLOAD_DIR}/CMakeLists.txt
|
||||||
|
"PROJECT(PSLIB)\n"
|
||||||
|
"cmake_minimum_required(VERSION 3.0)\n"
|
||||||
|
"install(DIRECTORY ${PSLIB_NAME}/include ${PSLIB_NAME}/lib \n"
|
||||||
|
" DESTINATION ${PSLIB_DST_DIR})\n")
|
||||||
|
|
||||||
|
ExternalProject_Add(
|
||||||
|
${PSLIB_PROJECT}
|
||||||
|
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||||
|
PREFIX ${PSLIB_SOURCE_DIR}
|
||||||
|
DOWNLOAD_DIR ${PSLIB_DOWNLOAD_DIR}
|
||||||
|
DOWNLOAD_COMMAND wget --no-check-certificate ${PSLIB_URL} -c -q -O ${PSLIB_NAME}.tar.gz
|
||||||
|
&& tar zxvf ${PSLIB_NAME}.tar.gz
|
||||||
|
DOWNLOAD_NO_PROGRESS 1
|
||||||
|
UPDATE_COMMAND ""
|
||||||
|
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PSLIB_INSTALL_ROOT}
|
||||||
|
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PSLIB_INSTALL_ROOT}
|
||||||
|
)
|
||||||
|
|
||||||
|
ADD_LIBRARY(pslib SHARED IMPORTED GLOBAL)
|
||||||
|
SET_PROPERTY(TARGET pslib PROPERTY IMPORTED_LOCATION ${PSLIB_LIB})
|
||||||
|
ADD_DEPENDENCIES(pslib ${PSLIB_PROJECT})
|
||||||
|
LIST(APPEND external_project_dependencies pslib)
|
||||||
|
|
||||||
|
IF(WITH_C_API)
|
||||||
|
INSTALL(FILES ${PSLIB_LIB} ${PSLIB_IOMP_LIB} DESTINATION lib)
|
||||||
|
ENDIF()
|
||||||
@ -0,0 +1,77 @@
|
|||||||
|
# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
IF(NOT ${WITH_PSLIB_BRPC})
|
||||||
|
return()
|
||||||
|
ENDIF(NOT ${WITH_PSLIB_BRPC})
|
||||||
|
|
||||||
|
IF(WIN32 OR APPLE)
|
||||||
|
MESSAGE(WARNING
|
||||||
|
"Windows or Mac is not supported with PSLIB_BRPC in Paddle yet."
|
||||||
|
"Force WITH_PSLIB_BRPC=OFF")
|
||||||
|
SET(WITH_PSLIB_BRPC OFF CACHE STRING "Disable PSLIB_BRPC package in Windows and MacOS" FORCE)
|
||||||
|
return()
|
||||||
|
ENDIF()
|
||||||
|
|
||||||
|
INCLUDE(ExternalProject)
|
||||||
|
|
||||||
|
SET(PSLIB_BRPC_PROJECT "extern_pslib_brpc")
|
||||||
|
IF((NOT DEFINED PSLIB_BRPC_NAME) OR (NOT DEFINED PSLIB_BRPC_URL))
|
||||||
|
MESSAGE(STATUS "use pre defined download url")
|
||||||
|
SET(PSLIB_BRPC_VER "0.1.0" CACHE STRING "" FORCE)
|
||||||
|
SET(PSLIB_BRPC_NAME "pslib_brpc" CACHE STRING "" FORCE)
|
||||||
|
SET(PSLIB_BRPC_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${PSLIB_BRPC_VER}/${PSLIB_BRPC_NAME}.tar.gz" CACHE STRING "" FORCE)
|
||||||
|
ENDIF()
|
||||||
|
MESSAGE(STATUS "PSLIB_BRPC_NAME: ${PSLIB_BRPC_NAME}, PSLIB_BRPC_URL: ${PSLIB_BRPC_URL}")
|
||||||
|
SET(PSLIB_BRPC_SOURCE_DIR "${THIRD_PARTY_PATH}/pslib_brpc")
|
||||||
|
SET(PSLIB_BRPC_DOWNLOAD_DIR "${PSLIB_BRPC_SOURCE_DIR}/src/${PSLIB_BRPC_PROJECT}")
|
||||||
|
SET(PSLIB_BRPC_DST_DIR "pslib_brpc")
|
||||||
|
SET(PSLIB_BRPC_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
|
||||||
|
SET(PSLIB_BRPC_INSTALL_DIR ${PSLIB_BRPC_INSTALL_ROOT}/${PSLIB_BRPC_DST_DIR})
|
||||||
|
SET(PSLIB_BRPC_ROOT ${PSLIB_BRPC_INSTALL_DIR})
|
||||||
|
SET(PSLIB_BRPC_INC_DIR ${PSLIB_BRPC_ROOT}/include)
|
||||||
|
SET(PSLIB_BRPC_LIB_DIR ${PSLIB_BRPC_ROOT}/lib)
|
||||||
|
SET(PSLIB_BRPC_LIB ${PSLIB_BRPC_LIB_DIR}/libbrpc.a)
|
||||||
|
SET(PSLIB_BRPC_IOMP_LIB ${PSLIB_BRPC_LIB_DIR}/libiomp5.so) #todo what is this
|
||||||
|
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${PSLIB_BRPC_ROOT}/lib")
|
||||||
|
|
||||||
|
INCLUDE_DIRECTORIES(${PSLIB_BRPC_INC_DIR})
|
||||||
|
|
||||||
|
FILE(WRITE ${PSLIB_BRPC_DOWNLOAD_DIR}/CMakeLists.txt
|
||||||
|
"PROJECT(PSLIB_BRPC)\n"
|
||||||
|
"cmake_minimum_required(VERSION 3.0)\n"
|
||||||
|
"install(DIRECTORY ${PSLIB_BRPC_NAME}/include ${PSLIB_BRPC_NAME}/lib \n"
|
||||||
|
" DESTINATION ${PSLIB_BRPC_DST_DIR})\n")
|
||||||
|
|
||||||
|
ExternalProject_Add(
|
||||||
|
${PSLIB_BRPC_PROJECT}
|
||||||
|
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||||
|
PREFIX ${PSLIB_BRPC_SOURCE_DIR}
|
||||||
|
DOWNLOAD_DIR ${PSLIB_BRPC_DOWNLOAD_DIR}
|
||||||
|
DOWNLOAD_COMMAND wget --no-check-certificate ${PSLIB_BRPC_URL} -c -q -O ${PSLIB_BRPC_NAME}.tar.gz
|
||||||
|
&& tar zxvf ${PSLIB_BRPC_NAME}.tar.gz
|
||||||
|
DOWNLOAD_NO_PROGRESS 1
|
||||||
|
UPDATE_COMMAND ""
|
||||||
|
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PSLIB_BRPC_INSTALL_ROOT}
|
||||||
|
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PSLIB_BRPC_INSTALL_ROOT}
|
||||||
|
)
|
||||||
|
|
||||||
|
ADD_LIBRARY(pslib_brpc SHARED IMPORTED GLOBAL)
|
||||||
|
SET_PROPERTY(TARGET pslib_brpc PROPERTY IMPORTED_LOCATION ${PSLIB_BRPC_LIB})
|
||||||
|
ADD_DEPENDENCIES(pslib_brpc ${PSLIB_BRPC_PROJECT})
|
||||||
|
LIST(APPEND external_project_dependencies pslib_brpc)
|
||||||
|
|
||||||
|
IF(WITH_C_API)
|
||||||
|
INSTALL(FILES ${PSLIB_BRPC_LIB} ${PSLIB_BRPC_IOMP_LIB} DESTINATION lib)
|
||||||
|
ENDIF()
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,12 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
@ -0,0 +1,105 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
|
||||||
|
from .node import DownpourServer
|
||||||
|
from .node import DownpourWorker
|
||||||
|
from ..backward import append_backward
|
||||||
|
import ps_pb2 as pslib
|
||||||
|
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table
|
||||||
|
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_inputs
|
||||||
|
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_outputs
|
||||||
|
from google.protobuf import text_format
|
||||||
|
|
||||||
|
|
||||||
|
class DownpourSGD(object):
|
||||||
|
"""
|
||||||
|
Distributed optimizer of downpour stochastic gradient descent
|
||||||
|
Standard implementation of Google's Downpour SGD
|
||||||
|
in Large Scale Distributed Deep Networks
|
||||||
|
|
||||||
|
Args:
|
||||||
|
learning_rate (float): the learning rate used to update parameters. \
|
||||||
|
Can be a float value
|
||||||
|
Examples:
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
downpour_sgd = fluid.distributed.DownpourSGD(learning_rate=0.2)
|
||||||
|
downpour_sgd.minimize(cost)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, learning_rate=0.001, window=1):
|
||||||
|
# todo(guru4elephant): add more optimizers here as argument
|
||||||
|
# todo(guru4elephant): make learning_rate as a variable
|
||||||
|
self.learning_rate_ = learning_rate
|
||||||
|
self.window_ = window
|
||||||
|
self.type = "downpour"
|
||||||
|
|
||||||
|
def minimize(self,
|
||||||
|
loss,
|
||||||
|
startup_program=None,
|
||||||
|
parameter_list=None,
|
||||||
|
no_grad_set=None):
|
||||||
|
"""
|
||||||
|
DownpounSGD is a distributed optimizer so
|
||||||
|
that user can call minimize to generate backward
|
||||||
|
operators and optimization operators within minmize function
|
||||||
|
Args:
|
||||||
|
loss(Variable): loss variable defined by user
|
||||||
|
startup_program(Program): startup program that defined by user
|
||||||
|
parameter_list(str list): parameter names defined by users
|
||||||
|
no_grad_set(set): a set of variables that is defined by users
|
||||||
|
so that these variables do not need gradient computation
|
||||||
|
Returns:
|
||||||
|
[ps_param, worker_skipped_ops]
|
||||||
|
ps_param: parameter server protobuf desc
|
||||||
|
worker_skipped_ops: operator names that need
|
||||||
|
to be skipped during execution
|
||||||
|
"""
|
||||||
|
params_grads = sorted(
|
||||||
|
append_backward(loss, parameter_list, no_grad_set),
|
||||||
|
key=lambda x: x[0].name)
|
||||||
|
table_name = find_distributed_lookup_table(loss.block.program)
|
||||||
|
prefetch_slots = find_distributed_lookup_table_inputs(
|
||||||
|
loss.block.program, table_name)
|
||||||
|
prefetch_slots_emb = find_distributed_lookup_table_outputs(
|
||||||
|
loss.block.program, table_name)
|
||||||
|
server = DownpourServer()
|
||||||
|
# window is communication strategy
|
||||||
|
worker = DownpourWorker(self.window_)
|
||||||
|
# Todo(guru4elephant): support multiple tables definitions
|
||||||
|
# currently support one big sparse table
|
||||||
|
sparse_table_index = 0
|
||||||
|
# currently merge all dense parameters into one dense table
|
||||||
|
dense_table_index = 1
|
||||||
|
params = []
|
||||||
|
grads = []
|
||||||
|
for i in params_grads:
|
||||||
|
params.append(i[0])
|
||||||
|
for i in params_grads:
|
||||||
|
grads.append(i[1])
|
||||||
|
server.add_sparse_table(sparse_table_index, self.learning_rate_,
|
||||||
|
prefetch_slots, prefetch_slots_emb)
|
||||||
|
server.add_dense_table(dense_table_index, self.learning_rate_, params,
|
||||||
|
grads)
|
||||||
|
worker.add_sparse_table(sparse_table_index, self.learning_rate_,
|
||||||
|
prefetch_slots, prefetch_slots_emb)
|
||||||
|
worker.add_dense_table(dense_table_index, self.learning_rate_, params,
|
||||||
|
grads)
|
||||||
|
ps_param = pslib.PSParameter()
|
||||||
|
ps_param.server_param.CopyFrom(server.get_desc())
|
||||||
|
ps_param.trainer_param.CopyFrom(worker.get_desc())
|
||||||
|
# Todo(guru4elephant): figure out how to support more sparse parameters
|
||||||
|
# currently only support lookup_table
|
||||||
|
worker_skipped_ops = ["lookup_table", "lookup_table_grad"]
|
||||||
|
ps_param.trainer_param.skip_op.extend(worker_skipped_ops)
|
||||||
|
return [ps_param, worker_skipped_ops]
|
||||||
@ -0,0 +1,85 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
class FileSystem(object):
|
||||||
|
"""
|
||||||
|
A file system that support async_executor hadoop client desc.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
fs_type (string): fs_type, for example is "afs"
|
||||||
|
user (string): hadoop param
|
||||||
|
passwd (string): hadoop param
|
||||||
|
hadoop bin (string): hadoop param
|
||||||
|
Examples:
|
||||||
|
fs = FileSystm()
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
fs_type="afs",
|
||||||
|
uri="afs://xx",
|
||||||
|
user=None,
|
||||||
|
passwd=None,
|
||||||
|
hadoop_bin=""):
|
||||||
|
assert user != None
|
||||||
|
assert passwd != None
|
||||||
|
assert hadoop_bin != None
|
||||||
|
import ps_pb2 as pslib
|
||||||
|
self.fs_client = pslib.FsClientParameter()
|
||||||
|
self.fs_client.uri = uri
|
||||||
|
self.fs_client.user = user
|
||||||
|
self.fs_client.passwd = passwd
|
||||||
|
#self.fs_client.buffer_size = 0
|
||||||
|
self.fs_client.hadoop_bin = hadoop_bin
|
||||||
|
#self.fs_client.afs_conf = afs_conf if not afs_conf else ""
|
||||||
|
|
||||||
|
def get_desc(self):
|
||||||
|
"""
|
||||||
|
get hadoop desc.
|
||||||
|
"""
|
||||||
|
return self.fs_client
|
||||||
|
|
||||||
|
|
||||||
|
class MPIHelper(object):
|
||||||
|
"""
|
||||||
|
MPIHelper is a wrapper of mpi4py, support get_rank get_size etc.
|
||||||
|
Args:
|
||||||
|
No params
|
||||||
|
Examples:
|
||||||
|
mh = MPIHelper()
|
||||||
|
mh.get_ip()
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
from mpi4py import MPI
|
||||||
|
self.comm = MPI.COMM_WORLD
|
||||||
|
self.MPI = MPI
|
||||||
|
|
||||||
|
def get_rank(self):
|
||||||
|
return self.comm.Get_rank()
|
||||||
|
|
||||||
|
def get_size(self):
|
||||||
|
return self.comm.Get_size()
|
||||||
|
|
||||||
|
def get_ip(self):
|
||||||
|
import socket
|
||||||
|
local_ip = socket.gethostbyname(socket.gethostname())
|
||||||
|
return local_ip
|
||||||
|
|
||||||
|
def get_hostname(self):
|
||||||
|
import socket
|
||||||
|
return socket.gethostname()
|
||||||
|
|
||||||
|
def finalize(self):
|
||||||
|
self.MPI.Finalize()
|
||||||
@ -0,0 +1,179 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
|
||||||
|
import ps_pb2 as pslib
|
||||||
|
|
||||||
|
|
||||||
|
class Server(object):
|
||||||
|
"""
|
||||||
|
A Server basic class.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Worker(object):
|
||||||
|
"""
|
||||||
|
A Worker basic class.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class DownpourServer(Server):
|
||||||
|
"""
|
||||||
|
DownpourServer class is used to generate server program_desc
|
||||||
|
Args:
|
||||||
|
server: it is pslib.ServerParameter()
|
||||||
|
Examples:
|
||||||
|
server = DownpourServer()
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.server_ = pslib.ServerParameter()
|
||||||
|
self.server_.downpour_server_param.service_param.start_server_port = 0
|
||||||
|
self.server_.downpour_server_param.service_param.server_class = "DownpourBrpcPsServer"
|
||||||
|
self.server_.downpour_server_param.service_param.client_class = "DownpourBrpcPsClient"
|
||||||
|
self.server_.downpour_server_param.service_param.service_class = "DownpourPsService"
|
||||||
|
self.server_.downpour_server_param.service_param.start_server_port = 0
|
||||||
|
self.server_.downpour_server_param.service_param.server_thread_num = 12
|
||||||
|
|
||||||
|
def add_sparse_table(self, table_id, learning_rate, slot_key_vars,
|
||||||
|
slot_value_var):
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
table_id(int): id of sparse params table
|
||||||
|
learning_rate(float): the learning rate used to update parameters. \
|
||||||
|
Can be a float value
|
||||||
|
slot_key_vars(string): slot key id
|
||||||
|
slot_value_var(string): slot key value after embedding
|
||||||
|
Returns:
|
||||||
|
return None
|
||||||
|
"""
|
||||||
|
table = self.server_.downpour_server_param.downpour_table_param.add()
|
||||||
|
table.table_id = table_id
|
||||||
|
table.table_class = "DownpourSparseTable"
|
||||||
|
table.type = pslib.PS_SPARSE_TABLE
|
||||||
|
table.accessor.accessor_class = "DownpourFeatureValueAccessor"
|
||||||
|
table.accessor.sparse_sgd_param.learning_rate = learning_rate
|
||||||
|
table.accessor.sparse_sgd_param.initial_g2sum = 3
|
||||||
|
table.accessor.sparse_sgd_param.initial_range = 1e-4
|
||||||
|
table.accessor.sparse_sgd_param.weight_bounds.extend([-10, 10])
|
||||||
|
|
||||||
|
table.accessor.embedx_dim = 8
|
||||||
|
table.accessor.embedx_threshold = 5
|
||||||
|
table.accessor.fea_dim = 11
|
||||||
|
table.accessor.downpour_accessor_param.nonclk_coeff = 0.1
|
||||||
|
table.accessor.downpour_accessor_param.click_coeff = 2
|
||||||
|
table.accessor.downpour_accessor_param.base_threshold = 0.2
|
||||||
|
table.accessor.downpour_accessor_param.delta_threshold = 0.15
|
||||||
|
table.accessor.downpour_accessor_param.delta_keep_days = 31
|
||||||
|
table.accessor.downpour_accessor_param.show_click_decay_rate = 0.999
|
||||||
|
table.accessor.downpour_accessor_param.delete_threshold = 0.8
|
||||||
|
|
||||||
|
def add_dense_table(self, table_id, learning_rate, param_var, grad_var):
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
table_id(int): id of sparse params table
|
||||||
|
learning_rate(float): the learning rate used to update parameters. \
|
||||||
|
Can be a float value
|
||||||
|
param_var(list): all dense param. it is a list.
|
||||||
|
grad_var(list): all dense grad parm it is a list.
|
||||||
|
Returns:
|
||||||
|
return None
|
||||||
|
"""
|
||||||
|
table = self.server_.downpour_server_param.downpour_table_param.add()
|
||||||
|
table.table_id = table_id
|
||||||
|
table.table_class = "DownpourDenseTable"
|
||||||
|
table.type = pslib.PS_DENSE_TABLE
|
||||||
|
table.accessor.accessor_class = "DownpourDenseValueAccessor"
|
||||||
|
table.accessor.dense_sgd_param.name = "adam"
|
||||||
|
table.accessor.dense_sgd_param.adam.learning_rate = learning_rate
|
||||||
|
table.accessor.dense_sgd_param.adam.avg_decay_rate = 0.999993
|
||||||
|
table.accessor.dense_sgd_param.adam.ada_decay_rate = 0.9999
|
||||||
|
table.accessor.dense_sgd_param.adam.ada_epsilon = 1e-8
|
||||||
|
table.accessor.dense_sgd_param.adam.mom_decay_rate = 0.99
|
||||||
|
table.accessor.dense_sgd_param.naive.learning_rate = 0.0002
|
||||||
|
fea_dim = 0
|
||||||
|
for param in filter(lambda x: x.name.find("embedding") == -1,
|
||||||
|
param_var):
|
||||||
|
fea_dim += reduce(lambda x, y: x * y, param.shape, 1)
|
||||||
|
table.accessor.fea_dim = fea_dim
|
||||||
|
|
||||||
|
def get_desc(self):
|
||||||
|
"""
|
||||||
|
Return downpour server program_desc
|
||||||
|
"""
|
||||||
|
return self.server_
|
||||||
|
|
||||||
|
|
||||||
|
class DownpourWorker(Worker):
|
||||||
|
"""
|
||||||
|
DownpourWorker class is used to generate worker program_desc
|
||||||
|
Args:
|
||||||
|
window (int): push params frequency
|
||||||
|
worker: it is pslib.DownpourTrainerParameter
|
||||||
|
Examples:
|
||||||
|
worker = DownpourWorker(1)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, window):
|
||||||
|
self.window = window
|
||||||
|
self.worker_ = pslib.DownpourTrainerParameter()
|
||||||
|
|
||||||
|
def add_sparse_table(self, table_id, learning_rate, slot_key_vars,
|
||||||
|
slot_value_vars):
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
table_id(int): id of sparse params table
|
||||||
|
learning_rate(float): the learning rate used to update parameters. \
|
||||||
|
Can be a float value
|
||||||
|
slot_key_vars(string): slot key id
|
||||||
|
slot_value_var(string): slot key value after embedding
|
||||||
|
Returns:
|
||||||
|
return None
|
||||||
|
"""
|
||||||
|
table = self.worker_.sparse_table.add()
|
||||||
|
table.table_id = table_id
|
||||||
|
table.slot_key.extend([var.name for var in slot_key_vars])
|
||||||
|
table.slot_value.extend([var.name for var in slot_value_vars])
|
||||||
|
table.slot_gradient.extend(
|
||||||
|
[var.name + "@GRAD" for var in slot_value_vars])
|
||||||
|
|
||||||
|
def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars):
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
table_id(int): id of sparse params table
|
||||||
|
learning_rate(float): the learning rate used to update parameters. \
|
||||||
|
Can be a float value
|
||||||
|
param_var(list): all dense param. it is a list.
|
||||||
|
grad_var(list): all dense grad parm it is a list.
|
||||||
|
Returns:
|
||||||
|
return None
|
||||||
|
"""
|
||||||
|
table = self.worker_.dense_table.add()
|
||||||
|
table.table_id = table_id
|
||||||
|
table.dense_variable_name.extend(
|
||||||
|
filter(lambda x: x.find("embedding") == -1,
|
||||||
|
[p.name for p in param_vars]))
|
||||||
|
table.dense_gradient_variable_name.extend(
|
||||||
|
filter(lambda x: x.find("embedding") == -1,
|
||||||
|
[g.name for g in grad_vars]))
|
||||||
|
|
||||||
|
def get_desc(self):
|
||||||
|
"""
|
||||||
|
Return downpour worker program_desc
|
||||||
|
"""
|
||||||
|
return self.worker_
|
||||||
@ -0,0 +1,148 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
|
||||||
|
from .helper import MPIHelper
|
||||||
|
|
||||||
|
|
||||||
|
class PaddlePSInstance(object):
|
||||||
|
"""
|
||||||
|
PaddlePSInstance class is used to generate A instance of server or worker
|
||||||
|
Args:
|
||||||
|
server_worker_mode: is a value 0 or 1, default is 1
|
||||||
|
proc_per_node: process per node, default is 2
|
||||||
|
Examples:
|
||||||
|
instance = PaddlePSInstance(1, 2)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, server_worker_mode, proc_per_node):
|
||||||
|
self.dh = MPIHelper()
|
||||||
|
self._rankid = self.dh.get_rank()
|
||||||
|
self._server_worker_mode = server_worker_mode
|
||||||
|
self._proc_per_node = proc_per_node
|
||||||
|
self._nodes = self.dh.get_size()
|
||||||
|
|
||||||
|
self._ip = 0
|
||||||
|
self._worker_num = self._nodes * self._proc_per_node / 2
|
||||||
|
self._server_num = self._nodes * self._proc_per_node / 2
|
||||||
|
self._total_server_worker = self._worker_num + self._server_num
|
||||||
|
self._node_type = None #IDLE=-1, WORKER=1, SERVER=0
|
||||||
|
self._set_nodetype()
|
||||||
|
self._comm = None
|
||||||
|
self._split_comm()
|
||||||
|
|
||||||
|
def _set_nodetype(self):
|
||||||
|
if self._server_worker_mode == 0:
|
||||||
|
if self._rankid < self._server_num:
|
||||||
|
self._node_type = 1
|
||||||
|
elif self._rankid < self._total_server_worker:
|
||||||
|
self._node_type = 0
|
||||||
|
else:
|
||||||
|
self._node_type = -1
|
||||||
|
elif self._server_worker_mode == 1:
|
||||||
|
if self._rankid < self._total_server_worker:
|
||||||
|
if 0 == self._rankid % self._proc_per_node % 2:
|
||||||
|
self._node_type = 0
|
||||||
|
else:
|
||||||
|
self._node_type = 1
|
||||||
|
else:
|
||||||
|
self._node_type = -1
|
||||||
|
else:
|
||||||
|
self._node_type = -1
|
||||||
|
|
||||||
|
def _split_comm(self):
|
||||||
|
if self.is_server():
|
||||||
|
self._comm = self.dh.comm.Split(self._node_type)
|
||||||
|
elif self.is_worker():
|
||||||
|
self._comm = self.dh.comm.Split(self._node_type)
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_worker_index(self):
|
||||||
|
"""
|
||||||
|
Return worker index
|
||||||
|
"""
|
||||||
|
if self._server_worker_mode == 0:
|
||||||
|
return self._rankid == self.server_num
|
||||||
|
else:
|
||||||
|
return self._rankid / self._proc_per_node
|
||||||
|
|
||||||
|
def get_server_index(self):
|
||||||
|
"""
|
||||||
|
Return server index
|
||||||
|
"""
|
||||||
|
if self._server_worker_mode == 0:
|
||||||
|
return self.rank_id
|
||||||
|
else:
|
||||||
|
return self.rank_id / self._proc_per_node
|
||||||
|
|
||||||
|
def is_worker(self):
|
||||||
|
"""
|
||||||
|
Return instance is worker or not
|
||||||
|
"""
|
||||||
|
return self._node_type == 1
|
||||||
|
|
||||||
|
def is_server(self):
|
||||||
|
"""
|
||||||
|
Return instance is server or not
|
||||||
|
"""
|
||||||
|
return self._node_type == 0
|
||||||
|
|
||||||
|
def is_first_worker(self):
|
||||||
|
"""
|
||||||
|
Return instance is first worker or not
|
||||||
|
"""
|
||||||
|
return self.is_worker() and 0 == self.get_worker_index()
|
||||||
|
|
||||||
|
def set_ip(self, ip):
|
||||||
|
"""
|
||||||
|
set server ip
|
||||||
|
"""
|
||||||
|
self._ip = ip
|
||||||
|
|
||||||
|
def gather_ips(self):
|
||||||
|
"""
|
||||||
|
Return all servers and workers ip throught mpi allgather
|
||||||
|
"""
|
||||||
|
self._ips = self.dh.comm.allgather(self._ip)
|
||||||
|
return self._ips
|
||||||
|
|
||||||
|
def get_node_cnt(self):
|
||||||
|
"""
|
||||||
|
Return node cnt
|
||||||
|
"""
|
||||||
|
return self._nodes
|
||||||
|
|
||||||
|
def barrier_all(self):
|
||||||
|
"""
|
||||||
|
barrier workers and servers
|
||||||
|
"""
|
||||||
|
self.dh.comm.barrier()
|
||||||
|
|
||||||
|
def barrier_worker(self):
|
||||||
|
"""
|
||||||
|
barrier workers
|
||||||
|
"""
|
||||||
|
if self.is_worker():
|
||||||
|
self._comm.barrier()
|
||||||
|
pass
|
||||||
|
|
||||||
|
def finalize(self):
|
||||||
|
"""
|
||||||
|
MPI finalize
|
||||||
|
"""
|
||||||
|
self.dh.finalize()
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
instance = PaddlePSInstance(1, 1, 2, 50)
|
||||||
|
instance.barrier_all()
|
||||||
File diff suppressed because one or more lines are too long
Loading…
Reference in new issue