lujun
6 years ago
commit
92c8ac8a74
@ -1,71 +0,0 @@
|
||||
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
IF(NOT ${WITH_WBAES})
|
||||
return()
|
||||
ENDIF(NOT ${WITH_WBAES})
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
SET(WBAES_DST_DIR "wbaes")
|
||||
SET(WBAES_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
|
||||
SET(WBAES_INSTALL_DIR ${WBAES_INSTALL_ROOT}/${WBAES_DST_DIR})
|
||||
SET(WBAES_ROOT ${WBAES_INSTALL_DIR})
|
||||
SET(WBAES_INC_DIR ${WBAES_ROOT}/include)
|
||||
SET(WBAES_LIB_DIR ${WBAES_ROOT}/lib)
|
||||
|
||||
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${WBAES_ROOT}/lib")
|
||||
SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
|
||||
|
||||
IF(APPLE)
|
||||
SET(WBAES_TAG "v1.0.0" CACHE STRING "" FORCE)
|
||||
SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.mac.${WBAES_TAG}.tgz" CACHE STRING "" FORCE)
|
||||
SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.dylib)
|
||||
SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.dylib)
|
||||
ELSEIF(WIN32)
|
||||
SET(WBAES_TAG "v1.0.0" CACHE STRING "" FORCE)
|
||||
SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.windows-x64.${WBAES_TAG}.tgz" CACHE STRING "" FORCE)
|
||||
SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.lib)
|
||||
SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.dll)
|
||||
ELSE()
|
||||
SET(WBAES_TAG "v1.0.2" CACHE STRING "" FORCE)
|
||||
SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.linux-x86_64.${WBAES_TAG}.tgz" CACHE STRING "" FORCE)
|
||||
SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.so)
|
||||
SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.so)
|
||||
ENDIF()
|
||||
|
||||
SET(WBAES_PROJECT "extern_wbaes")
|
||||
MESSAGE(STATUS "WBAES_URL: ${WBAES_URL}, WBAES_LIB: ${WBAES_LIB}")
|
||||
SET(WBAES_SOURCE_DIR "${THIRD_PARTY_PATH}/wbaes")
|
||||
SET(WBAES_DOWNLOAD_DIR "${WBAES_SOURCE_DIR}/src/${WBAES_PROJECT}")
|
||||
|
||||
ExternalProject_Add(
|
||||
${WBAES_PROJECT}
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
PREFIX ${WBAES_SOURCE_DIR}
|
||||
URL ${WBAES_URL}
|
||||
DOWNLOAD_DIR ${WBAES_DOWNLOAD_DIR}
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
${CMAKE_COMMAND} -E copy_directory ${WBAES_DOWNLOAD_DIR}/include ${WBAES_INC_DIR} &&
|
||||
${CMAKE_COMMAND} -E copy_directory ${WBAES_DOWNLOAD_DIR}/lib ${WBAES_LIB_DIR}
|
||||
)
|
||||
|
||||
INCLUDE_DIRECTORIES(${WBAES_INC_DIR})
|
||||
|
||||
ADD_LIBRARY(wbaes SHARED IMPORTED GLOBAL)
|
||||
SET_PROPERTY(TARGET wbaes PROPERTY IMPORTED_LOCATION ${WBAES_LIB})
|
||||
SET_PROPERTY(TARGET wbaes PROPERTY IMPORTED_NO_SONAME 1)
|
||||
ADD_DEPENDENCIES(wbaes ${WBAES_PROJECT})
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "paddle/fluid/framework/details/build_strategy.h"
|
||||
#include "paddle/fluid/framework/details/multi_devices_helper.h"
|
||||
#include "paddle/fluid/framework/ir/graph_helper.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
namespace details {
|
||||
|
||||
void SetFuseParameterGroupsSize(int group_size);
|
||||
int GetFuseParameterGroupsSize();
|
||||
|
||||
void SetFuseParameterMemorySize(uint64_t memory_size);
|
||||
uint64_t GetFuseParameterMemorySize();
|
||||
|
||||
class AllocContinuousSpaceForGradPass : public ir::Pass {
|
||||
protected:
|
||||
void ApplyImpl(ir::Graph *graph) const override;
|
||||
|
||||
template <typename AttrType>
|
||||
void ResetAttribute(const std::string &attr_name, ir::Graph *graph) const;
|
||||
|
||||
void SetGroupGradsAndParams(
|
||||
const std::unordered_map<std::string, ir::Node *> &var_nodes,
|
||||
const ParamsAndGrads ¶ms_grads,
|
||||
GroupGradsAndParams *group_grads_params) const;
|
||||
|
||||
void SetGroupAccordingToLayers(
|
||||
const std::unordered_map<std::string, ir::Node *> &var_nodes,
|
||||
const ParamsAndGrads ¶ms_grads,
|
||||
GroupGradsAndParams *group_grads_params) const;
|
||||
|
||||
void SetGroupAccordingToMemorySize(
|
||||
const std::unordered_map<std::string, ir::Node *> &var_nodes,
|
||||
GroupGradsAndParams *group_grads_params) const;
|
||||
|
||||
void SetGroupAccordingToGroupSize(
|
||||
const std::unordered_map<std::string, ir::Node *> &var_nodes,
|
||||
GroupGradsAndParams *group_grads_params) const;
|
||||
|
||||
private:
|
||||
bool IsSupportedVarType(const proto::VarType::Type &type) const;
|
||||
|
||||
void RecordParamsAndGrads(ir::Node *node, ParamsAndGrads *params_grads) const;
|
||||
|
||||
void InitFusedVarsAndAllocSpaceForVars(
|
||||
const std::vector<platform::Place> &places,
|
||||
const std::vector<Scope *> &local_scopes,
|
||||
const std::unordered_map<std::string, ir::Node *> &vars,
|
||||
const std::string &fused_var_name,
|
||||
const ParamsAndGrads ¶ms_grads) const;
|
||||
|
||||
void AppendAllocSpaceForVarsOp(const std::vector<std::string> ¶ms_name,
|
||||
const std::vector<std::string> &grads_name,
|
||||
const std::string &fused_var_name,
|
||||
BlockDesc *global_block) const;
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,37 @@
|
||||
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/framework/ir/expected_kernel_cache_pass.h"
|
||||
#include <memory>
|
||||
#include "paddle/fluid/framework/operator.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
namespace ir {
|
||||
|
||||
void ExpectedKernelCachePass::ApplyImpl(ir::Graph* graph) const {
|
||||
VLOG(3) << "Applies Expected Kernel Cache strategy.";
|
||||
for (const Node* n : graph->Nodes()) {
|
||||
if (n->IsOp()) {
|
||||
n->Op()->SetAttr(kEnableCacheExpectedKernel, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ir
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
|
||||
REGISTER_PASS(expected_kernel_cache_pass,
|
||||
paddle::framework::ir::ExpectedKernelCachePass);
|
||||
@ -0,0 +1,133 @@
|
||||
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/imperative/nccl_context.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace imperative {
|
||||
#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
|
||||
void NCCLParallelContext::RecvNCCLID(const std::string &ep,
|
||||
ncclUniqueId *nccl_id) {
|
||||
auto addr = paddle::string::Split(ep, ':');
|
||||
PADDLE_ENFORCE_EQ(addr.size(), 2UL,
|
||||
"The endpoint should contain host and port: %s", ep);
|
||||
std::string host = addr[0];
|
||||
int port = std::stoi(addr[1]);
|
||||
|
||||
int server_fd, new_socket;
|
||||
struct sockaddr_in address;
|
||||
int addrlen = sizeof(address);
|
||||
char buffer[1024] = {0};
|
||||
int opt = 0;
|
||||
// creating socket fd
|
||||
if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0)
|
||||
PADDLE_THROW("create server fd failed");
|
||||
if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)))
|
||||
PADDLE_THROW("set socket opt failed");
|
||||
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = INADDR_ANY;
|
||||
address.sin_port = htons(port);
|
||||
|
||||
if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0)
|
||||
PADDLE_THROW("binding failed on ep: %s", ep);
|
||||
VLOG(3) << "listening on: " << ep;
|
||||
if (listen(server_fd, 3) < 0) PADDLE_THROW("listen on server fd failed");
|
||||
|
||||
if ((new_socket =
|
||||
accept(server_fd, reinterpret_cast<struct sockaddr *>(&address),
|
||||
reinterpret_cast<socklen_t *>(&addrlen))) < 0)
|
||||
PADDLE_THROW("accept the new socket fd failed");
|
||||
|
||||
if (read(new_socket, buffer, 1024) < 0)
|
||||
PADDLE_THROW("reading the ncclUniqueId from socket failed");
|
||||
VLOG(3) << "recevived the ncclUniqueId";
|
||||
memcpy(nccl_id, buffer, NCCL_UNIQUE_ID_BYTES);
|
||||
|
||||
VLOG(3) << "closing the socket server: " << ep;
|
||||
close(server_fd);
|
||||
}
|
||||
|
||||
void NCCLParallelContext::SendNCCLID(const std::string &ep,
|
||||
ncclUniqueId *nccl_id) {
|
||||
auto addr = paddle::string::Split(ep, ':');
|
||||
PADDLE_ENFORCE_EQ(addr.size(), 2UL,
|
||||
"The endpoint should contain host and port: %s", ep);
|
||||
std::string host = addr[0];
|
||||
int port = std::stoi(addr[1]);
|
||||
// struct sockaddr_in address;
|
||||
int sock = 0;
|
||||
struct sockaddr_in serv_addr;
|
||||
char buffer[1024] = {0};
|
||||
|
||||
memcpy(buffer, nccl_id, NCCL_UNIQUE_ID_BYTES);
|
||||
if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0)
|
||||
PADDLE_THROW("create socket failed");
|
||||
|
||||
memset(&serv_addr, '0', sizeof(serv_addr));
|
||||
serv_addr.sin_family = AF_INET;
|
||||
serv_addr.sin_port = htons(port);
|
||||
|
||||
if (inet_pton(AF_INET, host.c_str(), &serv_addr.sin_addr) <= 0)
|
||||
PADDLE_THROW("invalied address: %s", ep);
|
||||
|
||||
while (true) {
|
||||
if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
|
||||
VLOG(0) << "worker: " << ep
|
||||
<< " is not ready, will retry after 3 seconds...";
|
||||
std::this_thread::sleep_for(std::chrono::seconds(3));
|
||||
continue;
|
||||
}
|
||||
VLOG(3) << "sending the ncclUniqueId to " << ep;
|
||||
send(sock, buffer, NCCL_UNIQUE_ID_BYTES, 0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void NCCLParallelContext::BcastNCCLId(ncclUniqueId *nccl_id, int root) {
|
||||
if (strategy_.local_rank_ == root) {
|
||||
for (auto ep : strategy_.trainer_endpoints_) {
|
||||
if (ep != strategy_.current_endpoint_) SendNCCLID(ep, nccl_id);
|
||||
}
|
||||
} else {
|
||||
RecvNCCLID(strategy_.current_endpoint_, nccl_id);
|
||||
}
|
||||
}
|
||||
|
||||
void NCCLParallelContext::Init() {
|
||||
ncclUniqueId nccl_id;
|
||||
ncclComm_t comm;
|
||||
if (strategy_.local_rank_ == 0) {
|
||||
// generate the unique ncclid on the root worker
|
||||
platform::dynload::ncclGetUniqueId(&nccl_id);
|
||||
BcastNCCLId(&nccl_id, 0);
|
||||
} else {
|
||||
BcastNCCLId(&nccl_id, 0);
|
||||
}
|
||||
int gpu_id = boost::get<platform::CUDAPlace>(place_).device;
|
||||
VLOG(0) << "init nccl context nranks: " << strategy_.nranks_
|
||||
<< " local rank: " << strategy_.local_rank_ << " gpu id: " << gpu_id;
|
||||
|
||||
PADDLE_ENFORCE(cudaSetDevice(gpu_id));
|
||||
PADDLE_ENFORCE(platform::dynload::ncclCommInitRank(
|
||||
&comm, strategy_.nranks_, nccl_id, strategy_.local_rank_));
|
||||
|
||||
platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance();
|
||||
auto *dev_ctx = static_cast<platform::CUDADeviceContext *>(pool.Get(place_));
|
||||
dev_ctx->set_nccl_comm(comm);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace imperative
|
||||
} // namespace paddle
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue