add heter ps mode (#25682)
* add heter ps mode * code style test=develop * add with_pslib test=develop * unitest test=develop * code style test=develop * code style test=develop * code style test=develop * code style test=develop * code style test=develop * code style test=develop * code style test=develop * code style test=develop * test monitor test=develop * prepare trainer test=develop * code style test=developrevert-24895-update_cub
Thunderbrook
5 years ago
committed by
GitHub
parent
c8d0d1419b
commit
0cb60c700d
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,123 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <ctime>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#ifdef PADDLE_WITH_PSLIB
|
||||
#include "paddle/fluid/framework/heter_service.h"
|
||||
#include "paddle/fluid/framework/scope.h"
|
||||
#include "paddle/fluid/framework/tensor.h"
|
||||
#include "paddle/fluid/framework/variable_helper.h"
|
||||
#include "paddle/fluid/platform/macros.h" // for DISABLE_COPY_AND_ASSIGN
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
|
||||
class HeterCpuWorker;
|
||||
|
||||
typedef std::function<void(void*)> HeterRpcCallbackFunc;
|
||||
|
||||
class OnHeterRpcDone : public google::protobuf::Closure {
|
||||
public:
|
||||
OnHeterRpcDone(HeterRpcCallbackFunc func) : handler_(func) {}
|
||||
virtual ~OnHeterRpcDone() {}
|
||||
void Run() {
|
||||
std::unique_ptr<OnHeterRpcDone> self_guard(this);
|
||||
handler_(this);
|
||||
}
|
||||
|
||||
HeterRpcCallbackFunc handler_;
|
||||
HeterResponse response;
|
||||
brpc::Controller cntl;
|
||||
};
|
||||
|
||||
class HeterWrapper {
|
||||
public:
|
||||
virtual ~HeterWrapper() {
|
||||
server_.Stop(1000);
|
||||
server_.Join();
|
||||
}
|
||||
|
||||
HeterWrapper() {}
|
||||
|
||||
static void HeterRpcCallBack(HeterResponse* response, brpc::Controller* cntl,
|
||||
HeterCpuWorker* worker,
|
||||
std::shared_ptr<HeterTask> task);
|
||||
|
||||
void CreateClient2XpuConnection();
|
||||
|
||||
void RegisterServiceHandler(int cmd, HeterServiceHandler func);
|
||||
|
||||
void StartXpuService(const std::string& ip, uint32_t port);
|
||||
|
||||
void CallRemoteXpu(std::shared_ptr<HeterTask> task, HeterCpuWorker* worker,
|
||||
int mpi_rank, std::vector<std::string>& send_vars);
|
||||
|
||||
void CallRemoteXpuSync(std::shared_ptr<HeterTask> task,
|
||||
HeterCpuWorker* worker, int mpi_rank,
|
||||
std::vector<std::string>& send_vars);
|
||||
|
||||
void StopXpuService(int num);
|
||||
|
||||
void EndPass(Scope* scope, int num);
|
||||
|
||||
void SerializeToReq(const std::string& varname, Scope* scope,
|
||||
VariableMessage* req_var);
|
||||
|
||||
framework::proto::VarType::Type ToVarType(VariableMessage::Type type);
|
||||
|
||||
#ifdef PADDLE_WITH_CUDA
|
||||
void DeSerializeToTensor(Scope* scope, const VariableMessage& req_var,
|
||||
platform::Place place,
|
||||
cudaStream_t stream = nullptr);
|
||||
#else
|
||||
void DeSerializeToTensor(Scope* scope, const VariableMessage& req_var,
|
||||
platform::Place place);
|
||||
#endif
|
||||
// HeterWrapper singleton
|
||||
static std::shared_ptr<HeterWrapper> GetInstance() {
|
||||
if (NULL == s_instance_) {
|
||||
s_instance_.reset(new paddle::framework::HeterWrapper());
|
||||
}
|
||||
return s_instance_;
|
||||
}
|
||||
|
||||
std::vector<std::string>& GetXpuList() { return xpu_list_; }
|
||||
|
||||
void SetXpuList(const std::vector<std::string>& xpu_list);
|
||||
|
||||
private:
|
||||
static std::shared_ptr<HeterWrapper> s_instance_;
|
||||
|
||||
protected:
|
||||
std::vector<std::shared_ptr<brpc::Channel>> xpu_channels_;
|
||||
brpc::Server server_;
|
||||
HeterXpuService service_;
|
||||
static bool is_initialized_;
|
||||
DISABLE_COPY_AND_ASSIGN(HeterWrapper);
|
||||
std::vector<std::string> xpu_list_;
|
||||
};
|
||||
|
||||
} // end namespace framework
|
||||
} // end namespace paddle
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,69 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
syntax = "proto2";
|
||||
package paddle.framework;
|
||||
option cc_generic_services = true;
|
||||
|
||||
// It can be: LoDTensor、SelectedRows or NCCL_ID
|
||||
enum VarType {
|
||||
LOD_TENSOR = 0;
|
||||
SELECTED_ROWS = 1;
|
||||
NCCL_ID = 2;
|
||||
}
|
||||
|
||||
// VariableMessage is serialized paddle variable message.
|
||||
// NOTICE(gongwb):don't modify this proto if you are not
|
||||
// not familar with how we serialize in sendrecvop_utils.h
|
||||
// and deserilize it in variable_response.h.
|
||||
message VariableMessage {
|
||||
enum Type {
|
||||
// Pod Types
|
||||
BOOL = 0;
|
||||
INT16 = 1;
|
||||
INT32 = 2;
|
||||
INT64 = 3;
|
||||
FP16 = 4;
|
||||
FP32 = 5;
|
||||
FP64 = 6;
|
||||
}
|
||||
|
||||
message LodData { repeated int64 lod_data = 1; }
|
||||
optional string varname = 1;
|
||||
// TODO(Yancey1989): reference framework::proto::VarDesc::VarType
|
||||
optional VarType type = 2;
|
||||
// bool persistable is not needed for sending.
|
||||
// tensor info:
|
||||
optional Type data_type = 3;
|
||||
repeated int64 dims = 4;
|
||||
|
||||
// lod details:
|
||||
optional int64 lod_level = 5;
|
||||
repeated LodData lod = 6;
|
||||
// selected_rows height, aka. original dim0
|
||||
optional int64 slr_height = 7;
|
||||
// tensor data
|
||||
optional bytes data = 8;
|
||||
}
|
||||
message HeterRequest {
|
||||
required int32 cmd = 1;
|
||||
optional int32 cur_batch = 2;
|
||||
repeated VariableMessage vars = 3;
|
||||
};
|
||||
|
||||
message HeterResponse {
|
||||
// optional VariableMessage vars = 1;
|
||||
repeated VariableMessage vars = 1;
|
||||
};
|
||||
|
||||
service HeterService { rpc service(HeterRequest) returns (HeterResponse); };
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,50 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
#include <fcntl.h>
|
||||
|
||||
#ifdef _POSIX_C_SOURCE
|
||||
#undef _POSIX_C_SOURCE
|
||||
#endif
|
||||
|
||||
#ifdef _XOPEN_SOURCE
|
||||
#undef _XOPEN_SOURCE
|
||||
#endif
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "google/protobuf/io/zero_copy_stream_impl.h"
|
||||
#include "google/protobuf/text_format.h"
|
||||
#include "paddle/fluid/framework/fleet/heter_wrapper.h"
|
||||
#include "paddle/fluid/pybind/heter_wrapper_py.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace paddle {
|
||||
namespace pybind {
|
||||
#ifdef PADDLE_WITH_PSLIB
|
||||
void BindHeterWrapper(py::module* m) {
|
||||
py::class_<framework::HeterWrapper, std::shared_ptr<framework::HeterWrapper>>(
|
||||
*m, "Heter")
|
||||
.def(py::init([]() { return framework::HeterWrapper::GetInstance(); }))
|
||||
.def("create_client2xpu_connection",
|
||||
&framework::HeterWrapper::CreateClient2XpuConnection)
|
||||
.def("set_xpu_list", &framework::HeterWrapper::SetXpuList)
|
||||
.def("start_xpu_service", &framework::HeterWrapper::StartXpuService)
|
||||
.def("end_pass", &framework::HeterWrapper::EndPass)
|
||||
.def("stop_xpu_service", &framework::HeterWrapper::StopXpuService);
|
||||
} // end HeterWrapper
|
||||
#endif
|
||||
} // end namespace pybind
|
||||
} // end namespace paddle
|
||||
@ -0,0 +1,29 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "pybind11/pybind11.h"
|
||||
#include "pybind11/stl.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace paddle {
|
||||
namespace pybind {
|
||||
|
||||
#ifdef PADDLE_WITH_PSLIB
|
||||
void BindHeterWrapper(py::module* m);
|
||||
#endif
|
||||
} // namespace pybind
|
||||
} // namespace paddle
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue