add pybind for fleet

paddle/fluid/framework/async_executor.cc

@@ -59,10 +59,6 @@ void AsyncExecutor::GatherServers(const std::vector<uint64_t>& host_sign_list,
   fleet_ptr_->GatherServers(host_sign_list, node_num);
 }
 
-void AsyncExecutor::InitModel() {}
-
-void AsyncExecutor::SaveModel(const std::string& path) {}
-
 void AsyncExecutor::RunFromFile(const ProgramDesc& main_program,
                                 const std::string& data_feed_desc_str,
                                 const std::vector<std::string>& filelist,

paddle/fluid/framework/data_feed.h

@@ -243,9 +243,6 @@ class MultiSlotDataFeed
   virtual bool ParseOneInstance(std::vector<MultiSlotType>* instance);
   virtual bool ParseOneInstanceFromPipe(std::vector<MultiSlotType>* instance);
   virtual void PutToFeedVec(const std::vector<MultiSlotType>& ins_vec);
-
- private:
-  BatchGenerator batch_gen_;
 };
 }  // namespace framework
 }  // namespace paddle

paddle/fluid/framework/fleet/fleet_wrapper.h

@@ -49,6 +49,7 @@ namespace framework {
 class FleetWrapper {
  public:
   virtual ~FleetWrapper() {}
+  FleetWrapper() {}
   // Pull sparse variables from server in Sync mode
   // Param<in>: scope, table_id, var_names, fea_keys
   // Param<out>: fea_values
@@ -123,9 +124,6 @@ class FleetWrapper {
  private:
   static std::shared_ptr<FleetWrapper> s_instance_;
 
- private:
-  FleetWrapper() {}
-
  protected:
   static bool is_initialized_;
   DISABLE_COPY_AND_ASSIGN(FleetWrapper);

paddle/fluid/pybind/CMakeLists.txt

@@ -1,11 +1,15 @@
-set(PYBIND_DEPS pybind python proto_desc memory executor async_executor prune
+set(PYBIND_DEPS pybind python proto_desc memory executor async_executor fleet_wrapper prune
   feed_fetch_method pass_builder parallel_executor profiler layer scope_pool
   tracer analysis_predictor imperative_profiler)
 
 if(WITH_PYTHON)
   list(APPEND PYBIND_DEPS py_func_op)
 endif()
+<<<<<<< HEAD
 set(PYBIND_SRCS pybind.cc exception.cc protobuf.cc const_value.cc recordio.cc reader_py.cc async_executor_py.cc imperative.cc ir.cc inference_api.cc)
+=======
+set(PYBIND_SRCS pybind.cc exception.cc protobuf.cc const_value.cc recordio.cc async_executor_py.cc fleet_wrapper_py.cc imperative.cc ir.cc inference_api.cc)
+>>>>>>> add pybind for fleet
 
 if(WITH_PYTHON)
   if(WITH_AMD_GPU)

paddle/fluid/pybind/fleet_wrapper_py.cc

@@ -0,0 +1,52 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include <fcntl.h>
+
+#ifdef _POSIX_C_SOURCE
+#undef _POSIX_C_SOURCE
+#endif
+
+#ifdef _XOPEN_SOURCE
+#undef _XOPEN_SOURCE
+#endif
+
+#include <string>
+#include <vector>
+
+#include "google/protobuf/io/zero_copy_stream_impl.h"
+#include "google/protobuf/text_format.h"
+#include "paddle/fluid/framework/async_executor.h"
+#include "paddle/fluid/framework/data_feed.h"
+#include "paddle/fluid/framework/data_feed.pb.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/inference/io.h"
+#include "paddle/fluid/platform/place.h"
+#include "paddle/fluid/platform/variant.h"
+#include "paddle/fluid/pybind/fleet_wrapper_py.h"
+
+namespace py = pybind11;
+namespace pd = paddle::framework;
+
+namespace paddle {
+namespace pybind {
+void BindFleetWrapper(py::module* m) {
+  py::class_<framework::FleetWrapper>(*m, "Fleet")
+      .def(py::init())
+      .def("init_server", &framework::FleetWrapper::InitServer)
+      .def("init_worker", &framework::FleetWrapper::InitWorker)
+      .def("stop_server", &framework::FleetWrapper::StopServer)
+      .def("gather_servers", &framework::FleetWrapper::GatherServers);
+}  // end FleetWrapper
+}  // end namespace pybind
+}  // end namespace paddle

paddle/fluid/pybind/fleet_wrapper_py.h

@@ -0,0 +1,28 @@
+//   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "pybind11/pybind11.h"
+#include "pybind11/stl.h"
+
+namespace py = pybind11;
+
+namespace paddle {
+namespace pybind {
+
+void BindFleetWrapper(py::module* m);
+
+}  // namespace pybind
+}  // namespace paddle

paddle/fluid/pybind/pybind.cc

@@ -51,6 +51,7 @@ limitations under the License. */
 #include "paddle/fluid/pybind/async_executor_py.h"
 #include "paddle/fluid/pybind/const_value.h"
 #include "paddle/fluid/pybind/exception.h"
+#include "paddle/fluid/pybind/fleet_wrapper_py.h"
 #include "paddle/fluid/pybind/imperative.h"
 #include "paddle/fluid/pybind/inference_api.h"
 #include "paddle/fluid/pybind/ir.h"
@@ -59,7 +60,6 @@ limitations under the License. */
 #include "paddle/fluid/pybind/reader_py.h"
 #include "paddle/fluid/pybind/recordio.h"
 #include "paddle/fluid/pybind/tensor_py.h"
-
 #include "paddle/fluid/string/to_string.h"
 
 #ifdef PADDLE_WITH_CUDA