Put everything in paddle python wheel package enable pip install paddle (#3102)

* put everything in paddle wheel

* update

* update

* fix unitest

* with platform specs

paddle/CMakeLists.txt

@@ -22,7 +22,5 @@ if(WITH_C_API)
 endif()
 
 if(WITH_SWIG_PY)
-  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in
-          ${CMAKE_CURRENT_SOURCE_DIR}/setup.py)
   add_subdirectory(api)
 endif()

paddle/api/CMakeLists.txt

@@ -82,9 +82,7 @@ SWIG_LINK_LIBRARIES(swig_paddle
 add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so
     COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle
     COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/_swig_paddle.so ${PROJ_ROOT}/paddle/py_paddle
-    COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
+    COMMAND ${CMAKE_COMMAND} -E touch .timestamp
-    COMMAND ${CMAKE_COMMAND} -E touch dist/.timestamp
-    COMMAND rm -rf py_paddle.egg-info build
     WORKING_DIRECTORY ${PROJ_ROOT}/paddle
     DEPENDS _swig_paddle
 )
@@ -92,10 +90,6 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so
 # TODO(yuyang18) : make wheel name calculated by cmake
 add_custom_target(python_api_wheel ALL DEPENDS ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so)
 
-install(DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/dist/
-    DESTINATION opt/paddle/share/wheels
-)
-
 if(WITH_TESTING)
     IF(NOT PY_PIP_FOUND)
         SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip)
@@ -108,7 +102,7 @@ if(WITH_TESTING)
             BUILD_COMMAND       ""
             INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
             BUILD_IN_SOURCE     1
-            DEPENDS python setuptools python_api_wheel
+            #DEPENDS python setuptools python_api_wheel
         )
     ENDIF()
     add_subdirectory(test)

paddle/scripts/CMakeLists.txt

@@ -1,17 +1,15 @@
 configure_file(submit_local.sh.in
-    submit_local.sh
+    paddle
     @ONLY)
 
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/submit_local.sh DESTINATION bin
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/paddle DESTINATION bin
         PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
-            GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ
+            GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ)
-        RENAME paddle)
 
 configure_file(tools/usage_stat/usage.sh
-    usage.sh
+    paddle_usage
     @ONLY)
 
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/usage.sh DESTINATION opt/paddle/bin
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/paddle_usage DESTINATION opt/paddle/bin
         PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
-            GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ
+            GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ)
-        RENAME paddle_usage)

paddle/setup.py.in

@@ -1,32 +0,0 @@
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from setuptools import setup, Extension
-
-setup(name="py_paddle",
-      version="${PADDLE_VERSION}",
-      packages=['py_paddle'],
-      include_package_data=True,
-      package_data={'py_paddle':['*.py','_swig_paddle.so']},
-      install_requires = [
-        'nltk>=3.2.2',
-        # We use `numpy.flip` in `test_image.py`.
-        # `numpy.flip` is introduced in `1.12.0`
-        'numpy>=1.12.0',      # The numpy is required.
-        'protobuf==${PROTOBUF_VERSION}'    # The paddle protobuf version
-      ],
-      url='http://www.paddlepaddle.org/',
-      license='Apache 2.0',
-)

python/CMakeLists.txt

@@ -39,7 +39,7 @@ add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
     DEPENDS gen_proto_py copy_paddle_pybind framework_py_proto ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER})
 
 add_custom_target(paddle_python ALL DEPENDS
-    ${OUTPUT_DIR}/.timestamp)
+    ${OUTPUT_DIR}/.timestamp paddle_pserver_main paddle_trainer paddle_merge_model python_api_wheel)
 
 set(PADDLE_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/)
 

python/setup.py.in

@@ -1,4 +1,8 @@
-from setuptools import setup
+from setuptools import setup, Distribution
+
+class BinaryDistribution(Distribution):
+    def has_ext_modules(foo):
+        return True
 
 packages=['paddle',
           'paddle.proto',
@@ -11,7 +15,8 @@ packages=['paddle',
           'paddle.v2.master',
           'paddle.v2.plot',
           'paddle.v2.framework',
-          'paddle.v2.framework.proto']
+          'paddle.v2.framework.proto',
+          'py_paddle']
 
 setup_requires=["requests",
                 "numpy>=1.12",
@@ -21,23 +26,33 @@ setup_requires=["requests",
                 "rarfile",
                 "scipy>=0.19.0",
                 "Pillow",
-                "nltk"]
+                "nltk>=3.2.2"]
 
 if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']:
     setup_requires+=["opencv-python"]
 
-setup(name='paddle',
+setup(name='paddlepaddle',
       version='${PADDLE_VERSION}',
       description='Parallel Distributed Deep Learning',
       install_requires=setup_requires,
       packages=packages,
-      package_data={'paddle.v2.master': ['libpaddle_master.so'],
+      package_data={
-            'paddle.v2.framework': ['core.so']
+        'paddle.v2.master': ['libpaddle_master.so'],
+        'paddle.v2.framework': ['core.so'],
+        'py_paddle':['*.py','_swig_paddle.so']
       },
       package_dir={
           '': '${CMAKE_CURRENT_SOURCE_DIR}',
           # The paddle.v2.framework.proto will be generated while compiling.
           # So that package points to other directory.
-          'paddle.v2.framework.proto': '${PROJ_BINARY_ROOT}/paddle/framework'
+          'paddle.v2.framework.proto': '${PROJ_BINARY_ROOT}/paddle/framework',
+          'py_paddle': '${PROJ_ROOT}/paddle/py_paddle'
       },
+      scripts=['${PROJ_BINARY_ROOT}/paddle/scripts/paddle'],
+      distclass=BinaryDistribution,
+      data_files=[('/usr/local/opt/paddle/bin',
+                       ['${PROJ_BINARY_ROOT}/paddle/scripts/paddle_usage',
+                        '${PROJ_BINARY_ROOT}/paddle/trainer/paddle_trainer',
+                        '${PROJ_BINARY_ROOT}/paddle/trainer/paddle_merge_model',
+                        '${PROJ_BINARY_ROOT}/paddle/pserver/paddle_pserver_main'])]
 )