You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
245 lines
9.0 KiB
245 lines
9.0 KiB
from setuptools import setup, Distribution, Extension
|
|
import subprocess
|
|
import os
|
|
import re
|
|
import shutil
|
|
class BinaryDistribution(Distribution):
|
|
def has_ext_modules(foo):
|
|
return True
|
|
|
|
RC = 0
|
|
|
|
ext_name = '.dll' if os.name == 'nt' else '.so'
|
|
|
|
def git_commit():
|
|
try:
|
|
cmd = ['git', 'rev-parse', 'HEAD']
|
|
git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
|
|
cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
|
|
except:
|
|
git_commit = 'Unknown'
|
|
git_commit = git_commit.decode()
|
|
return str(git_commit)
|
|
|
|
def _get_version_detail(idx):
|
|
assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
|
|
so detail index must less than 3"
|
|
|
|
if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
|
|
version_details = '@PADDLE_VERSION@'.split('.')
|
|
|
|
if len(version_details) >= 3:
|
|
return version_details[idx]
|
|
|
|
return 0
|
|
|
|
def get_major():
|
|
return int(_get_version_detail(0))
|
|
|
|
def get_minor():
|
|
return int(_get_version_detail(1))
|
|
|
|
def get_patch():
|
|
return str(_get_version_detail(2))
|
|
|
|
def is_taged():
|
|
try:
|
|
cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
|
|
git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
|
|
git_tag = git_tag.decode()
|
|
except:
|
|
return False
|
|
|
|
if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def write_version_py(filename='paddle/version.py'):
|
|
cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
|
|
#
|
|
full_version = '%(major)d.%(minor)d.%(patch)s'
|
|
major = '%(major)d'
|
|
minor = '%(minor)d'
|
|
patch = '%(patch)s'
|
|
rc = '%(rc)d'
|
|
istaged = %(istaged)s
|
|
commit = '%(commit)s'
|
|
with_mkl = '%(with_mkl)s'
|
|
|
|
def show():
|
|
if istaged:
|
|
print('full_version:', full_version)
|
|
print('major:', major)
|
|
print('minor:', minor)
|
|
print('patch:', patch)
|
|
print('rc:', rc)
|
|
else:
|
|
print('commit:', commit)
|
|
|
|
def mkl():
|
|
return with_mkl
|
|
'''
|
|
commit = git_commit()
|
|
with open(filename, 'w') as f:
|
|
f.write(cnt % {
|
|
'major': get_major(),
|
|
'minor': get_minor(),
|
|
'patch': get_patch(),
|
|
'rc': RC,
|
|
'version': '${PADDLE_VERSION}',
|
|
'commit': commit,
|
|
'istaged': is_taged(),
|
|
'with_mkl': '@WITH_MKL@'})
|
|
|
|
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
|
|
|
|
|
|
packages=['paddle',
|
|
'paddle.libs',
|
|
'paddle.utils',
|
|
'paddle.dataset',
|
|
'paddle.reader',
|
|
'paddle.fluid',
|
|
'paddle.fluid.proto',
|
|
'paddle.fluid.proto.profiler',
|
|
'paddle.fluid.layers',
|
|
'paddle.fluid.contrib',
|
|
'paddle.fluid.contrib.decoder',
|
|
'paddle.fluid.contrib.quantize',
|
|
'paddle.fluid.transpiler',
|
|
'paddle.fluid.transpiler.details']
|
|
|
|
if '${WITH_FLUID_ONLY}'== 'OFF':
|
|
packages+=['paddle.proto',
|
|
'paddle.trainer',
|
|
'paddle.trainer_config_helpers',
|
|
'paddle.v2',
|
|
'paddle.v2.master',
|
|
'paddle.v2.plot',
|
|
'paddle.v2.reader',
|
|
'paddle.v2.dataset',
|
|
'py_paddle']
|
|
|
|
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
|
|
setup_requires = f.read().splitlines()
|
|
|
|
if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']:
|
|
setup_requires+=['opencv-python']
|
|
|
|
# the prefix is sys.prefix which should always be usr
|
|
paddle_bins = ''
|
|
if '${WITH_FLUID_ONLY}'== 'OFF':
|
|
paddle_bin_dir = 'opt/paddle/bin'
|
|
paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/legacy/trainer/paddle_trainer',
|
|
'${PADDLE_BINARY_DIR}/paddle/legacy/trainer/paddle_merge_model',
|
|
'${PADDLE_BINARY_DIR}/paddle/legacy/pserver/paddle_pserver_main',
|
|
'${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
|
|
|
|
package_data={'paddle.fluid': ['core' + (ext_name if os.name != 'nt' else '.pyd')]}
|
|
if os.name == 'nt':
|
|
package_data['paddle.fluid'] += ['openblas' + ext_name]
|
|
|
|
if '${WITH_FLUID_ONLY}'== 'OFF':
|
|
package_data['paddle.v2.master']=['libpaddle_master' + ext_name]
|
|
package_data['py_paddle']=['*.py','_swig_paddle' + ext_name]
|
|
|
|
package_dir={
|
|
'': '${PADDLE_BINARY_DIR}/python',
|
|
# The paddle.fluid.proto will be generated while compiling.
|
|
# So that package points to other directory.
|
|
'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
|
|
'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
|
|
'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
|
|
}
|
|
if '${WITH_FLUID_ONLY}'== 'OFF':
|
|
package_dir['py_paddle']='${PADDLE_BINARY_DIR}/python/py_paddle'
|
|
|
|
# put all thirdparty libraries in paddle.libs
|
|
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
|
|
if os.name != 'nt':
|
|
package_data['paddle.libs']= []
|
|
package_data['paddle.libs']=['libwarpctc' + ext_name]
|
|
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
|
|
if '${WITH_MKL}' == 'ON':
|
|
shutil.copy('${MKLML_LIB}', libs_path)
|
|
shutil.copy('${MKLML_IOMP_LIB}', libs_path)
|
|
package_data['paddle.libs']+=['libmklml_intel' + ext_name,'libiomp5' + ext_name]
|
|
if '${WITH_MKLDNN}' == 'ON':
|
|
if '${CMAKE_BUILD_TYPE}' == 'Release':
|
|
# only change rpath in Release mode.
|
|
# TODO(typhoonzero): use install_name_tool to patch mkl libs once
|
|
# we can support mkl on mac.
|
|
#
|
|
# change rpath of libmkldnn.so.0, add $ORIGIN/ to it.
|
|
# The reason is that all thirdparty libraries in the same directory,
|
|
# thus, libmkldnn.so.0 will find libmklml_intel.so and libiomp5.so.
|
|
command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
|
|
if os.system(command) != 0:
|
|
raise Exception("patch libmkldnn.so failed, command: %s" % command)
|
|
package_data['paddle.libs']+=['libmkldnn.so.0']
|
|
shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
|
|
if '${WITH_NGRAPH}' == 'ON':
|
|
# only change rpath in Release mode,
|
|
# since in Debug mode, nGraph lib may be too large to be changed?
|
|
if '${CMAKE_BUILD_TYPE}' == 'Release':
|
|
if os.name != 'nt':
|
|
if "@APPLE@" == "1":
|
|
command = "install_name_tool -id \"@loader_path/\" ${NGRAPH_SHARED_LIB}"
|
|
else:
|
|
command = "patchelf --set-rpath '$ORIGIN/' ${NGRAPH_SHARED_LIB}"
|
|
if os.system(command) != 0:
|
|
raise Exception("patch ${NGRAPH_SHARED_LIB_NAME} failed, command: %s" % command)
|
|
shutil.copy('${NGRAPH_SHARED_LIB}', libs_path)
|
|
shutil.copy('${NGRAPH_CPU_LIB}', libs_path)
|
|
shutil.copy('${NGRAPH_TBB_LIB}', libs_path)
|
|
package_data['paddle.libs']+=['${NGRAPH_SHARED_LIB_NAME}',
|
|
'${NGRAPH_CPU_LIB_NAME}',
|
|
'${NGRAPH_TBB_LIB_NAME}']
|
|
# remove unused paddle/libs/__init__.py
|
|
if os.path.isfile(libs_path+'/__init__.py'):
|
|
os.remove(libs_path+'/__init__.py')
|
|
package_dir['paddle.libs']=libs_path
|
|
|
|
# change rpath of core.ext, add $ORIGIN/../libs/ to it.
|
|
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
|
|
# core.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
|
|
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
|
|
if '${CMAKE_BUILD_TYPE}' == 'Release':
|
|
if os.name != 'nt':
|
|
# only change rpath in Release mode, since in Debug mode, core.xx is too large to be changed.
|
|
if "@APPLE@" == "1":
|
|
command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/core" + ext_name
|
|
else:
|
|
command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/core" + ext_name
|
|
if os.system(command) != 0:
|
|
raise Exception("patch core.%s failed, command: %s" % (ext_name, command))
|
|
if '${WITH_FLUID_ONLY}'== 'OFF':
|
|
# change rpath of _swig_paddle.xx.
|
|
if "@APPLE@" == "1":
|
|
command = "install_name_tool -id \"@loader_path/../paddle/libs/\" ${PADDLE_BINARY_DIR}/python/py_paddle/_swig_paddle" + ext_name
|
|
else:
|
|
command = "patchelf --set-rpath '$ORIGIN/../paddle/libs/' ${PADDLE_BINARY_DIR}/python/py_paddle/_swig_paddle" + ext_name
|
|
if os.system(command) != 0:
|
|
raise Exception("patch _swig_paddle.%s failed, command: %s" % (ext_name, command))
|
|
|
|
ext_modules = [Extension('_foo', ['stub.cc'])]
|
|
if os.name == 'nt':
|
|
# fix the path separator under windows
|
|
fix_package_dir = {}
|
|
for k, v in package_dir.items():
|
|
fix_package_dir[k] = v.replace('/', '\\')
|
|
package_dir = fix_package_dir
|
|
ext_modules = []
|
|
|
|
setup(name='${PACKAGE_NAME}',
|
|
version='${PADDLE_VERSION}',
|
|
description='Parallel Distributed Deep Learning',
|
|
install_requires=setup_requires,
|
|
packages=packages,
|
|
ext_modules=ext_modules,
|
|
package_data=package_data,
|
|
package_dir=package_dir,
|
|
scripts=paddle_bins
|
|
)
|