m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add extern python interp

Browse Source
avx_docs
liaogang 9 years ago
parent aee0857838
commit 8acd1ac31a
15 changed files with 62 additions and 36 deletions
Show Stats Download Patch File Download Diff File

32
paddle/api/CMakeLists.txt
Unescape View File

@ -1,3 +1,21 @@
FUNCTION(generate_python_api target_name)
ADD_CUSTOM_COMMAND(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py
${PROJ_ROOT}/paddle/Paddle_wrap.cxx
${PROJ_ROOT}/paddle/Paddle_wrap.h
COMMAND ${SWIG_EXECUTABLE} -python -c++ -outcurrentdir -I../ api/Paddle.swig
&& mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py
DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig
${PROJ_ROOT}/paddle/api/PaddleAPI.h
${external_project_dependencies}
WORKING_DIRECTORY ${PROJ_ROOT}/paddle
COMMENT "Generate Python API from swig")
ADD_CUSTOM_TARGET(${target_name} ALL DEPENDS
${PROJ_ROOT}/paddle/Paddle_wrap.cxx
${PROJ_ROOT}/paddle/Paddle_wrap.h
${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py
${external_project_dependencies})
ENDFUNCTION(generate_python_api)
set(API_SOURCES
Arguments.cpp
ConfigParser.cpp
@ -42,7 +60,7 @@ file(GLOB PY_PADDLE_PYTHON_FILES ${PROJ_ROOT}/paddle/py_paddle/*.py)
# TODO(yuyang18) : make wheel name calculated by cmake
add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp
COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel
COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
COMMAND ${CMAKE_COMMAND} -E touch dist/.timestamp
COMMAND rm -rf py_paddle.egg-info build
WORKING_DIRECTORY ${PROJ_ROOT}/paddle
@ -76,5 +94,17 @@ add_dependencies(python_api_wheel python_swig_sources
paddle_cuda)
if(WITH_TESTING)
SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip)
ExternalProject_Add(pip
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY https://github.com/pypa/pip.git
GIT_TAG 9.0.1
PREFIX ${PIP_SOURCES_DIR}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
BUILD_IN_SOURCE 1
DEPENDS python setuptools python_api_wheel
)
add_subdirectory(test)
endif()

2
paddle/api/test/CMakeLists.txt
Unescape View File

@ -1,2 +1,2 @@
add_test(NAME test_swig_api
COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh)
COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh ${PYTHON_EXECUTABLE} ${PYTHON_INSTALL_DIR}/bin/pip)

10
paddle/api/test/run_tests.sh
Unescape View File

@ -20,11 +20,11 @@ popd > /dev/null
cd $SCRIPTPATH
rm -rf .test_env
virtualenv .test_env
source .test_env/bin/activate
# rm -rf .test_env
# virtualenv .test_env
# source .test_env/bin/activate
pip --timeout 600 install ../../dist/*.whl
$1 -m pip install ../../dist/*.whl
test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py"
@ -33,7 +33,7 @@ export PYTHONPATH=$PWD/../../../python/
for fn in $test_list
do
echo "test $fn"
python $fn
$1 $fn
if [ $? -ne 0 ]; then
exit 1
fi

9
paddle/scripts/travis/build_and_test.sh
Unescape View File

@ -1,15 +1,13 @@
#!/bin/bash
./build_submodules.sh
source ./common.sh
CMAKE_EXTRA=""
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
CMAKE_EXTRA="-DPYTHON_LIBRARY=/usr/local/Cellar/python/2.7.12_1/Frameworks/Python.framework/Versions/2.7/lib/python2.7/config/libpython2.7.dylib"
CMAKE_EXTRA="-DWITH_SWIG_PY=OFF"
else
CMAKE_EXTRA="-DWITH_SWIG_PY=ON"
fi
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA}
cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA}
NPROC=1
if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
@ -22,6 +20,5 @@ elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC"
fi
sudo make install
sudo paddle version

2
paddle/setup.py.in
Unescape View File

@ -14,7 +14,9 @@
# This file is used to build paddle python binding package.
# It will be invoked by Makefile that generated by COMAKE
from setuptools import setup, Extension
import numpy as np
import api.paddle_ld_flags
import platform

5
paddle/trainer/tests/CMakeLists.txt
Unescape View File

@ -17,9 +17,10 @@ add_test(NAME test_Compare
################# test_Trainer ###########################
add_unittest_without_exec(test_Trainer
test_Trainer.cpp)
set(diy_dll_dir ${CMAKE_CURRENT_BINARY_DIR}/../../gserver/tests)
add_test(NAME test_Trainer
COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
${PYTHON_EXECUTABLE} ${PROJ_ROOT}/paddle/trainer/tests/gen_proto_data.py &&
${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
${CMAKE_CURRENT_BINARY_DIR}/test_Trainer
WORKING_DIRECTORY ${PROJ_ROOT}/paddle/)
@ -82,5 +83,5 @@ add_test(NAME test_PyDataProviderWrapper
#################### test_config_parser #########################
add_test(NAME test_config_parser
COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
python ${PROJ_ROOT}/paddle/trainer/tests/config_parser_test.py
${PYTHON_EXECUTABLE} ${PROJ_ROOT}/paddle/trainer/tests/config_parser_test.py
WORKING_DIRECTORY ${PROJ_ROOT}/paddle/)

5
paddle/trainer/tests/test_Trainer.cpp
Unescape View File

@ -96,11 +96,6 @@ TEST(checkGradient, multi) {
TEST(checkGradient, hsigmoid) { checkGradientTest(configFile2, false, false); }
TEST(checkGradient, chunk) {
#if defined(__APPLE__) || defined(__OSX__)
EXPECT_EQ(0, system("python trainer/tests/gen_proto_data.py"));
#else
EXPECT_EQ(0, system("python2 trainer/tests/gen_proto_data.py"));
#endif
checkGradientTest(configFile3, false, false);
#ifndef PADDLE_ONLY_CPU
checkGradientTest(configFile3, true, true);

1
paddle/utils/.gitignore vendored
Unescape View File

@ -1 +1,2 @@
enable_virtualenv.c
PythonUtil.cpp

2
paddle/utils/CMakeLists.txt
Unescape View File

@ -1,5 +1,7 @@
# The utilities for paddle
configure_file(PythonUtil.cpp.in ${PROJ_ROOT}/paddle/utils/PythonUtil.cpp)
file(GLOB UTIL_HEADERS . *.h)
file(GLOB UTIL_SOURCES . *.cpp)
create_resources(enable_virtualenv.py enable_virtualenv.c)

2
paddle/utils/PythonUtil.cpp → paddle/utils/PythonUtil.cpp.in
Unescape View File

@ -195,6 +195,8 @@ extern const char enable_virtualenv_py[];
}
void initPython(int argc, char** argv) {
#ifndef PADDLE_NO_PYTHON
char PythonHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
Py_SetPythonHome(PythonHome);
Py_SetProgramName(argv[0]);
Py_Initialize();
PySys_SetArgv(argc, argv);

4
proto/CMakeLists.txt
Unescape View File

@ -18,7 +18,7 @@ foreach(filename ${proto_filenames})
${PROTO_GEN}
${CUR_PROTO_GEN})
add_custom_command(OUTPUT ${CUR_PROTO_GEN}
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE}
--cpp_out ${CMAKE_CURRENT_BINARY_DIR}
--proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename}
DEPENDS ${filename} ${external_project_dependencies})
@ -29,7 +29,7 @@ foreach(filename ${proto_filenames})
${CUR_PROTO_GEN_PY}
${PROTO_GEN_PY})
add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY}
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto
COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto
--proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename}
DEPENDS ${filename} ${external_project_dependencies})
endforeach()

6
python/CMakeLists.txt
Unescape View File

@ -14,17 +14,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in
${CMAKE_CURRENT_BINARY_DIR}/setup.py)
add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel
COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp
DEPENDS gen_proto_py ${PY_FILES})
add_custom_target(paddle_python ALL DEPENDS
${OUTPUT_DIR}/.timestamp)
find_python_module(pip REQUIRED)
find_python_module(wheel REQUIRED)
find_python_module(google.protobuf REQUIRED)
add_subdirectory(paddle/trainer_config_helpers/tests)
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/

8
python/paddle/trainer_config_helpers/tests/CMakeLists.txt
Unescape View File

@ -1,12 +1,12 @@
#################### test_config_parser #########################
add_test(NAME layers_test
COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
python ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/layers_test.py
${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/layers_test.py
WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
add_test(NAME test_reset_hook
COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
python ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/test_reset_hook.py
${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/test_reset_hook.py
WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
if (PROTOBUF_3)
@ -14,12 +14,12 @@ if (PROTOBUF_3)
ProtobufEqualMain.cpp)
add_test(NAME test_layerHelpers
COMMAND
${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh
${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh ${PYTHON_EXECUTABLE}
${CMAKE_CURRENT_BINARY_DIR}/protobuf_equal
)
else()
add_test(NAME test_layerHelpers
COMMAND
${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh
${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh ${PYTHON_EXECUTABLE}
)
endif()

8
python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh
Unescape View File

@ -10,13 +10,13 @@ protostr=$PWD/protostr
for conf in ${configs[*]}
do
echo "Generating " $conf
python -m paddle.utils.dump_config $conf.py > $protostr/$conf.protostr.unittest
cat ${conf}.py |python test_config_parser_for_non_file_config.py > $protostr/$conf.protostr.non_file_config.unittest
$1 -m paddle.utils.dump_config $conf.py > $protostr/$conf.protostr.unittest
cat ${conf}.py |$1 test_config_parser_for_non_file_config.py > $protostr/$conf.protostr.non_file_config.unittest
done
for conf in ${whole_configs[*]}
do
echo "Generating " $conf
python -m paddle.utils.dump_config $conf.py "" --whole > $protostr/$conf.protostr.unittest
cat ${conf}.py |python test_config_parser_for_non_file_config.py --whole > $protostr/$conf.protostr.non_file_config.unittest
$1 -m paddle.utils.dump_config $conf.py "" --whole > $protostr/$conf.protostr.unittest
cat ${conf}.py |$1 test_config_parser_for_non_file_config.py --whole > $protostr/$conf.protostr.non_file_config.unittest
done

2
python/paddle/trainer_config_helpers/tests/configs/run_tests.sh
Unescape View File

@ -7,7 +7,7 @@ protostr=`dirname $0`/protostr
files=`ls $protostr | grep -v "unittest"`
./generate_protostr.sh
./generate_protostr.sh $1
. ./file_list.sh

Write Preview
Loading…
Cancel
Save