m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

set default python3,fix incompatible,cache dir for third party,unify error code,for windows (#26178)

Browse Source 
* set default python3 for paddle windows,test=win

* set default python3,cache dir for third party,error code,test=win

* fix some incompatible

* fix some error

* set virtual environment,test=win
revert-24895-update_cub
Zhou Wei 5 years ago committed by GitHub
parent 6d22f5c73e
commit 5017aa76e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 103 additions and 33 deletions
Show Stats Download Patch File Download Diff File

115
paddle/scripts/paddle_build.bat
Unescape View File

@ -1,4 +1,4 @@
@ECHO OFF
@ECHO ON
SETLOCAL
rem Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
@ -19,22 +19,66 @@ rem =================================================
rem Paddle CI Task On Windows Platform
rem =================================================
set work_dir=%cd%
if exist build rmdir build /s/q
mkdir build
cd /d build
rem ------initialize the virtual environment------
if not defined PYTHON_ROOT set PYTHON_ROOT=c:\Python37
set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe
%PYTHON_EXECUTABLE% -m pip install virtualenv
if not exist paddle_winci (%PYTHON_EXECUTABLE% -m virtualenv paddle_winci)
call paddle_winci\Scripts\activate.bat
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
rem ------pre install requirement----------
where python
where pip
pip install --upgrade pip
pip install wheel
pip install gym
pip install -r %work_dir%\python\requirements.txt
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
rem ------initialize common variable------
if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0"
if not defined BRANCH set BRANCH=develop
if not defined PYTHON_ROOT set PYTHON_ROOT=c:\Python27
if not defined WITH_MKL set WITH_MKL=ON
if not defined WITH_AVX set WITH_AVX=ON
if not defined WITH_AVX set WITH_AVX=ON
if not defined WITH_GPU set WITH_GPU=OFF
if not defined WITH_TESTING set WITH_TESTING=ON
if not defined WITH_PYTHON set WITH_PYTHON=ON
if not defined ON_INFER set ON_INFER=ON
if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=OFF
if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=d:/.cache/inference_demo
if not defined THIRD_PARTY_PATH set THIRD_PARTY_PATH=%work_dir:\=/%/build/third_party
set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe
dir d:\.cache
if not defined WITH_TPCACHE set WITH_TPCACHE=ON
rem ------set cache third_party------
set cache_dir=%work_dir%\..\cache
dir %cache_dir%
set INFERENCE_DEMO_INSTALL_DIR=%cache_dir:\=/%/inference_demo
if not exist %cache_dir%\tools (
git clone https://github.com/zhouwei25/tools %cache_dir%\tools
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
)
if "%WITH_TPCACHE%"=="OFF" (
set THIRD_PARTY_PATH=%work_dir:\=/%/build/third_party
goto :CASE_%1
)
echo set -ex > cache.sh
echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake ^|md5sum ^| awk '{print $1}') >> cache.sh
echo echo ${md5_content}^>md5.txt >> cache.sh
%cache_dir%\tools\busybox64.exe cat cache.sh
%cache_dir%\tools\busybox64.exe bash cache.sh
set /p md5=< md5.txt
if "%WITH_GPU%"=="ON" (
set THIRD_PARTY_PATH=%cache_dir:\=/%/third_party_GPU/%md5%
) else (
set THIRD_PARTY_PATH=%cache_dir:\=/%/third_party/%md5%
)
goto :CASE_%1
@ -45,6 +89,8 @@ echo "wincheck_openbals: run Windows OPENBLAS/CPU CI tasks on Windows"
exit /b 1
:CASE_wincheck_mkl
set WITH_MKL=ON
set WITH_GPU=OFF
call :cmake || goto cmake_error
call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error
@ -54,6 +100,8 @@ call :check_change_of_unittest || goto check_change_of_unittest_error
goto:success
:CASE_wincheck_openblas
set WITH_MKL=OFF
set WITH_GPU=ON
call :cmake || goto cmake_error
call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error
@ -70,13 +118,14 @@ echo ========================================
echo Step 1. Cmake ...
echo ========================================
mkdir build
cd /d build
cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0" -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH%
echo cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_TOOLKIT_ROOT_DIR% -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH%
cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_TOOLKIT_ROOT_DIR% -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH%
goto:eof
:cmake_error
exit /b %ERRORLEVEL%
call paddle_winci\Scripts\deactivate.bat
echo cmake failed!
exit /b 1
rem ---------------------------------------------------------------------------------------------
:build
@ -115,7 +164,9 @@ if %ERRORLEVEL% NEQ 0 (
goto:eof
:build_error
exit /b %ERRORLEVEL%
call paddle_winci\Scripts\deactivate.bat
echo build paddle failed!
exit /b 7
rem ---------------------------------------------------------------------------------------------
:test_whl_pacakage
@ -124,20 +175,23 @@ echo Step 3. Test pip install whl package ...
echo ========================================
dir /s /b python\dist\*.whl > whl_file.txt
set /p PADDLE_WHL_FILE_WIN=< whl_file.txt
%PYTHON_EXECUTABLE% -m pip install -U %PADDLE_WHL_FILE_WIN%
pip install -U %PADDLE_WHL_FILE_WIN%
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
echo import paddle.fluid;print(paddle.__version__) > test_whl.py
%PYTHON_EXECUTABLE% test_whl.py
python test_whl.py
goto:eof
:test_whl_pacakage_error
exit /b %ERRORLEVEL%
call paddle_winci\Scripts\deactivate.bat
exit /b 3
rem ---------------------------------------------------------------------------------------------
:unit_test
echo ========================================
echo Step 4. Running unit tests ...
echo ========================================
%PYTHON_EXECUTABLE% -m pip install --upgrade pip
dir %THIRD_PARTY_PATH:/=\%\install\openblas\lib
dir %THIRD_PARTY_PATH:/=\%\install\openblas\bin
@ -147,27 +201,28 @@ dir %THIRD_PARTY_PATH:/=\%\install\mkldnn\bin
dir %THIRD_PARTY_PATH:/=\%\install\warpctc\bin
set PATH=%THIRD_PARTY_PATH:/=\%\install\openblas\lib;%THIRD_PARTY_PATH:/=\%\install\openblas\bin;%THIRD_PARTY_PATH:/=\%\install\zlib\bin;%THIRD_PARTY_PATH:/=\%\install\mklml\lib;%THIRD_PARTY_PATH:/=\%\install\mkldnn\bin;%THIRD_PARTY_PATH:/=\%\install\warpctc\bin;%PATH%
ctest.exe --output-on-failure -C Release -j 10
ctest.exe --output-on-failure -C Release -j 7
goto:eof
:unit_test_error
exit /b %ERRORLEVEL%
call paddle_winci\Scripts\deactivate.bat
exit /b 8
rem ---------------------------------------------------------------------------------------------
:test_inference
echo ========================================
echo Step 5. Testing fluid library for inference ...
echo ========================================
if NOT EXIST "d:\.cache\tools" (
git clone https://github.com/zhouwei25/tools.git d:\.cache\tools
)
cd %work_dir%\paddle\fluid\inference\api\demo_ci
d:\.cache\tools\busybox64.exe bash run.sh %work_dir:\=/% %WITH_MKL% %WITH_GPU% d:/.cache/inference_demo
%cache_dir%\tools\busybox64.exe bash run.sh %work_dir:\=/% %WITH_MKL% %WITH_GPU% %cache_dir:\=/%/inference_demo
goto:eof
:test_inference_error
exit /b %ERRORLEVEL%
call paddle_winci\Scripts\deactivate.bats
echo Testing fluid library for inference failed!
exit /b 5
rem ---------------------------------------------------------------------------------------------
:check_change_of_unittest
@ -205,7 +260,7 @@ echo git fetch upstream $BRANCH # develop is not fetched>> check_change_of_
echo fi>> check_change_of_unittest.sh
echo git checkout -b origin_pr >> check_change_of_unittest.sh
echo git checkout -f $BRANCH >> check_change_of_unittest.sh
echo cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE:\=\\% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0" -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% >> check_change_of_unittest.sh
echo cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_TOOLKIT_ROOT_DIR% -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% >> check_change_of_unittest.sh
echo cat ^<^<EOF>> check_change_of_unittest.sh
echo ============================================ >> check_change_of_unittest.sh
echo Generate unit tests.spec of develop. >> check_change_of_unittest.sh
@ -236,11 +291,12 @@ echo exit 1 >> check_change_of_unittest.sh
echo fi>> check_change_of_unittest.sh
echo fi>> check_change_of_unittest.sh
echo git checkout -f origin_pr >> check_change_of_unittest.sh
d:\.cache\tools\busybox64.exe bash check_change_of_unittest.sh
%cache_dir%\tools\busybox64.exe bash check_change_of_unittest.sh
goto:eof
:check_change_of_unittest_error
exit /b %ERRORLEVEL%
call paddle_winci\Scripts\deactivate.bat
exit /b 6
rem ---------------------------------------------------------------------------------------------
@ -258,6 +314,7 @@ taskkill /f /im git-remote-https.exe 2>NUL
taskkill /f /im vctip.exe 2>NUL
taskkill /f /im cvtres.exe 2>NUL
taskkill /f /im rc.exe 2>NUL
call paddle_winci\Scripts\deactivate.bat
echo Windows CI run successfully!
exit /b 0

5
python/paddle/fluid/contrib/tests/test_distributed_reader.py
Unescape View File

@ -36,8 +36,9 @@ class TestDistributedReader(unittest.TestCase):
data = next(reader())
assert data == 1
os.unsetenv('PADDLE_TRAINER_ID')
os.unsetenv('PADDLE_TRAINERS_NUM')
#Note: windows python3 don't have unsetenv
del os.environ['PADDLE_TRAINER_ID']
del os.environ['PADDLE_TRAINERS_NUM']
if __name__ == '__main__':

1
python/paddle/fluid/tests/unittests/CMakeLists.txt
Unescape View File

@ -191,6 +191,7 @@ function(py_test_modules TARGET_NAME)
endif()
endfunction()
function(bash_test_modules TARGET_NAME)
if(NOT WITH_TESTING)
return()

13
python/paddle/fluid/tests/unittests/__init__.py
Unescape View File

@ -10,4 +10,15 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# limitations under the License.p
# Note: On Windows, import form subdirectories such as dirA()->dirB(), current directory
# will still be dirA(), But is should be dirB(). So it will ModulNotFoundError
# please refer to https://stackoverflow.com/questions/8953844/import-module-from-subfolder
import os
if os.name == 'nt':
import sys
dirname, filename = os.path.split(os.path.abspath(__file__))
sys.path.insert(0, dirname)
print(sys.path)

2
python/requirements.txt
Unescape View File

@ -6,7 +6,7 @@ gast==0.3.3
matplotlib<=2.2.4 ; python_version<"3.6"
scipy>=0.19.0, <=1.2.1 ; python_version<"3.5"
nltk>=3.2.2, <=3.4 ; python_version<"3.5"
matplotlib ; python_version>="3.6"
matplotlib<=3.2.1 ; python_version>="3.6"
scipy<=1.3.1 ; python_version=="3.5"
scipy ; python_version>"3.5"
nltk ; python_version>="3.5"

Write Preview
Loading…
Cancel
Save