m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[Custom OP]Fix problem of custom op unitests on Windows CI (#31114)

Browse Source 
* fix some problem of Windows custom op

* fix some problem of Windows custom op

* fix some problem of Windows custom op
revert-31068-fix_conv3d_windows
Zhou Wei 4 years ago committed by GitHub
parent 70131b475f
commit 4b220550ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 39 additions and 40 deletions
Show Stats Download Patch File Download Diff File

12
paddle/scripts/paddle_build.bat
Unescape View File

@ -123,6 +123,7 @@ where pip
pip install wheel --user
pip install -r %work_dir%\python\unittest_py\requirements.txt --user
pip install -r %work_dir%\python\requirements.txt --user
if %ERRORLEVEL% NEQ 0 (
echo pip install requirements.txt failed!
exit /b 7
@ -234,7 +235,6 @@ set start=%start:~4,10%
@ECHO ON
if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0
set PATH=%TENSORRT_ROOT:/=\%\lib;%CUDA_TOOLKIT_ROOT_DIR%\bin;%CUDA_TOOLKIT_ROOT_DIR%\libnvvp;%PATH%
set CUDA_PATH=%CUDA_TOOLKIT_ROOT_DIR%
rem ------set third_party cache dir------
@ -246,7 +246,11 @@ set /p day_before=< %cache_dir%\day.txt
if %day_now% NEQ %day_before% (
echo %day_now% > %cache_dir%\day.txt
type %cache_dir%\day.txt
if %day_now% EQU 20 (
if %day_now% EQU 25 (
rmdir %cache_dir%\third_party_GPU/ /s/q
rmdir %cache_dir%\third_party/ /s/q
)
if %day_now% EQU 10 (
rmdir %cache_dir%\third_party_GPU/ /s/q
rmdir %cache_dir%\third_party/ /s/q
)
@ -452,7 +456,7 @@ if "%WITH_GPU%"=="ON" (
:parallel_test_base_gpu
echo ========================================
echo Running GPU unit tests...
echo Running GPU unit tests in parallel way ...
echo ========================================
setlocal enabledelayedexpansion
@ -471,6 +475,7 @@ goto:eof
echo ========================================
echo Running CPU unit tests in parallel way ...
echo ========================================
ctest.exe -E "(%disable_ut_quickly%)" -LE %nightly_label% --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4
goto:eof
@ -676,6 +681,7 @@ taskkill /f /im vctip.exe 2>NUL
taskkill /f /im cvtres.exe 2>NUL
taskkill /f /im rc.exe 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="python.exe" call terminate 2>NUL
taskkill /f /im python.exe 2>NUL
echo Windows CI run successfully!
exit /b 0

7
python/paddle/fluid/tests/CMakeLists.txt
Unescape View File

@ -10,13 +10,8 @@ add_subdirectory(unittests)
add_subdirectory(book)
# TODO: support New Custom OP on Mac
if(LINUX)
if(NOT APPLE)
add_subdirectory(custom_op)
endif()
# Windows CPU machine doesn't have CUDA, can't compile .cu file
# if(WIN32 AND WITH_GPU)
# add_subdirectory(custom_op)
# endif()
set_tests_properties(test_beam_search_decoder PROPERTIES TIMEOUT 120)

16
python/paddle/fluid/tests/custom_op/CMakeLists.txt
Unescape View File

@ -1,11 +1,13 @@
# New custom OP can support Windows/Linux now
# 'test_custom_relu_op_setup/jit' compile .cc and .cu file
py_test(test_custom_relu_op_setup SRCS test_custom_relu_op_setup.py)
py_test(test_custom_relu_op_jit SRCS test_custom_relu_op_jit.py)
# Compiling shared library will cost some time, but running process is very fast.
set_tests_properties(test_custom_relu_op_setup PROPERTIES TIMEOUT 250)
set_tests_properties(test_custom_relu_op_jit PROPERTIES TIMEOUT 180)
if(WITH_GPU)
# 'test_custom_relu_op_setup/jit' compile .cc and .cu file
py_test(test_custom_relu_op_setup SRCS test_custom_relu_op_setup.py)
py_test(test_custom_relu_op_jit SRCS test_custom_relu_op_jit.py)
# Compiling shared library will cost some time, but running process is very fast.
set_tests_properties(test_custom_relu_op_setup PROPERTIES TIMEOUT 250)
set_tests_properties(test_custom_relu_op_jit PROPERTIES TIMEOUT 180)
endif()
py_test(test_sysconfig SRCS test_sysconfig.py)

6
python/paddle/utils/cpp_extension/cpp_extension.py
Unescape View File

@ -24,7 +24,7 @@ from setuptools.command.easy_install import easy_install
from setuptools.command.build_ext import build_ext
from .extension_utils import find_cuda_home, normalize_extension_kwargs, add_compile_flag, bootstrap_context
from .extension_utils import is_cuda_file, prepare_unix_cflags, prepare_win_cflags, add_std_without_repeat, get_build_directory
from .extension_utils import is_cuda_file, prepare_unix_cudaflags, prepare_win_cudaflags, add_std_without_repeat, get_build_directory
from .extension_utils import _import_module_from_library, CustomOpInfo, _write_setup_file, _jit_compile, parse_op_name_from
from .extension_utils import check_abi_compatibility, log_v, IS_WINDOWS, OS_NAME
from .extension_utils import use_new_custom_op_load_method, MSVC_COMPILE_FLAGS
@ -267,7 +267,7 @@ class BuildExtension(build_ext, object):
if isinstance(cflags, dict):
cflags = cflags['nvcc']
else:
cflags = prepare_unix_cflags(cflags)
cflags = prepare_unix_cudaflags(cflags)
# cxx compile Cpp source
elif isinstance(cflags, dict):
cflags = cflags['cxx']
@ -332,7 +332,7 @@ class BuildExtension(build_ext, object):
else:
cflags = []
cflags = prepare_win_cflags(cflags) + ['--use-local-env']
cflags = prepare_win_cudaflags(cflags) + ['--use-local-env']
for flag in MSVC_COMPILE_FLAGS:
cflags = ['-Xcompiler', flag] + cflags
cmd = [nvcc_cmd, '-c', src, '-o', obj

38
python/paddle/utils/cpp_extension/extension_utils.py
Unescape View File

@ -41,17 +41,13 @@ IS_WINDOWS = OS_NAME.startswith('win')
MSVC_COMPILE_FLAGS = [
'/MT', '/wd4819', '/wd4251', '/wd4244', '/wd4267', '/wd4275', '/wd4018',
'/wd4190', '/EHsc', '/w', '/DPADDLE_WITH_CUDA', '/DEIGEN_USE_GPU',
'/DNDEBUG'
'/wd4190', '/EHsc', '/w', '/DGOOGLE_GLOG_DLL_DECL',
'/DBOOST_HAS_STATIC_ASSERT', '/DNDEBUG', '/DPADDLE_USE_DSO'
]
MSVC_LINK_FLAGS = [
'/MACHINE:X64', 'paddle_framework.lib', 'cudadevrt.lib', 'cudart_static.lib'
]
MSVC_LINK_FLAGS = ['/MACHINE:X64', 'paddle_framework.lib']
COMMON_NVCC_FLAGS = [
'-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU', '-DPADDLE_USE_DSO', '-O3'
]
COMMON_NVCC_FLAGS = ['-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU', '-O3']
GCC_MINI_VERSION = (5, 4, 0)
# Give warning if using wrong compiler
@ -216,7 +212,7 @@ class CustomOpInfo:
return next(reversed(self.op_info_map.items()))
def prepare_unix_cflags(cflags):
def prepare_unix_cudaflags(cflags):
"""
Prepare all necessary compiled flags for nvcc compiling CUDA files.
"""
@ -228,13 +224,11 @@ def prepare_unix_cflags(cflags):
return cflags
def prepare_win_cflags(cflags):
def prepare_win_cudaflags(cflags):
"""
Prepare all necessary compiled flags for nvcc compiling CUDA files.
"""
cflags = COMMON_NVCC_FLAGS + [
'-DGOOGLE_GLOG_DLL_DECL', '-DBOOST_HAS_STATIC_ASSERT', '-w'
] + cflags + get_cuda_arch_flags(cflags)
cflags = COMMON_NVCC_FLAGS + ['-w'] + cflags + get_cuda_arch_flags(cflags)
return cflags
@ -269,6 +263,7 @@ def normalize_extension_kwargs(kwargs, use_cuda=False):
# append necessary include dir path of paddle
include_dirs = kwargs.get('include_dirs', [])
include_dirs.extend(find_paddle_includes(use_cuda))
kwargs['include_dirs'] = include_dirs
# append necessary lib path of paddle
@ -282,6 +277,8 @@ def normalize_extension_kwargs(kwargs, use_cuda=False):
# append link flags
extra_link_args = kwargs.get('extra_link_args', [])
extra_link_args.extend(MSVC_LINK_FLAGS)
if use_cuda:
extra_link_args.extend(['cudadevrt.lib', 'cudart_static.lib'])
kwargs['extra_link_args'] = extra_link_args
else:
# append compile flags
@ -323,6 +320,7 @@ def find_cuda_home():
if six.PY3:
nvcc_path = nvcc_path.decode()
nvcc_path = nvcc_path.rstrip('\r\n')
# for example: /usr/local/cuda/bin/nvcc
cuda_home = os.path.dirname(os.path.dirname(nvcc_path))
except:
@ -368,10 +366,9 @@ def find_paddle_includes(use_cuda=False):
third_party_dir = os.path.join(paddle_include_dir, 'third_party')
include_dirs = [paddle_include_dir, third_party_dir]
#TODO(zhouwei): because eigen need cuda_runtime.h
#So, extend cuda_include_dir always
cuda_include_dir = find_cuda_includes()
include_dirs.extend(cuda_include_dir)
if use_cuda:
cuda_include_dir = find_cuda_includes()
include_dirs.extend(cuda_include_dir)
return include_dirs
@ -400,10 +397,9 @@ def find_paddle_libraries(use_cuda=False):
# pythonXX/site-packages/paddle/libs
paddle_lib_dirs = [get_lib()]
#TODO(zhouwei): because eigen need cuda_runtime.h
#So, extend cuda_lib_dir always
cuda_lib_dir = find_cuda_libraries()
paddle_lib_dirs.extend(cuda_lib_dir)
if use_cuda:
cuda_lib_dir = find_cuda_libraries()
paddle_lib_dirs.extend(cuda_lib_dir)
return paddle_lib_dirs

Write Preview
Loading…
Cancel
Save