Use external project for NCCL (#5028)

CMakeLists.txt

@@ -127,9 +127,9 @@ include(external/warpctc)   # download, build, install warpctc
 include(external/any)       # download libn::any
 include(external/eigen)     # download eigen3
 include(external/pybind11)    # download pybind11
+include(external/nccl)
 
 include(cudnn)              # set cudnn libraries, must before configure
-include(nccl)               # set nccl libraries
 include(configure)          # add paddle env configuration
 include(generic)            # simplify cmake module
 include(package)            # set paddle packages

cmake/configure.cmake

@@ -68,13 +68,6 @@ else()
     if(NOT CUDNN_FOUND)
         message(FATAL_ERROR "Paddle needs cudnn to compile")
     endif()
-    if (NOT NCCL_INCLUDE_DIR)
-        message(FATAL_ERROR "Paddle needs nccl header to compile")
-    endif()
-    if (NOT WITH_DSO AND NOT NCCL_LIBRARY)
-        message(FATAL_ERROR "Paddle needs nccl libraries when WITH_DSO=OFF")
-    endif()
-
 
     set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SIMD_FLAG}")
 

cmake/external/nccl.cmake

@@ -0,0 +1,50 @@
+INCLUDE(ExternalProject)
+
+SET(NCCL_SOURCE_DIR ${THIRD_PARTY_PATH}/nccl)
+
+INCLUDE_DIRECTORIES(${NCCL_SOURCE_DIR}/src/extern_nccl)
+
+
+if(WITH_DSO)
+  # If we use DSO, we do not build nccl, just download the dependencies
+  set(NCCL_BUILD_COMMAND "")
+  set(NCCL_INSTALL_COMMAND "")
+  set(NCCL_INSTALL_DIR "")
+else()
+  # otherwise, we build nccl and link it.
+  set(NCCL_BUILD_COMMAND "make -j 8")
+  set(NCCL_INSTALL_COMMAND  "make install")
+  SET(NCCL_INSTALL_DIR ${THIRD_PARTY_PATH}/install/nccl)
+endif()
+
+ExternalProject_Add(
+        extern_nccl
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        GIT_REPOSITORY  "https://github.com/NVIDIA/nccl.git"
+        GIT_TAG         "v1.3.4-1"
+        PREFIX          "${NCCL_SOURCE_DIR}"
+        UPDATE_COMMAND  ""
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND     "${NCCL_BUILD_COMMAND}"
+        INSTALL_COMMAND   "${NCCL_INSTALL_COMMAND}"
+        INSTALL_DIR       "${NCCL_INSTALL_DIR}"
+        TEST_COMMAND      ""
+)
+
+if (WITH_DSO)
+  if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
+    set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/lib_any_dummy.c)
+    file(WRITE ${dummyfile} "const char * dummy_any = \"${dummyfile}\";")
+    add_library(nccl STATIC ${dummyfile})
+  else()
+    add_library(nccl INTERFACE)
+  endif()
+else()
+  ADD_LIBRARY(nccl STATIC IMPORTED GLOBAL)
+  SET_PROPERTY(TARGET nccl PROPERTY IMPORTED_LOCATION
+          ${NCCL_INSTALL_DIR}/lib/libnccl.a)
+endif()
+
+add_dependencies(nccl extern_nccl)
+
+LIST(APPEND external_project_dependencies nccl)

cmake/nccl.cmake

@@ -1,30 +0,0 @@
-if (NOT WITH_GPU)
-  return ()
-endif()
-
-set(NCCL_ROOT "/usr" CACHE PATH "CUDNN ROOT")
-find_path(NCCL_INCLUDE_DIR nccl.h PATHS
-        ${NCCL_ROOT} ${NCCL_ROOT}/include
-        $ENV{NCCL_ROOT} $ENV{NCCL_ROOT}/include ${CUDA_TOOLKIT_INCLUDE}
-        NO_DEFAULT_PATH)
-
-get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH)
-
-set(TARGET_ARCH "x86_64")
-if(NOT ${CMAKE_SYSTEM_PROCESSOR})
-  set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR})
-endif()
-
-list(APPEND NCCL_CHECK_LIBRARY_DIRS
-        ${NCCL_ROOT}
-        ${NCCL_ROOT}/lib64
-        ${NCCL_ROOT}/lib
-        ${NCCL_ROOT}/lib/${TARGET_ARCH}-linux-gnu
-        $ENV{NCCL_ROOT}
-        $ENV{NCCL_ROOT}/lib64
-        $ENV{NCCL_ROOT}/lib
-        /usr/lib)
-find_library(NCCL_LIBRARY NAMES libnccl.so libnccl.dylib # libcudnn_static.a
-        PATHS ${NCCL_CHECK_LIBRARY_DIRS} ${NCCL_INCLUDE_DIR} ${__libpath_hist}
-        NO_DEFAULT_PATH
-        DOC "Path to nccl library.")

paddle/platform/dynload/CMakeLists.txt

@@ -1,2 +1,3 @@
 cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags)
-nv_library(dynload_cuda SRCS cublas.cc cudnn.cc curand.cc nccl.cc DEPS dynamic_loader)
+nv_library(dynload_cuda SRCS cublas.cc cudnn.cc curand.cc nccl.cc
+        DEPS dynamic_loader nccl)