move more third party library related logic to third_party.cmake (#20927)

CMakeLists.txt

@@ -76,6 +76,7 @@ OPTION(WITH_LIBXSMM     "Compile with libxsmm"                          OFF)
 option(COVERALLS_UPLOAD "Package code coverage data to coveralls"       OFF)
 option(WITH_PSLIB       "Compile with pslib support"                    OFF)
 option(WITH_BOX_PS      "Compile with box_ps support"                   OFF)
+option(WITH_XBYAK       "Compile with xbyak support"                    ON)
 option(WITH_CONTRIB     "Compile the third-party contributation"        OFF)
 option(REPLACE_ENFORCE_GLOG "Replace PADDLE_ENFORCE with glog/CHECK for better debug." OFF)
 option(WITH_GRPC     "Use grpc as the default rpc framework"            ${WITH_DISTRIBUTE})

cmake/external/cares.cmake

@@ -11,16 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
-IF(NOT WITH_DISTRIBUTE)
-    return()
-ENDIF()
+# NOTE: c-ares is needed when linking with grpc.
 
 include (ExternalProject)
 
-# NOTE: c-ares is needed when linking with grpc.
-
 SET(CARES_SOURCES_DIR ${THIRD_PARTY_PATH}/cares)
 SET(CARES_INSTALL_DIR ${THIRD_PARTY_PATH}/install/cares)
 SET(CARES_INCLUDE_DIR "${CARES_INSTALL_DIR}/include/" CACHE PATH "cares include directory." FORCE)

cmake/external/cub.cmake

@@ -1,6 +1,16 @@
-if(NOT WITH_GPU)
-  return()
-endif()
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 include(ExternalProject)
 

cmake/external/dlpack.cmake

@@ -1,3 +1,17 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 include(ExternalProject)
 
 set(DLPACK_SOURCE_DIR ${THIRD_PARTY_PATH}/dlpack)

cmake/external/eigen.cmake

@@ -1,3 +1,17 @@
+# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 INCLUDE(ExternalProject)
 
 SET(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3)

cmake/external/grpc.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +13,6 @@
 # limitations under the License.
 #
 
-IF(NOT WITH_DISTRIBUTE)
-    return()
-ENDIF()
-
 include (ExternalProject)
 
 SET(GRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/grpc)

cmake/external/libmct.cmake

@@ -12,18 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-IF(NOT ${WITH_LIBMCT})
-  return()
-ENDIF(NOT ${WITH_LIBMCT})
-
-IF(WIN32 OR APPLE)
-    MESSAGE(WARNING
-        "Windows or Mac is not supported with LIBMCT in Paddle yet."
-        "Force WITH_LIBMCT=OFF")
-    SET(WITH_LIBMCT OFF CACHE STRING "Disable LIBMCT package in Windows and MacOS" FORCE)
-    return()
-ENDIF()
-
 INCLUDE(ExternalProject)
 
 SET(LIBMCT_PROJECT       "extern_libmct")

cmake/external/mkldnn.cmake

@@ -100,9 +100,6 @@ if(WIN32)
     SET(MKLDNN_SHARED_LIB ${MKLDNN_INSTALL_DIR}/bin/mkldnn.dll)
 else(WIN32)
     SET(MKLDNN_SHARED_LIB ${MKLDNN_INSTALL_DIR}/libmkldnn.so.0)
-    ADD_CUSTOM_COMMAND(OUTPUT ${MKLDNN_SHARED_LIB}
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB}
-            DEPENDS mkldnn shared_mkldnn)
+    ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB})
 endif(WIN32)
-ADD_CUSTOM_TARGET(mkldnn_shared_lib ALL DEPENDS ${MKLDNN_SHARED_LIB})
-ADD_DEPENDENCIES(mkldnn_shared_lib ${MKLDNN_PROJECT} mkldnn)

cmake/external/pslib.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,18 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-IF(NOT ${WITH_PSLIB})
-  return()
-ENDIF(NOT ${WITH_PSLIB})
-
-IF(WIN32 OR APPLE)
-    MESSAGE(WARNING
-        "Windows or Mac is not supported with PSLIB in Paddle yet."
-        "Force WITH_PSLIB=OFF")
-    SET(WITH_PSLIB OFF CACHE STRING "Disable PSLIB package in Windows and MacOS" FORCE)
-    return()
-ENDIF()
-
 INCLUDE(ExternalProject)
 
 SET(PSLIB_PROJECT       "extern_pslib")

cmake/external/pslib_brpc.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,18 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-IF(NOT ${WITH_PSLIB_BRPC})
-  return()
-ENDIF(NOT ${WITH_PSLIB_BRPC})
-
-IF(WIN32 OR APPLE)
-    MESSAGE(WARNING
-        "Windows or Mac is not supported with PSLIB_BRPC in Paddle yet."
-        "Force WITH_PSLIB_BRPC=OFF")
-    SET(WITH_PSLIB_BRPC OFF CACHE STRING "Disable PSLIB_BRPC package in Windows and MacOS" FORCE)
-    return()
-ENDIF()
-
 INCLUDE(ExternalProject)
 
 SET(PSLIB_BRPC_PROJECT       "extern_pslib_brpc")

cmake/external/rocprim.cmake

@@ -1,6 +1,16 @@
-if (NOT WITH_AMD_GPU)
-    return()
-endif()
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # rocprim is "ROCm Parallel Primitives" for short.
 # It is a header-only library providing HIP and HC parallel primitives

cmake/external/threadpool.cmake

@@ -1,3 +1,17 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 INCLUDE(ExternalProject)
 
 SET(THREADPOOL_SOURCE_DIR ${THIRD_PARTY_PATH}/threadpool)

cmake/external/xbyak.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,12 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set(WITH_XBYAK ON)
-if(WIN32 OR APPLE)
-    SET(WITH_XBYAK OFF CACHE STRING "Disable XBYAK in Windows and MacOS" FORCE)
-    return()
-endif()
-
 include(ExternalProject)
 
 set(XBYAK_PROJECT       extern_xbyak)

cmake/external/xxhash.cmake

@@ -1,3 +1,17 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 INCLUDE(ExternalProject)
 
 set(XXHASH_SOURCE_DIR ${THIRD_PARTY_PATH}/xxhash)

cmake/third_party.cmake

@@ -29,6 +29,9 @@ if (APPLE)
 endif()
 
 if(WIN32 OR APPLE)
+    MESSAGE(STATUS "Disable XBYAK in Windows and MacOS")
+    SET(WITH_XBYAK OFF CACHE STRING "Disable XBYAK in Windows and MacOS" FORCE)
+
     if(WITH_LIBXSMM)
         MESSAGE(WARNING 
             "Windows, Mac are not supported with libxsmm in Paddle yet."
@@ -49,6 +52,27 @@ if(WIN32 OR APPLE)
             "Force WITH_BOX_PS=OFF")
         SET(WITH_BOX_PS OFF CACHE STRING "Disable BOX_PS package in Windows and MacOS" FORCE)
     endif()
+
+    if(WITH_PSLIB)
+        MESSAGE(WARNING
+            "Windows or Mac is not supported with PSLIB in Paddle yet."
+            "Force WITH_PSLIB=OFF")
+        SET(WITH_PSLIB OFF CACHE STRING "Disable PSLIB package in Windows and MacOS" FORCE)
+    endif()
+
+    if(WITH_LIBMCT)
+        MESSAGE(WARNING
+            "Windows or Mac is not supported with LIBMCT in Paddle yet."
+            "Force WITH_LIBMCT=OFF")
+        SET(WITH_LIBMCT OFF CACHE STRING "Disable LIBMCT package in Windows and MacOS" FORCE)
+    endif()
+
+    if(WITH_PSLIB_BRPC)
+        MESSAGE(WARNING
+            "Windows or Mac is not supported with PSLIB_BRPC in Paddle yet."
+            "Force WITH_PSLIB_BRPC=OFF")
+        SET(WITH_PSLIB_BRPC OFF CACHE STRING "Disable PSLIB_BRPC package in Windows and MacOS" FORCE)
+    endif()
 endif()
 
 set(WITH_MKLML ${WITH_MKL})
@@ -75,18 +99,19 @@ include(external/threadpool)# download threadpool
 include(external/dlpack)    # download dlpack
 include(external/xxhash)    # download, build, install xxhash
 include(external/warpctc)   # download, build, install warpctc
-include(external/rocprim)   # download, build, install rocprim
-include(external/xbyak)     # download, build, install xbyak
 
-set(third_party_deps eigen3 gflags glog boost xxhash zlib dlpack warpctc simple_threadpool)
+set(third_party_deps)
+list(APPEND third_party_deps extern_eigen3 extern_gflags extern_glog extern_boost extern_xxhash)
+list(APPEND third_party_deps extern_zlib extern_dlpack extern_warpctc extern_threadpool)
 
 if(WITH_AMD_GPU)
-    list(APPEND third_party_deps rocprim)
+    include(external/rocprim)   # download, build, install rocprim
+    list(APPEND third_party_deps extern_rocprim)
 endif()
 
 if(WITH_MKLML)
     include(external/mklml)     # download, install mklml package
-    list(APPEND third_party_deps mklml)
+    list(APPEND third_party_deps extern_mklml)
 endif()
 include(external/openblas)      # find first, then download, build, install openblas
 if(NOT CBLAS_FOUND)
@@ -95,7 +120,7 @@ endif()
 
 if(WITH_MKLDNN)
     include(external/mkldnn)    # download, build, install mkldnn
-    list(APPEND third_party_deps mkldnn_shared_lib)
+    list(APPEND third_party_deps extern_mkldnn)
 endif()
 
 include(external/protobuf)  # find first, then download, build, install protobuf
@@ -106,7 +131,7 @@ endif()
 if(WITH_PYTHON)
     include(external/python)    # find python and python_module
     include(external/pybind11)  # download pybind11
-    list(APPEND third_party_deps pybind)
+    list(APPEND third_party_deps extern_pybind)
 endif()
 
 IF(WITH_TESTING OR (WITH_DISTRIBUTE AND NOT WITH_GRPC))
@@ -116,34 +141,41 @@ ENDIF()
 
 if(WITH_GPU)
     include(external/cub)       # download cub
-    list(APPEND third_party_deps cub)
+    list(APPEND third_party_deps extern_cub)
 endif(WITH_GPU)
 
 if(WITH_PSLIB)
-    include(external/libmct)
-    include(external/pslib)
-    include(external/pslib_brpc)
+    include(external/pslib)          # download, build, install pslib
+    list(APPEND third_party_deps extern_pslib)
+    if(WITH_LIBMCT)
+        include(external/libmct)     # download, build, install libmct
+        list(APPEND third_party_deps extern_libxsmm)
+    endif()
+    if(WITH_PSLIB_BRPC)
+        include(external/pslib_brpc) # download, build, install pslib_brpc
+        list(APPEND third_party_deps extern_pslib_brpc)
+    endif()
 endif(WITH_PSLIB)
 
 if(WITH_BOX_PS)
     include(external/box_ps)
-    list(APPEND third_party_deps box_ps)
+    list(APPEND third_party_deps extern_box_ps)
 endif(WITH_BOX_PS)
 
 if(WITH_DISTRIBUTE)
-    list(APPEND third_party_deps cares)
+    list(APPEND third_party_deps extern_cares)
     if(WITH_GRPC)
-        list(APPEND third_party_deps grpc++_unsecure)
+        list(APPEND third_party_deps extern_grpc)
     else()
-        list(APPEND third_party_deps leveldb)
-        list(APPEND third_party_deps brpc)
+        list(APPEND third_party_deps extern_leveldb)
+        list(APPEND third_party_deps extern_brpc)
     endif()
 endif()
 
 if(WITH_NGRAPH)
     if(WITH_MKLDNN)
         include(external/ngraph)    # download, build, install nGraph
-        list(APPEND third_party_deps ngraph)
+        list(APPEND third_party_deps extern_ngraph)
     else()
         MESSAGE(WARNING
             "nGraph needs mkl-dnn to be enabled."
@@ -153,19 +185,20 @@ if(WITH_NGRAPH)
 endif()
 
 if(WITH_XBYAK)
-    list(APPEND third_party_deps xbyak)
+    include(external/xbyak)     # download, build, install xbyak
+    list(APPEND third_party_deps extern_xbyak)
 endif()
 
 if(WITH_LIBXSMM)
     include(external/libxsmm)   # download, build, install libxsmm
-    list(APPEND third_party_deps libxsmm)
+    list(APPEND third_party_deps extern_libxsmm)
 endif()
 
 if(WITH_DGC)
     message(STATUS "add dgc lib.")
     include(external/dgc)       # download, build, install dgc
     add_definitions(-DPADDLE_WITH_DGC)
-    list(APPEND third_party_deps dgc)
+    list(APPEND third_party_deps extern_dgc)
 endif()
 
 add_custom_target(third_party DEPENDS ${third_party_deps})