update compilation with C++14 (#31815 )

* update compilation with C++14, test=develop * fix compilation error in eigen, test=develop
fix whl package push pypi (#31585 )
1506 changed files with 58471 additions and 13872 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License

-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.15)
+cmake_policy(VERSION 3.10)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 set(PADDLE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(PADDLE_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
@ -31,13 +32,12 @@ option(WITH_GPU         "Compile PaddlePaddle with NVIDIA GPU"          ${CUDA_F
 option(WITH_TENSORRT    "Compile PaddlePaddle with NVIDIA TensorRT"     OFF)
 option(WITH_XPU         "Compile PaddlePaddle with BAIDU KUNLUN XPU"    OFF)
 option(WITH_WIN_DUMP_DBG "Compile with windows core dump debug mode"    OFF)
+option(WITH_ASCEND         "Compile PaddlePaddle with ASCEND"        OFF)
 if (WITH_GPU  AND WITH_XPU)
    message(FATAL_ERROR "Error when compile GPU and XPU at the same time")
 endif()
-# cmake 3.12, 3.13, 3.14 will append gcc link options to nvcc, and nvcc doesn't recognize them.
-if(WITH_GPU AND (${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.12) AND (${CMAKE_VERSION} VERSION_LESS 3.15))
-    message(FATAL_ERROR "cmake ${CMAKE_VERSION} is not supported when WITH_GPU=ON because of bug https://cmake.org/pipermail/cmake/2018-September/068195.html. "
-       "You can use cmake 3.16 (recommended), 3.10, 3.11, 3.15 or 3.17. Please refer to the install document: https://cmake.org/install/")
+if (WITH_GPU  AND WITH_ASCEND)
+    message(FATAL_ERROR "Error when compile GPU and ASCEND at the same time")
 endif()

 if(WITH_GPU AND NOT APPLE)
@ -57,18 +57,31 @@ if(WITH_MUSL)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations -Wno-deprecated-declarations -Wno-error=pessimizing-move -Wno-error=deprecated-copy")
 endif()

+
 if(WIN32)
    option(MSVC_STATIC_CRT "use static C Runtime library by default" ON)

    set(CMAKE_SUPPRESS_REGENERATION ON)
    set(CMAKE_STATIC_LIBRARY_PREFIX lib)

+    set(CMAKE_C_FLAGS_DEBUG   "${CMAKE_C_FLAGS_DEBUG} /bigobj")
+    set(CMAKE_C_FLAGS_RELEASE  "${CMAKE_C_FLAGS_RELEASE} /bigobj")
+    set(CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} /bigobj")
+    set(CMAKE_CXX_FLAGS_RELEASE   "${CMAKE_CXX_FLAGS_RELEASE} /bigobj")
+
+    if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
+        set(CMAKE_C_FLAGS_DEBUG   "${CMAKE_C_FLAGS_DEBUG} /Zc:inline")
+        set(CMAKE_C_FLAGS_RELEASE  "${CMAKE_C_FLAGS_RELEASE} /Zc:inline")
+        set(CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} /Zc:inline")
+        set(CMAKE_CXX_FLAGS_RELEASE   "${CMAKE_CXX_FLAGS_RELEASE} /Zc:inline")
+    endif()
+
    if (MSVC_STATIC_CRT)
        message(STATUS "Use static C runtime time, refer to https://docs.microsoft.com/en-us/cpp/c-runtime-library/crt-library-features?view=vs-2019")
-        set(CMAKE_C_FLAGS_DEBUG   "${CMAKE_C_FLAGS_DEBUG} /bigobj /MTd")
-        set(CMAKE_C_FLAGS_RELEASE  "${CMAKE_C_FLAGS_RELEASE} /bigobj /MT")
-        set(CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} /bigobj /MTd")
-        set(CMAKE_CXX_FLAGS_RELEASE   "${CMAKE_CXX_FLAGS_RELEASE} /bigobj /MT")
+        set(CMAKE_C_FLAGS_DEBUG   "${CMAKE_C_FLAGS_DEBUG} /MTd")
+        set(CMAKE_C_FLAGS_RELEASE  "${CMAKE_C_FLAGS_RELEASE} /MT")
+        set(CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
+        set(CMAKE_CXX_FLAGS_RELEASE   "${CMAKE_CXX_FLAGS_RELEASE} /MT")
        foreach(flag_var
            CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
            CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO
@ -79,7 +92,9 @@ if(WIN32)
            endif()
        endforeach(flag_var)
    endif()
-
+    
+    # NOTE(Avin0323): Less parallel count result in faster compilation.
+    math(EXPR PROCESS_MAX "${CPU_CORES} * 2 / 3")
    # windows build turn off warnings, use parallel compiling.
    foreach(flag_var
        CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
@ -87,13 +102,7 @@ if(WIN32)
        CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
        CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO)
        string(REGEX REPLACE "/W[1-4]" " /W0 " ${flag_var} "${${flag_var}}")
-        # NOTE(Avin0323): Less parallel count result in faster compilation with
-        # Unity Build on GPU.
-        if(WITH_UNITY_BUILD AND WITH_GPU)
-            set(${flag_var} "${${flag_var}} /MP8")
-        else()
-            set(${flag_var} "${${flag_var}} /MP")
-        endif()
+        set(${flag_var} "${${flag_var}} /MP${PROCESS_MAX}")
    endforeach(flag_var)
    foreach(flag_var CMAKE_CXX_FLAGS CMAKE_C_FLAGS)
        set(${flag_var} "${${flag_var}} /w")
@ -111,6 +120,10 @@ if(WIN32)
    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4068 /wd4129 /wd4244 /wd4267 /wd4297 /wd4530 /wd4577 /wd4819 /wd4838")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4068 /wd4129 /wd4244 /wd4267 /wd4297 /wd4530 /wd4577 /wd4819 /wd4838")

+    foreach(flag_var CMAKE_SHARED_LINKER_FLAGS CMAKE_STATIC_LINKER_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_LINKER_FLAGS)
+        set(${flag_var} "${${flag_var}} /ignore:4049 /ignore:4217 /ignore:4006 /ignore:4221")
+    endforeach(flag_var)
+
    if (WITH_WIN_DUMP_DBG)
        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Zi")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi")
@ -148,8 +161,9 @@ option(WITH_DISTRIBUTE  "Compile with distributed support"              OFF)
 option(WITH_BRPC_RDMA     "Use brpc rdma as the rpc protocal"           OFF)
 option(ON_INFER         "Turn on inference optimization and inference-lib generation" OFF)
 ################################ Internal Configurations #######################################
-option(WITH_ROCM_PLATFORM     "Compile PaddlePaddle with ROCM platform"             OFF)
-option(WITH_NV_JETSON     "Compile PaddlePaddle with NV JETSON"             OFF)
+option(WITH_ROCM        "Compile PaddlePaddle with ROCM platform"         OFF)
+option(WITH_RCCL        "Compile PaddlePaddle with RCCL support"          OFF)
+option(WITH_NV_JETSON   "Compile PaddlePaddle with NV JETSON"             OFF)
 option(WITH_PROFILER    "Compile PaddlePaddle with GPU profiler and gperftools"        OFF)
 option(WITH_COVERAGE    "Compile PaddlePaddle with code coverage"       OFF)
 option(WITH_INCREMENTAL_COVERAGE    "Generate coverage reports only for incremental code"       OFF)
@ -278,19 +292,25 @@ include(configure)          # add paddle env configuration

 include_directories("${PADDLE_SOURCE_DIR}")

-if(NOT DEFINED ENV{ROCM_PATH})
-    set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCm has been installed")
-    set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed")
-else()
-    set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCm has been installed")
-    set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed")
+if(WITH_ROCM)
+    include(hip)
+endif(WITH_ROCM)
+
+if (NOT WITH_ROCM AND WITH_RCCL)
+    MESSAGE(WARNING
+        "Disable RCCL when compiling without GPU. Force WITH_RCCL=OFF.")
+    set(WITH_NCCL OFF CACHE STRING
+        "Disable RCCL when compiling without GPU" FORCE)
 endif()
-set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})

-if(WITH_ROCM_PLATFORM)
-    find_package(HIP)
-    include(hip)
-endif(WITH_ROCM_PLATFORM)
+if(WITH_RCCL)
+     add_definitions("-DPADDLE_WITH_RCCL")
+     include(rccl)
+else()
+     if(WITH_ROCM)
+         MESSAGE(WARNING "If the environment is multi-card, the WITH_RCCL option needs to be turned on, otherwise only a single card can be used.")
+     endif()
+endif()

 if(WITH_NV_JETSON)
    set(WITH_ARM ON CACHE STRING "Set WITH_ARM=ON when compiling WITH_NV_JETSON=ON." FORCE)
@ -323,6 +343,8 @@ set(PADDLE_PYTHON_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/python/build")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG")
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG")

+add_definitions(-DPADDLE_DLL_EXPORT)
+
 if(ON_INFER)
    # you can trun off the paddle fluid and inference lib by set ON_INFER=OFF
    message(STATUS "On inference mode, will take place some specific optimization.")
--- a/README.md
+++ b/README.md
@ -8,8 +8,8 @@
 English | [简体中文](./README_cn.md)

 [![Build Status](https://travis-ci.org/PaddlePaddle/Paddle.svg?branch=develop)](https://travis-ci.org/PaddlePaddle/Paddle)
-[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://www.paddlepaddle.org.cn/documentation/docs/en/1.8/beginners_guide/index_en.html)
-[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://www.paddlepaddle.org.cn/documentation/docs/zh/1.8/beginners_guide/index_cn.html)
+[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)
+[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](https://paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)
 [![Release](https://img.shields.io/github/release/PaddlePaddle/Paddle.svg)](https://github.com/PaddlePaddle/Paddle/releases)
 [![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE)

@ -22,23 +22,21 @@ PaddlePaddle is originated from industrial practices with dedication and commitm

 ## Installation

-### Latest PaddlePaddle Release: [v1.8](https://github.com/PaddlePaddle/Paddle/tree/release/1.8)
+### Latest PaddlePaddle Release: [v2.0](https://github.com/PaddlePaddle/Paddle/tree/release/2.0)

 Our vision is to enable deep learning for everyone via PaddlePaddle.
 Please refer to our [release announcement](https://github.com/PaddlePaddle/Paddle/releases) to track the latest features of PaddlePaddle.
 ### Install Latest Stable Release:
 ```
-# Linux CPU
+# CPU
 pip install paddlepaddle
-# Linux GPU cuda10cudnn7
+# GPU
 pip install paddlepaddle-gpu
-# Linux GPU cuda9cudnn7
-pip install paddlepaddle-gpu==1.8.5.post97

 ```
-It is recommended to read [this doc](https://www.paddlepaddle.org.cn/documentation/docs/en/beginners_guide/install/index_en.html) on our website.
+More infomation about installation, please view [Quick Install](https://www.paddlepaddle.org.cn/install/quick)

-Now our developers can acquire Tesla V100 online computing resources for free. If you create a program by AI Studio, you will obtain 12 hours to train models online per day. If you can insist on that for five consecutive days, then you will receive an extra 48 hours. [Click here to start](http://ai.baidu.com/support/news?action=detail&id=981).
+Now our developers can acquire Tesla V100 online computing resources for free. If you create a program by AI Studio, you will obtain 10 hours to train models online per day. [Click here to start](https://aistudio.baidu.com/aistudio/index).

 ## FOUR LEADING TECHNOLOGIES

@ -67,38 +65,30 @@ Now our developers can acquire Tesla V100 online computing resources for free. I

 ## Documentation

-We provide [English](http://www.paddlepaddle.org.cn/documentation/docs/en/1.8/beginners_guide/index_en.html) and
-[Chinese](http://www.paddlepaddle.org.cn/documentation/docs/zh/1.8/beginners_guide/index_cn.html) documentation.
+We provide [English](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html) and
+[Chinese](https://www.paddlepaddle.org.cn/documentation/docs/zh/guide/index_cn.html) documentation.

- [Basic Deep Learning Models](https://www.paddlepaddle.org.cn/documentation/docs/en/beginners_guide/index_en.html)
+- [Guides](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)

  You might want to start from how to implement deep learning basics with PaddlePaddle.

-
- [User Guides](https://www.paddlepaddle.org.cn/documentation/docs/en/user_guides/index_en.html)
-
-  You might have got the hang of Beginner’s Guide, and wish to model practical problems and build your original networks.
-  
-  
- [Advanced User Guides](https://www.paddlepaddle.org.cn/documentation/docs/en/advanced_guide/index_en.html)
+- [Practice](https://www.paddlepaddle.org.cn/documentation/docs/zh/tutorial/index_cn.html)

  So far you have already been familiar with Fluid. And the next step should be building a more efficient model or inventing your original Operator. 

-
 - [API Reference](https://www.paddlepaddle.org.cn/documentation/docs/en/api/index_en.html)

   Our new API enables much shorter programs.

-
- [How to Contribute](https://www.paddlepaddle.org.cn/documentation/docs/en/advanced_guide/addon_development/contribute_code/index_en.html)
+- [How to Contribute](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/08_contribution/index_en.html)

   We appreciate your contributions!

 ## Communication

 - [Github Issues](https://github.com/PaddlePaddle/Paddle/issues): bug reports, feature requests, install issues, usage issues, etc.
- QQ discussion group: 796771754 (PaddlePaddle).
- [Forums](http://ai.baidu.com/forum/topic/list/168?pageNo=1): discuss implementations, research, etc.
+- QQ discussion group: 778260830 (PaddlePaddle).
+- [Forums](https://ai.baidu.com/forum/topic/list/168?pageNo=1): discuss implementations, research, etc.

 ## Copyright and License
 PaddlePaddle is provided under the [Apache-2.0 license](LICENSE).
--- a/README_cn.md
+++ b/README_cn.md
@ -8,8 +8,8 @@
 [English](./README.md) | 简体中文

 [![Build Status](https://travis-ci.org/PaddlePaddle/Paddle.svg?branch=develop)](https://travis-ci.org/PaddlePaddle/Paddle)
-[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://www.paddlepaddle.org.cn/documentation/docs/en/1.8/beginners_guide/index_en.html)
-[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://www.paddlepaddle.org.cn/documentation/docs/zh/1.8/beginners_guide/index_cn.html)
+[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)
+[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](https://paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)
 [![Release](https://img.shields.io/github/release/PaddlePaddle/Paddle.svg)](https://github.com/PaddlePaddle/Paddle/releases)
 [![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE)

@ -19,23 +19,20 @@

 ## 安装

-### PaddlePaddle最新版本: [v1.8](https://github.com/PaddlePaddle/Paddle/tree/release/1.8)
+### PaddlePaddle最新版本: [v2.0](https://github.com/PaddlePaddle/Paddle/tree/release/2.0)

 跟进PaddlePaddle最新特性请参考我们的[版本说明](https://github.com/PaddlePaddle/Paddle/releases)

 ### 安装最新稳定版本:
 ```
-# Linux CPU
+# CPU
 pip install paddlepaddle
-# Linux GPU cuda10cudnn7
+# GPU
 pip install paddlepaddle-gpu
-# Linux GPU cuda9cudnn7
-pip install paddlepaddle-gpu==1.8.5.post97
-
 ```
 更多安装信息详见官网 [安装说明](https://www.paddlepaddle.org.cn/install/quick)

-PaddlePaddle用户可领取**免费Tesla V100在线算力资源**，训练模型更高效。**每日登陆即送12小时**，**连续五天运行再加送48小时**，[前往使用免费算力](https://ai.baidu.com/support/news?action=detail&id=981)。
+PaddlePaddle用户可领取**免费Tesla V100在线算力资源**，训练模型更高效。**每日登陆即送10小时**，[前往使用免费算力](https://aistudio.baidu.com/aistudio/index)。

 ## 四大领先技术

@ -64,38 +61,30 @@ PaddlePaddle用户可领取**免费Tesla V100在线算力资源**，训练模型

 ## 文档

-我们提供 [英文](http://www.paddlepaddle.org.cn/documentation/docs/en/1.8/beginners_guide/index_en.html) 和
-[中文](http://www.paddlepaddle.org.cn/documentation/docs/zh/1.8/beginners_guide/index_cn.html) 文档
+我们提供 [英文](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html) 和
+[中文](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html) 文档

- [深度学习基础教程](https://www.paddlepaddle.org.cn/documentation/docs/zh/beginners_guide/index_cn.html)
+- [使用指南](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)

   或许您想从深度学习基础开始学习飞桨
  
+- [应用实践](https://www.paddlepaddle.org.cn/documentation/docs/zh/tutorial/index_cn.html)

- [典型案例](https://www.paddlepaddle.org.cn/documentation/docs/zh/user_guides/index_cn.html)
-
-   或许您已经掌握了新手入门阶段的内容，期望可以针对实际问题建模、搭建自己网络
-  
-
- [进阶指南](https://www.paddlepaddle.org.cn/documentation/docs/zh/advanced_guide/index_cn.html)
-
-   或许您已比较熟练使用PaddlePaddle来完成常规任务，期望获得更高效的模型或者定义自己的Operator
-  
  
- [API Reference](https://www.paddlepaddle.org.cn/documentation/docs/zh/api_cn/index_cn.html)
+- [API Reference](https://www.paddlepaddle.org.cn/documentation/docs/zh/api/index_cn.html)

   新的API支持代码更少更简洁的程序
   

- [贡献方式](https://www.paddlepaddle.org.cn/documentation/docs/zh/advanced_guide/addon_development/contribute_code/index_cn.html)
+- [贡献方式](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/08_contribution/index_cn.html)

   欢迎您的贡献!

 ## 交流与反馈

 - 欢迎您通过[Github Issues](https://github.com/PaddlePaddle/Paddle/issues)来提交问题、报告与建议
- QQ群: 796771754 (PaddlePaddle)
- [论坛](http://ai.baidu.com/forum/topic/list/168): 欢迎大家在PaddlePaddle论坛分享在使用PaddlePaddle中遇到的问题和经验, 营造良好的论坛氛围
+- QQ群: 778260830 (PaddlePaddle)
+- [论坛](https://ai.baidu.com/forum/topic/list/168): 欢迎大家在PaddlePaddle论坛分享在使用PaddlePaddle中遇到的问题和经验, 营造良好的论坛氛围

 ## 版权和许可证
 PaddlePaddle由[Apache-2.0 license](LICENSE)提供
--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@ -78,6 +78,10 @@ if(WITH_BOX_PS)
    add_definitions(-DPADDLE_WITH_BOX_PS)
 endif()

+if(WITH_ASCEND)
+    add_definitions(-DPADDLE_WITH_ASCEND)
+endif()
+
 if(WITH_XPU)
    message(STATUS "Compile with XPU!")
    add_definitions(-DPADDLE_WITH_XPU)
@ -126,14 +130,10 @@ if(WITH_GPU)
        endif()
        include_directories(${TENSORRT_INCLUDE_DIR})
    endif()
-elseif(WITH_ROCM_PLATFORM)
+elseif(WITH_ROCM)
    add_definitions(-DPADDLE_WITH_HIP)
+    add_definitions(-DEIGEN_USE_GPU)
    add_definitions(-DEIGEN_USE_HIP)
-    add_definitions(-D__HIP_PLATFORM_HCC__)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__HIP_PLATFORM_HCC__")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__HIP_PLATFORM_HCC__")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP")
-    set(THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_HIP)
 else()
    add_definitions(-DHPPL_STUB_FUNC)
    list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu)
--- a/cmake/coveralls.cmake
+++ b/cmake/coveralls.cmake
@ -63,7 +63,10 @@ endfunction()

 if(WITH_COVERAGE)
    if (WITH_INCREMENTAL_COVERAGE)
-        if (NOT ("$ENV{PADDLE_GIT_DIFF_H_FILE}" STREQUAL ""))
+        # if *.h changed, generate coverage report totaly.
+        # if pybind.cc changed, generate coverage report totaly.
+        # Because if pybind.cc add '-g -O0 -fprofile-arcs -ftest-coverage' only, some testcase will fail.
+        if ( (NOT ("$ENV{PADDLE_GIT_DIFF_H_FILE}" STREQUAL "")) OR ("$ENV{PADDLE_GIT_DIFF_CC_FILE}" MATCHES "pybind.cc") )
            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage")
            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage")
        endif()
--- a/cmake/cuda.cmake
+++ b/cmake/cuda.cmake
@ -74,7 +74,7 @@ endfunction()
 #   select_nvcc_arch_flags(out_variable)
 function(select_nvcc_arch_flags out_variable)
  # List of arch names
-  set(archs_names "Kepler" "Maxwell" "Pascal" "Volta" "Turing" "All" "Manual")
+  set(archs_names "Kepler" "Maxwell" "Pascal" "Volta" "Turing" "Ampere" "All" "Manual")
  set(archs_name_default "Auto")
  list(APPEND archs_names "Auto")

@ -91,7 +91,7 @@ function(select_nvcc_arch_flags out_variable)

  if(${CUDA_ARCH_NAME} STREQUAL "Manual")
    set(CUDA_ARCH_BIN ${paddle_known_gpu_archs} CACHE STRING "Specify 'real' GPU architectures to build binaries for, BIN(PTX) format is supported")
-    set(CUDA_ARCH_PTX "50"                     CACHE STRING "Specify 'virtual' PTX architectures to build PTX intermediate code for")
+    set(CUDA_ARCH_PTX ""                        CACHE STRING "Specify 'virtual' PTX architectures to build PTX intermediate code for")
    mark_as_advanced(CUDA_ARCH_BIN CUDA_ARCH_PTX)
  else()
    unset(CUDA_ARCH_BIN CACHE)
@ -108,6 +108,8 @@ function(select_nvcc_arch_flags out_variable)
    set(cuda_arch_bin "70")
  elseif(${CUDA_ARCH_NAME} STREQUAL "Turing")
    set(cuda_arch_bin "75")
+  elseif(${CUDA_ARCH_NAME} STREQUAL "Ampere")
+    set(cuda_arch_bin "80")
  elseif(${CUDA_ARCH_NAME} STREQUAL "All")
    set(cuda_arch_bin ${paddle_known_gpu_archs})
  elseif(${CUDA_ARCH_NAME} STREQUAL "Auto")
@ -175,14 +177,22 @@ elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 10.0) # CUDA 9.x
  set(paddle_known_gpu_archs ${paddle_known_gpu_archs9})
  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_MWAITXINTRIN_H_INCLUDED")
  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__STRICT_ANSI__")
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
 elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) # CUDA 10.x
  set(paddle_known_gpu_archs ${paddle_known_gpu_archs10})
  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_MWAITXINTRIN_H_INCLUDED")
  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__STRICT_ANSI__")
-elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0) # CUDA 11.x
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.2) # CUDA 11.0/11.1
  set(paddle_known_gpu_archs ${paddle_known_gpu_archs11})
  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_MWAITXINTRIN_H_INCLUDED")
  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__STRICT_ANSI__")
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0) # CUDA 11.2+
+  set(paddle_known_gpu_archs "${paddle_known_gpu_archs11} 86")
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_MWAITXINTRIN_H_INCLUDED")
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__STRICT_ANSI__")
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
 endif()

 if (NOT ${CMAKE_CUDA_COMPILER_VERSION} LESS 10.0)
@ -198,14 +208,11 @@ select_nvcc_arch_flags(NVCC_FLAGS_EXTRA)
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${NVCC_FLAGS_EXTRA}")
 message(STATUS "NVCC_FLAGS_EXTRA: ${NVCC_FLAGS_EXTRA}")

-# Set C++11 support
+# Set C++14 support
 set(CUDA_PROPAGATE_HOST_FLAGS OFF)
 # Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc.
 # So, don't set these flags here.
-if (NOT WIN32) # windows msvc2015 support c++11 natively.
-    # -std=c++11 -fPIC not recoginize by msvc, -Xcompiler will be added by cmake.
-  set(CMAKE_CUDA_STANDARD 11)
-endif(NOT WIN32)
+set(CMAKE_CUDA_STANDARD 14)

 # (Note) For windows, if delete /W[1-4], /W1 will be added defaultly and conflic with -w
 # So replace /W[1-4] with /W0
@ -216,6 +223,8 @@ endif(WIN32)
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -w")
 # Set :expt-relaxed-constexpr to suppress Eigen warnings
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr")
+# Set :expt-extended-lambda to enable HOSTDEVICE annotation on lambdas
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda")

 if(WIN32)
  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler \"/wd4244 /wd4267 /wd4819 \"")
--- a/cmake/cudnn.cmake
+++ b/cmake/cudnn.cmake
@ -94,7 +94,7 @@ macro(find_cudnn_version cudnn_header_file)
                "${CUDNN_MAJOR_VERSION} * 1000 +
                 ${CUDNN_MINOR_VERSION} * 100 + ${CUDNN_PATCHLEVEL_VERSION}")
            message(STATUS "Current cuDNN header is ${cudnn_header_file} "
-              "Current cuDNN version is v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}. ")
+              "Current cuDNN version is v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}.${CUDNN_PATCHLEVEL_VERSION}. ")
        endif()
    endif()
 endmacro()
--- a/cmake/external/ascend.cmake
+++ b/cmake/external/ascend.cmake
@ -0,0 +1,61 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+INCLUDE(ExternalProject)
+
+SET(ASCEND_PROJECT       "extern_ascend")
+IF((NOT DEFINED ASCEND_VER) OR (NOT DEFINED ASCEND_URL))
+  MESSAGE(STATUS "use pre defined download url")
+  SET(ASCEND_VER "0.1.1" CACHE STRING "" FORCE)
+  SET(ASCEND_NAME "ascend" CACHE STRING "" FORCE)
+  SET(ASCEND_URL "http://paddle-ascend.bj.bcebos.com/ascend.tar.gz" CACHE STRING "" FORCE)
+ENDIF()
+MESSAGE(STATUS "ASCEND_NAME: ${ASCEND_NAME}, ASCEND_URL: ${ASCEND_URL}")
+SET(ASCEND_SOURCE_DIR    "${THIRD_PARTY_PATH}/ascend")
+SET(ASCEND_DOWNLOAD_DIR  "${ASCEND_SOURCE_DIR}/src/${ASCEND_PROJECT}")
+SET(ASCEND_DST_DIR       "ascend")
+SET(ASCEND_INSTALL_ROOT  "${THIRD_PARTY_PATH}/install")
+SET(ASCEND_INSTALL_DIR   ${ASCEND_INSTALL_ROOT}/${ASCEND_DST_DIR})
+SET(ASCEND_ROOT          ${ASCEND_INSTALL_DIR})
+SET(ASCEND_INC_DIR       ${ASCEND_ROOT}/include)
+SET(ASCEND_LIB_DIR       ${ASCEND_ROOT}/lib)
+SET(ASCEND_LIB           ${ASCEND_LIB_DIR}/libge_runner.so)
+SET(ASCEND_GRAPH_LIB           ${ASCEND_LIB_DIR}/libgraph.so)
+SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${ASCEND_ROOT}/lib")
+
+INCLUDE_DIRECTORIES(${ASCEND_INC_DIR})
+FILE(WRITE ${ASCEND_DOWNLOAD_DIR}/CMakeLists.txt
+  "PROJECT(ASCEND)\n"
+  "cmake_minimum_required(VERSION 3.0)\n"
+  "install(DIRECTORY ${ASCEND_NAME}/include ${ASCEND_NAME}/lib \n"
+  "        DESTINATION ${ASCEND_DST_DIR})\n")
+ExternalProject_Add(
+    ${ASCEND_PROJECT}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    PREFIX                ${ASCEND_SOURCE_DIR}
+    DOWNLOAD_DIR          ${ASCEND_DOWNLOAD_DIR}
+    DOWNLOAD_COMMAND      wget --no-check-certificate ${ASCEND_URL} -c -q -O ${ASCEND_NAME}.tar.gz
+                          && tar zxvf ${ASCEND_NAME}.tar.gz
+    DOWNLOAD_NO_PROGRESS  1
+    UPDATE_COMMAND        ""
+    CMAKE_ARGS            -DCMAKE_INSTALL_PREFIX=${ASCEND_INSTALL_ROOT}
+    CMAKE_CACHE_ARGS      -DCMAKE_INSTALL_PREFIX:PATH=${ASCEND_INSTALL_ROOT}
+)
+ADD_LIBRARY(ascend SHARED IMPORTED GLOBAL)
+SET_PROPERTY(TARGET ascend PROPERTY IMPORTED_LOCATION ${ASCEND_LIB})
+
+ADD_LIBRARY(ascend_graph SHARED IMPORTED GLOBAL)
+SET_PROPERTY(TARGET ascend_graph PROPERTY IMPORTED_LOCATION ${ASCEND_GRAPH_LIB})
+ADD_DEPENDENCIES(ascend ascend_graph ${ASCEND_PROJECT})
+
--- a/cmake/external/cryptopp.cmake
+++ b/cmake/external/cryptopp.cmake
@ -22,6 +22,13 @@ SET(CRYPTOPP_TAG        CRYPTOPP_8_2_0)

 IF(WIN32)
  SET(CRYPTOPP_LIBRARIES "${CRYPTOPP_INSTALL_DIR}/lib/cryptopp-static.lib" CACHE FILEPATH "cryptopp library." FORCE)
+  # There is a compilation parameter 'FI\"winapifamily.h\"' can't be used correctly
+  # with Ninja on Windows. The only difference between the patch file and original
+  # file is that the compilation parameters are changed to 'FIwinapifamily.h'. This
+  # patch command can be removed when upgrading to a higher version.
+  if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
+    set(CRYPTOPP_PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different "${PADDLE_SOURCE_DIR}/patches/cryptopp/CMakeLists.txt" "<SOURCE_DIR>/")
+  endif()
 ELSE(WIN32)
  SET(CRYPTOPP_LIBRARIES "${CRYPTOPP_INSTALL_DIR}/lib/libcryptopp.a" CACHE FILEPATH "cryptopp library." FORCE)
 ENDIF(WIN32)
@ -53,11 +60,13 @@ ExternalProject_Add(
    "${CRYPTOPP_DOWNLOAD_CMD}"
    PREFIX          ${CRYPTOPP_PREFIX_DIR}
    SOURCE_DIR      ${CRYPTOPP_SOURCE_DIR}
+    UPDATE_COMMAND  ""
    PATCH_COMMAND
    COMMAND ${CMAKE_COMMAND} -E remove_directory "<SOURCE_DIR>/cmake/"
    COMMAND git clone ${GIT_URL}/noloader/cryptopp-cmake "<SOURCE_DIR>/cmake"
    COMMAND cd "<SOURCE_DIR>/cmake" && git checkout tags/${CRYPTOPP_TAG} -b ${CRYPTOPP_TAG}
    COMMAND ${CMAKE_COMMAND} -E copy_directory "<SOURCE_DIR>/cmake/" "<SOURCE_DIR>/"
+    COMMAND ${CRYPTOPP_PATCH_COMMAND}
    INSTALL_DIR     ${CRYPTOPP_INSTALL_DIR}
    CMAKE_ARGS ${CRYPTOPP_CMAKE_ARGS}
    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${CRYPTOPP_INSTALL_DIR}
--- a/cmake/external/eigen.cmake
+++ b/cmake/external/eigen.cmake
@ -20,28 +20,28 @@ set(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3/src/extern_eigen3)
 set(EIGEN_REPOSITORY https://gitlab.com/libeigen/eigen.git)
 set(EIGEN_TAG        4da2c6b1974827b1999bab652a3d4703e1992d26)

-# the recent version of eigen will cause compilation error on windows
-if(WIN32)
-    set(EIGEN_REPOSITORY ${GIT_URL}/eigenteam/eigen-git-mirror.git)
-    set(EIGEN_TAG        917060c364181f33a735dc023818d5a54f60e54c)
-endif()
-
-# eigen on cuda9.1 missing header of math_funtions.hpp
-# https://stackoverflow.com/questions/43113508/math-functions-hpp-not-found-when-using-cuda-with-eigen
-if(WITH_ROCM_PLATFORM)
-    set(EIGEN_REPOSITORY ${GIT_URL}/sabreshao/hipeigen.git)
-    set(EIGEN_TAG        7cb2b6e5a4b4a1efe658abb215cd866c6fb2275e)
-endif()
-
 cache_third_party(extern_eigen3
    REPOSITORY    ${EIGEN_REPOSITORY}
    TAG           ${EIGEN_TAG}
    DIR           EIGEN_SOURCE_DIR)

 if(WIN32)
+    add_definitions(-DEIGEN_STRONG_INLINE=inline)
    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Half.h native_src)
    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/CUDA/Half.h native_dst)
-    set(EIGEN_PATCH_COMMAND copy ${native_src} ${native_dst} /Y)
+    # For Windows
+    # which will cause a compilation error in Tensor:74:
+    # "can not open file 'unistd.h'"
+    # so use following patch to solve compilation error On Windows.
+    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Tensor native_src2)
+    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/unsupported/Eigen/CXX11/Tensor native_dst2)
+    # For VS2015
+    # which will cause a compilation error in TensorBlock.h:1028:
+    # "syntax error"
+    # so use following patch to solve compilation error On Windows.
+    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/TensorBlock.h native_src3)
+    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorBlock.h native_dst3)
+    set(EIGEN_PATCH_COMMAND copy ${native_src} ${native_dst} /Y && copy ${native_src2} ${native_dst2} /Y && copy ${native_src3} ${native_dst3} /Y)
 elseif(LINUX)
    # For gxx=4.8, __GXX_ABI_VERSION is less than 1004
    # which will cause a compilation error in Geometry_SSE.h:38:
@ -56,43 +56,38 @@ elseif(LINUX)
    # add patch to avoid compilation error in c++11
    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/MathFunctions.h native_src2)
    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/MathFunctions.h native_dst2)
-    set(EIGEN_PATCH_COMMAND cp ${native_src1} ${native_dst1} && cp ${native_src2} ${native_dst2})
+    if(WITH_ROCM)
+        # For HIPCC Eigen::internal::device::numeric_limits is not EIGEN_DEVICE_FUNC
+        # which will cause compiler error of using __host__ funciont in __host__ __device__
+        file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Meta.h native_src3)
+        file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/util/Meta.h native_dst3)
+        # For HIPCC Eigen::internal::scalar_sum_op<bool,bool> is not EIGEN_DEVICE_FUNC
+        # which will cause compiler error of using __host__ funciont in __host__ __device__
+        file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/BinaryFunctors.h native_src4)
+        file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/functors/BinaryFunctors.h native_dst4)
+        set(EIGEN_PATCH_COMMAND cp ${native_src1} ${native_dst1} && cp ${native_src2} ${native_dst2} && cp ${native_src3} ${native_dst3} && cp ${native_src4} ${native_dst4})
+    else()
+        set(EIGEN_PATCH_COMMAND cp ${native_src1} ${native_dst1} && cp ${native_src2} ${native_dst2})
+    endif()
 endif()

 set(EIGEN_INCLUDE_DIR ${EIGEN_SOURCE_DIR})
 INCLUDE_DIRECTORIES(${EIGEN_INCLUDE_DIR})

-if(WITH_AMD_GPU)
-    ExternalProject_Add(
-        extern_eigen3
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        ${SHALLOW_CLONE}
-        "${EIGEN_DOWNLOAD_CMD}"
-        PREFIX          ${EIGEN_PREFIX_DIR}
-        SOURCE_DIR      ${EIGEN_SOURCE_DIR}
-        UPDATE_COMMAND    ""
-        PATCH_COMMAND   ${EIGEN_PATCH_COMMAND}
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND     ""
-        INSTALL_COMMAND   ""
-        TEST_COMMAND      ""
-    )
-else()
-    ExternalProject_Add(
-        extern_eigen3
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        ${SHALLOW_CLONE}
-        "${EIGEN_DOWNLOAD_CMD}"
-        PREFIX          ${EIGEN_PREFIX_DIR}
-        SOURCE_DIR      ${EIGEN_SOURCE_DIR}
-        UPDATE_COMMAND    ""
-        PATCH_COMMAND   ${EIGEN_PATCH_COMMAND}
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND     ""
-        INSTALL_COMMAND   ""
-        TEST_COMMAND      ""
-    )
-endif()
+ExternalProject_Add(
+    extern_eigen3
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    ${SHALLOW_CLONE}
+    "${EIGEN_DOWNLOAD_CMD}"
+    PREFIX          ${EIGEN_PREFIX_DIR}
+    SOURCE_DIR      ${EIGEN_SOURCE_DIR}
+    UPDATE_COMMAND    ""
+    PATCH_COMMAND   ${EIGEN_PATCH_COMMAND}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND     ""
+    INSTALL_COMMAND   ""
+    TEST_COMMAND      ""
+)

 add_library(eigen3 INTERFACE)

--- a/cmake/external/gflags.cmake
+++ b/cmake/external/gflags.cmake
@ -30,8 +30,6 @@ ENDIF(WIN32)

 INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR})

-set(GFLAGS_NAMESPACE "paddle_gflags")
-
 cache_third_party(extern_gflags
    REPOSITORY   ${GFLAGS_REPOSITORY}
    TAG          ${GFLAGS_TAG}
@ -59,7 +57,6 @@ ExternalProject_Add(
                    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
                    -DBUILD_TESTING=OFF
                    -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-                    -DGFLAGS_NAMESPACE=${GFLAGS_NAMESPACE} 
                    ${EXTERNAL_OPTIONAL_ARGS}
    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
                     -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
--- a/cmake/external/lite.cmake
+++ b/cmake/external/lite.cmake
@ -58,6 +58,7 @@ if (NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
                           -DLITE_BUILD_EXTRA=ON
                           -DLITE_WITH_XPU=${LITE_WITH_XPU}
                           -DXPU_SDK_ROOT=${XPU_SDK_ROOT}
+                           -DLITE_WITH_CODE_META_INFO=OFF
                           -DLITE_WITH_ARM=ON)
    ExternalProject_Add(
      ${LITE_PROJECT}
@ -99,6 +100,7 @@ if (NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
                           -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME}
                           -DLITE_WITH_XPU=${LITE_WITH_XPU}
                           -DXPU_SDK_ROOT=${XPU_SDK_ROOT}
+                           -DLITE_WITH_CODE_META_INFO=OFF
                           -DLITE_WITH_ARM=OFF)

    ExternalProject_Add(
--- a/cmake/external/mkldnn.cmake
+++ b/cmake/external/mkldnn.cmake
@ -20,7 +20,7 @@ SET(MKLDNN_SOURCE_DIR     ${THIRD_PARTY_PATH}/mkldnn/src/extern_mkldnn)
 SET(MKLDNN_INSTALL_DIR    ${THIRD_PARTY_PATH}/install/mkldnn)
 SET(MKLDNN_INC_DIR        "${MKLDNN_INSTALL_DIR}/include" CACHE PATH "mkldnn include directory." FORCE)
 SET(MKLDNN_REPOSITORY     ${GIT_URL}/oneapi-src/oneDNN.git)
-SET(MKLDNN_TAG            a18f78f1f058437e9efee403655d671633360f98)
+SET(MKLDNN_TAG            72efa005effb49595933e033cc732f215ef0445a)

 # Introduce variables:
 # * CMAKE_INSTALL_LIBDIR
--- a/cmake/external/protobuf.cmake
+++ b/cmake/external/protobuf.cmake
@ -250,5 +250,8 @@ IF(NOT PROTOBUF_FOUND)

    SET(PROTOBUF_PROTOC_EXECUTABLE ${extern_protobuf_PROTOC_EXECUTABLE}
        CACHE FILEPATH "protobuf executable." FORCE)
+    # `EXTERN_PROTOBUF_DEPEND` used in cmake function `proto_library` to ensure
+    # `protoc.exe` existed before calling it.
+    set(EXTERN_PROTOBUF_DEPEND extern_protobuf)
    PROMPT_PROTOBUF_LIB(extern_protobuf)
 ENDIF(NOT PROTOBUF_FOUND)
--- a/cmake/external/warpctc.cmake
+++ b/cmake/external/warpctc.cmake
@ -14,11 +14,15 @@

 INCLUDE(ExternalProject)

+IF(WITH_ROCM)
+    add_definitions(-DWARPCTC_WITH_HIP)
+ENDIF()
+
 SET(WARPCTC_PREFIX_DIR  ${THIRD_PARTY_PATH}/warpctc)
 SET(WARPCTC_SOURCE_DIR  ${THIRD_PARTY_PATH}/warpctc/src/extern_warpctc)
 SET(WARPCTC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warpctc)
 set(WARPCTC_REPOSITORY  ${GIT_URL}/baidu-research/warp-ctc.git)
-set(WARPCTC_TAG         95a461eddeabd51099ef059dcfada1117eb1bfb8)
+set(WARPCTC_TAG         c690fc5755abbdbdc98ef78d51ec10a6748a8cd1)

 SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include"
    CACHE PATH "Warp-ctc Directory" FORCE)
@ -49,14 +53,15 @@ ExternalProject_Add(
    BUILD_ALWAYS    1
    CMAKE_ARGS      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-                    -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-                    -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-                    -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-                    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-                    -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-                    -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
+                    -DCMAKE_C_FLAGS=$<FILTER:${CMAKE_C_FLAGS},EXCLUDE,/Zc:inline>
+                    -DCMAKE_C_FLAGS_DEBUG=$<FILTER:${CMAKE_C_FLAGS_DEBUG},EXCLUDE,/Zc:inline>
+                    -DCMAKE_C_FLAGS_RELEASE=$<FILTER:${CMAKE_C_FLAGS_RELEASE},EXCLUDE,/Zc:inline>
+                    -DCMAKE_CXX_FLAGS=$<FILTER:${CMAKE_CXX_FLAGS},EXCLUDE,/Zc:inline>
+                    -DCMAKE_CXX_FLAGS_RELEASE=$<FILTER:${CMAKE_CXX_FLAGS_RELEASE},EXCLUDE,/Zc:inline>
+                    -DCMAKE_CXX_FLAGS_DEBUG=$<FILTER:${CMAKE_CXX_FLAGS_DEBUG},EXCLUDE,/Zc:inline>
                    -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR}
                    -DWITH_GPU=${WITH_GPU}
+                    -DWITH_ROCM=${WITH_ROCM}
                    -DWITH_OMP=${USE_OMP}
                    -DWITH_TORCH=OFF
                    -DCMAKE_DISABLE_FIND_PACKAGE_Torch=ON
--- a/cmake/external/xbyak.cmake
+++ b/cmake/external/xbyak.cmake
@ -20,7 +20,7 @@ SET(XBYAK_SOURCE_DIR     ${THIRD_PARTY_PATH}/xbyak/src/extern_xbyak)
 set(XBYAK_INSTALL_ROOT  ${THIRD_PARTY_PATH}/install/xbyak)
 set(XBYAK_INC_DIR       ${XBYAK_INSTALL_ROOT}/include)
 set(XBYAK_REPOSITORY    ${GIT_URL}/herumi/xbyak.git)
-set(XBYAK_TAG           v5.661) # Jul 26th
+set(XBYAK_TAG           v5.81) # Dec 19, 2019

 include_directories(${XBYAK_INC_DIR})
 include_directories(${XBYAK_INC_DIR}/xbyak)
@ -44,7 +44,7 @@ ExternalProject_Add(
    DEPENDS             ""
    PREFIX              ${XBYAK_PREFIX_DIR}
    SOURCE_DIR          ${XBYAK_SOURCE_DIR}
-    UPDATE_COMMAND      ""
+    # UPDATE_COMMAND      ""
    CMAKE_ARGS          -DCMAKE_INSTALL_PREFIX=${XBYAK_INSTALL_ROOT}
    CMAKE_CACHE_ARGS    -DCMAKE_INSTALL_PREFIX:PATH=${XBYAK_INSTALL_ROOT}
 )
--- a/cmake/external/xpu.cmake
+++ b/cmake/external/xpu.cmake
@ -4,49 +4,55 @@ endif()

 INCLUDE(ExternalProject)
 SET(XPU_PROJECT                 "extern_xpu")
-
-if (WITH_AARCH64)
-    SET(XPU_URL    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/aarch64/xpu_2020_1229.tar.gz" CACHE STRING "" FORCE)
-elseif(WITH_SUNWAY)
-    SET(XPU_URL    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/sunway/xpu_2020_1227.tar.gz" CACHE STRING "" FORCE)
-else()
-    SET(XPU_URL    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/xpu_2021_01_13.tar.gz" CACHE STRING "" FORCE)
-endif()
-
-SET(XPU_SOURCE_DIR              "${THIRD_PARTY_PATH}/xpu")
-SET(XPU_DOWNLOAD_DIR            "${XPU_SOURCE_DIR}/src/${XPU_PROJECT}")
-SET(XPU_INSTALL_DIR             "${THIRD_PARTY_PATH}/install/xpu")
-SET(XPU_API_INC_DIR             "${THIRD_PARTY_PATH}/install/xpu/include")
-SET(XPU_LIB_DIR                 "${THIRD_PARTY_PATH}/install/xpu/lib")
-
 SET(XPU_API_LIB_NAME            "libxpuapi.so")
 SET(XPU_RT_LIB_NAME             "libxpurt.so")
-SET(XPU_API_LIB                 "${XPU_LIB_DIR}/${XPU_API_LIB_NAME}")
-SET(XPU_RT_LIB                  "${XPU_LIB_DIR}/${XPU_RT_LIB_NAME}")

-SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${XPU_INSTALL_DIR}/lib")
+if(NOT XPU_SDK_ROOT)
+  if (WITH_AARCH64)
+      SET(XPU_URL    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/aarch64/xpu_2021_01_13.tar.gz" CACHE STRING "" FORCE)
+  elseif(WITH_SUNWAY)
+      SET(XPU_URL    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/sunway/xpu_2021_01_13.tar.gz" CACHE STRING "" FORCE)
+  else()
+      SET(XPU_URL    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/xpu_2021_03_30.tar.gz" CACHE STRING "" FORCE)
+  endif()

-INCLUDE_DIRECTORIES(${XPU_API_INC_DIR})
+  SET(XPU_SOURCE_DIR              "${THIRD_PARTY_PATH}/xpu")
+  SET(XPU_DOWNLOAD_DIR            "${XPU_SOURCE_DIR}/src/${XPU_PROJECT}")
+  SET(XPU_INSTALL_DIR             "${THIRD_PARTY_PATH}/install/xpu")
+  SET(XPU_API_INC_DIR             "${THIRD_PARTY_PATH}/install/xpu/include")
+  SET(XPU_LIB_DIR                 "${THIRD_PARTY_PATH}/install/xpu/lib")
+
+  SET(XPU_API_LIB                 "${XPU_LIB_DIR}/${XPU_API_LIB_NAME}")
+  SET(XPU_RT_LIB                  "${XPU_LIB_DIR}/${XPU_RT_LIB_NAME}")

-FILE(WRITE ${XPU_DOWNLOAD_DIR}/CMakeLists.txt
-  "PROJECT(XPU)\n"
-  "cmake_minimum_required(VERSION 3.0)\n"
-  "install(DIRECTORY xpu/include xpu/lib \n"
-  "        DESTINATION ${XPU_INSTALL_DIR})\n")
+  SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${XPU_INSTALL_DIR}/lib")

-ExternalProject_Add(
-    ${XPU_PROJECT}
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    PREFIX                ${XPU_SOURCE_DIR}
-    DOWNLOAD_DIR          ${XPU_DOWNLOAD_DIR}
-    DOWNLOAD_COMMAND      wget --no-check-certificate ${XPU_URL} -c -q -O xpu.tar.gz
-                          && tar xvf xpu.tar.gz
-    DOWNLOAD_NO_PROGRESS  1
-    UPDATE_COMMAND        ""
-    CMAKE_ARGS            -DCMAKE_INSTALL_PREFIX=${XPU_INSTALL_ROOT}
-    CMAKE_CACHE_ARGS      -DCMAKE_INSTALL_PREFIX:PATH=${XPU_INSTALL_ROOT}
-)
+  FILE(WRITE ${XPU_DOWNLOAD_DIR}/CMakeLists.txt
+    "PROJECT(XPU)\n"
+    "cmake_minimum_required(VERSION 3.0)\n"
+    "install(DIRECTORY xpu/include xpu/lib \n"
+    "        DESTINATION ${XPU_INSTALL_DIR})\n")

+  ExternalProject_Add(
+      ${XPU_PROJECT}
+      ${EXTERNAL_PROJECT_LOG_ARGS}
+      PREFIX                ${XPU_SOURCE_DIR}
+      DOWNLOAD_DIR          ${XPU_DOWNLOAD_DIR}
+      DOWNLOAD_COMMAND      wget --no-check-certificate ${XPU_URL} -c -q -O xpu.tar.gz
+                            && tar xvf xpu.tar.gz
+      DOWNLOAD_NO_PROGRESS  1
+      UPDATE_COMMAND        ""
+      CMAKE_ARGS            -DCMAKE_INSTALL_PREFIX=${XPU_INSTALL_ROOT}
+      CMAKE_CACHE_ARGS      -DCMAKE_INSTALL_PREFIX:PATH=${XPU_INSTALL_ROOT}
+  )
+else()
+  SET(XPU_API_INC_DIR   "${XPU_SDK_ROOT}/XTDK/include/")
+  SET(XPU_API_LIB "${XPU_SDK_ROOT}/XTDK/shlib/libxpuapi.so")
+  SET(XPU_RT_LIB "${XPU_SDK_ROOT}/XTDK/runtime/shlib/libxpurt.so")
+  SET(XPU_LIB_DIR "${XPU_SDK_ROOT}/XTDK/shlib/")
+endif()
+
+INCLUDE_DIRECTORIES(${XPU_API_INC_DIR})
 ADD_LIBRARY(shared_xpuapi SHARED IMPORTED GLOBAL)
 set_property(TARGET shared_xpuapi PROPERTY IMPORTED_LOCATION "${XPU_API_LIB}")

@ -69,4 +75,14 @@ else(WITH_XPU_BKCL)
  TARGET_LINK_LIBRARIES(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} )
 endif(WITH_XPU_BKCL)

-ADD_DEPENDENCIES(xpulib ${XPU_PROJECT})
+if(NOT XPU_SDK_ROOT)
+  ADD_DEPENDENCIES(xpulib ${XPU_PROJECT})
+else()
+  ADD_CUSTOM_TARGET(extern_xpu DEPENDS xpulib)
+endif()
+
+# Ensure that xpu/api.h can be included without dependency errors.
+file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/.xpu_headers_dummy.cc CONTENT "")
+add_library(xpu_headers_dummy STATIC ${CMAKE_CURRENT_BINARY_DIR}/.xpu_headers_dummy.cc)
+add_dependencies(xpu_headers_dummy extern_xpu)
+link_libraries(xpu_headers_dummy)
--- a/cmake/flags.cmake
+++ b/cmake/flags.cmake
@ -4,10 +4,10 @@ include(CheckCCompilerFlag)
 include(CheckCXXSymbolExists)
 include(CheckTypeSize)

-function(CheckCompilerCXX11Flag)
+function(CheckCompilerCXX14Flag)
    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-        if(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 4.8)
-            message(FATAL_ERROR "Unsupported GCC version. GCC >= 4.8 required.")
+        if(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 5.4)
+            message(FATAL_ERROR "Unsupported GCC version. GCC >= 5.4 required.")
        elseif(${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 8.2)
            message(WARNING "Found GCC ${CMAKE_CXX_COMPILER_VERSION} which is too high, recommended to use GCC 8.2")
        endif()
@ -20,23 +20,15 @@ function(CheckCompilerCXX11Flag)
                message(FATAL_ERROR "Unsupported AppleClang version. AppleClang >= 5.1 required.")
            endif()
        else()
-            if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.3)
-                message(FATAL_ERROR "Unsupported Clang version. Clang >= 3.3 required.")
+            if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.4)
+                message(FATAL_ERROR "Unsupported Clang version. Clang >= 3.4 required.")
            endif()
        endif()
    endif()
 endfunction()

-CheckCompilerCXX11Flag()
-if (WITH_GPU)
-    if (${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.0)
-       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
-    else()
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-    endif()
-else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-endif()
+CheckCompilerCXX14Flag()
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
 # safe_set_flag
 #
 # Set a compile flag only if compiler is support
@ -155,7 +147,7 @@ set(COMMON_FLAGS
 )

 if(NOT APPLE)
-    if((${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 8.0) OR (WITH_ROCM_PLATFORM AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 7.3))
+    if((${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 8.0) OR (WITH_ROCM))
        set(COMMON_FLAGS
                ${COMMON_FLAGS}
                -Wno-format-truncation # Warning in boost gcc 8.2
@ -213,5 +205,17 @@ foreach(flag ${GPU_COMMON_FLAGS})
    safe_set_nvflag(${flag})
 endforeach()

-set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${SAFE_GPU_COMMON_FLAGS}")
+if(WITH_GPU)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${SAFE_GPU_COMMON_FLAGS}")
+endif()
+
+if(WITH_ROCM)
+    set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} ${SAFE_GPU_COMMON_FLAGS}")
+endif()
+
+ # Disable -Werror, otherwise the compile will fail for rocblas_gemm_ex
+if(WITH_ROCM)
+    string (REPLACE "-Werror" "-Wno-error" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+    string (REPLACE "-Werror" "-Wno-error" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
+endif()

--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@ -260,15 +260,16 @@ function(merge_static_libs TARGET_NAME)
    # msvc will put libarary in directory of "/Release/xxxlib" by default
    #       COMMAND cmake -E remove "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/${TARGET_NAME}.lib"
    add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
-      COMMAND cmake -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}"
-      COMMAND lib /OUT:${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/lib${TARGET_NAME}.lib ${libfiles}
+      COMMAND cmake -E make_directory $<TARGET_FILE_DIR:${TARGET_NAME}>
+      COMMAND lib /OUT:$<TARGET_FILE:${TARGET_NAME}> ${libfiles}
      )
  endif(WIN32)
 endfunction(merge_static_libs)

 function(check_coverage_opt TARGET_NAME SRCS)
  if(WITH_COVERAGE AND WITH_INCREMENTAL_COVERAGE)
-    if ("$ENV{PADDLE_GIT_DIFF_H_FILE}" STREQUAL "")
+    # if pybind.cc add '-g -O0 -fprofile-arcs -ftest-coverage' only, some testcase will fail.
+    if ("$ENV{PADDLE_GIT_DIFF_H_FILE}" STREQUAL "" AND (NOT ("$ENV{PADDLE_GIT_DIFF_CC_FILE}" MATCHES "pybind.cc")))
      if (NOT ("$ENV{PADDLE_GIT_DIFF_CC_FILE}" STREQUAL ""))
        string(REPLACE "," ";" CC_FILE_LIST $ENV{PADDLE_GIT_DIFF_CC_FILE})
        set(use_coverage_opt FALSE)
@ -381,6 +382,9 @@ function(cc_binary TARGET_NAME)
  endif()
  get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
  target_link_libraries(${TARGET_NAME} ${os_dependency_modules})
+  if(WITH_ROCM)
+    target_link_libraries(${TARGET_NAME} ${ROCM_HIPRTC_LIB})
+  endif()

  check_coverage_opt(${TARGET_NAME} ${cc_binary_SRCS})

@ -402,6 +406,9 @@ function(cc_test_build TARGET_NAME)
    target_link_libraries(${TARGET_NAME} ${cc_test_DEPS} ${os_dependency_modules} paddle_gtest_main lod_tensor memory gtest gflags glog)
    add_dependencies(${TARGET_NAME} ${cc_test_DEPS} paddle_gtest_main lod_tensor memory gtest gflags glog)
    common_link(${TARGET_NAME})
+    if(WITH_ROCM)
+      target_link_libraries(${TARGET_NAME} ${ROCM_HIPRTC_LIB})
+    endif()
  endif()

  check_coverage_opt(${TARGET_NAME} ${cc_test_SRCS})
@ -485,7 +492,7 @@ function(nv_library TARGET_NAME)
        message(FATAL "Please specify source file or library in nv_library.")
      endif()
    endif(nv_library_SRCS)
-    if (WIN32 AND ${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0)
+    if((CUDA_VERSION GREATER 9.2) AND (CUDA_VERSION LESS 11.0) AND (MSVC_VERSION LESS 1910))
      set_target_properties(${TARGET_NAME} PROPERTIES VS_USER_PROPS ${WIN_PROPS})
    endif()
  endif()
@ -503,7 +510,7 @@ function(nv_binary TARGET_NAME)
      add_dependencies(${TARGET_NAME} ${nv_binary_DEPS})
      common_link(${TARGET_NAME})
    endif()
-    if (WIN32 AND ${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0)
+    if((CUDA_VERSION GREATER 9.2) AND (CUDA_VERSION LESS 11.0) AND (MSVC_VERSION LESS 1910))
      set_target_properties(${TARGET_NAME} PROPERTIES VS_USER_PROPS ${WIN_PROPS})
    endif()
  endif()
@ -530,40 +537,31 @@ function(nv_test TARGET_NAME)
    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true)
    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true)
    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true)
-    if (WIN32 AND ${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0)
+    if((CUDA_VERSION GREATER 9.2) AND (CUDA_VERSION LESS 11.0) AND (MSVC_VERSION LESS 1910))
      set_target_properties(${TARGET_NAME} PROPERTIES VS_USER_PROPS ${WIN_PROPS})
    endif()
  endif()
 endfunction(nv_test)

 function(hip_library TARGET_NAME)
-  if (WITH_ROCM_PLATFORM)
+  if (WITH_ROCM)
    set(options STATIC static SHARED shared)
    set(oneValueArgs "")
    set(multiValueArgs SRCS DEPS)
    cmake_parse_arguments(hip_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-    set(_sources ${hip_library_SRCS})
-    set_source_files_properties(${_sources} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
-    HIP_PREPARE_TARGET_COMMANDS(${TARGET_NAME} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-      list(REMOVE_ITEM _sources ${_source_files})
-    endif()
    if(hip_library_SRCS)
+      # FindHIP.cmake defined hip_add_library, HIP_SOURCE_PROPERTY_FORMAT is requried if no .cu files found
+      if(NOT ${CMAKE_CURRENT_SOURCE_DIR} MATCHES ".*/operators")
+        set_source_files_properties(${hip_library_SRCS} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
+      endif()
      if (hip_library_SHARED OR hip_library_shared) # build *.so
-        add_library(${TARGET_NAME} SHARED ${_cmake_options} ${_generated_files} ${_sources})
-        set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE HIP)
+        hip_add_library(${TARGET_NAME} SHARED ${hip_library_SRCS})
      else()
-        add_library(${TARGET_NAME} STATIC ${_cmake_options} ${_generated_files} ${_sources})
-        set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-        target_link_libraries(${TARGET_NAME} ${ROCM_PATH}/hip/lib/libhip_hcc.so)
+        hip_add_library(${TARGET_NAME} STATIC ${hip_library_SRCS})
        find_fluid_modules(${TARGET_NAME})
      endif()
-      if("${hip_library_DEPS}" MATCHES "ARCHIVE_START")
-        # Support linking flags: --whole-archive (Linux) / -force_load (MacOS).
-        # WARNING: Please don't use ARCHIVE_START&ARCHIVE_END if TARGET_NAME will be linked by other libraries.
-        target_circle_link_libraries(${TARGET_NAME} ${hip_library_DEPS})
-        list(REMOVE_ITEM hip_library_DEPS ARCHIVE_START ARCHIVE_END)
-      else()
+      if (hip_library_DEPS)
+        add_dependencies(${TARGET_NAME} ${hip_library_DEPS})
        target_link_libraries(${TARGET_NAME} ${hip_library_DEPS})
      endif()
      # cpplint code style
@ -573,72 +571,27 @@ function(hip_library TARGET_NAME)
          list(APPEND hip_library_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h)
        endif()
      endforeach()
-
-      check_coverage_opt(${TARGET_NAME} ${hip_library_SRCS})
-
    else(hip_library_SRCS)
      if (hip_library_DEPS)
-        merge_static_libs(${TARGET_NAME} ${hip_library_DEPS})
+        list(REMOVE_DUPLICATES hip_library_DEPS)
+        generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:hip_library")
+
+        target_link_libraries(${TARGET_NAME} ${hip_library_DEPS})
+        add_dependencies(${TARGET_NAME} ${hip_library_DEPS})
      else()
-        message(FATAL "Please specify source file or library in nv_library.")
+        message(FATAL "Please specify source file or library in hip_library.")
      endif()
    endif(hip_library_SRCS)
  endif()
 endfunction(hip_library)

-function(hip_library_ops TARGET_NAME)
-  if (WITH_ROCM_PLATFORM)
-    set(options STATIC static SHARED shared)
-    set(oneValueArgs "")
-    set(multiValueArgs SRCS DEPS)
-    cmake_parse_arguments(hip_library_ops "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-    set(_sources ${hip_library_ops_SRCS})
-    HIP_PREPARE_TARGET_COMMANDS(${TARGET_NAME} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-      list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if(hip_library_ops_SRCS)
-      if (hip_library_ops_SHARED OR hip_library_ops_shared) # build *.so
-        add_library(${TARGET_NAME} SHARED ${_cmake_options} ${_generated_files} ${_sources})
-        set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE HIP)
-      else()
-        add_library(${TARGET_NAME} STATIC ${_cmake_options} ${_generated_files} ${_sources})
-        set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-        target_link_libraries(${TARGET_NAME} ${ROCM_PATH}/hip/lib/libhip_hcc.so)
-        find_fluid_modules(${TARGET_NAME})
-      endif()
-      if("${hip_library_ops_DEPS}" MATCHES "ARCHIVE_START")
-        # Support linking flags: --whole-archive (Linux) / -force_load (MacOS).
-        # WARNING: Please don't use ARCHIVE_START&ARCHIVE_END if TARGET_NAME will be linked by other libraries.
-        target_circle_link_libraries(${TARGET_NAME} ${hip_library_ops_DEPS})
-        list(REMOVE_ITEM hip_library_ops_DEPS ARCHIVE_START ARCHIVE_END)
-      else()
-        target_link_libraries(${TARGET_NAME} ${hip_library_ops_DEPS})
-      endif()
-      # cpplint code style
-      foreach(source_file ${hip_library_ops_SRCS})
-        string(REGEX REPLACE "\\.[^.]*$" "" source ${source_file})
-        if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h)
-          list(APPEND hip_library_ops_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h)
-        endif()
-      endforeach()
-    else(hip_library_ops_SRCS)
-      if (hip_library_ops_DEPS)
-        merge_static_libs(${TARGET_NAME} ${hip_library_ops_DEPS})
-      else()
-        message(FATAL "Please specify source file or library in nv_library.")
-      endif()
-    endif(hip_library_ops_SRCS)
-  endif()
-endfunction(hip_library_ops)
-
 function(hip_binary TARGET_NAME)
-  if (WITH_ROCM_PLATFORM)
+  if (WITH_ROCM)
    set(options "")
    set(oneValueArgs "")
    set(multiValueArgs SRCS DEPS)
    cmake_parse_arguments(hip_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-    set_source_files_properties(${_sources} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
+    # FindHIP.cmake defined hip_add_executable, HIP_SOURCE_PROPERTY_FORMAT is requried for .cc files
    hip_add_executable(${TARGET_NAME} ${hip_binary_SRCS})
    if(hip_binary_DEPS)
      target_link_libraries(${TARGET_NAME} ${hip_binary_DEPS})
@ -646,34 +599,29 @@ function(hip_binary TARGET_NAME)
      common_link(${TARGET_NAME})
    endif()
  endif()
-
-  check_coverage_opt(${TARGET_NAME} ${hip_binary_SRCS})
-
 endfunction(hip_binary)

 function(hip_test TARGET_NAME)
-  if (WITH_ROCM_PLATFORM AND WITH_TESTING)
-    set(options "")
+  # The environment variable `CI_SKIP_CPP_TEST` is used to skip the compilation
+  # and execution of test in CI. `CI_SKIP_CPP_TEST` is set to ON when no files
+  # other than *.py are modified.
+  if (WITH_ROCM AND WITH_TESTING AND NOT "$ENV{CI_SKIP_CPP_TEST}" STREQUAL "ON")
    set(oneValueArgs "")
    set(multiValueArgs SRCS DEPS)
    cmake_parse_arguments(hip_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-    set(_sources ${hip_test_SRCS})
-    set_source_files_properties(${_sources} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
-    HIP_PREPARE_TARGET_COMMANDS(${TARGET_NAME} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-      list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    add_executable(${TARGET_NAME} ${_cmake_options} ${_generated_files} ${_sources})
-    set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE HIP)
+    # FindHIP.cmake defined hip_add_executable, HIP_SOURCE_PROPERTY_FORMAT is requried for .cc files
+    hip_add_executable(${TARGET_NAME} ${hip_test_SRCS})
+    # "-pthread -ldl -lrt" is defined in CMAKE_CXX_LINK_EXECUTABLE
+    target_link_options(${TARGET_NAME} PRIVATE -pthread -ldl -lrt)
    get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
-    target_link_libraries(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main memory gtest gflags ${os_dependency_modules})
-    add_dependencies(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main memory gtest gflags)
+    target_link_libraries(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main lod_tensor memory gtest gflags glog ${os_dependency_modules})
+    add_dependencies(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main lod_tensor memory gtest gflags glog)
    common_link(${TARGET_NAME})
    add_test(${TARGET_NAME} ${TARGET_NAME})
+    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true)
+    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true)
+    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true)
  endif()
-
-  check_coverage_opt(${TARGET_NAME} ${hip_test_SRCS})
-
 endfunction(hip_test)

 function(go_library TARGET_NAME)
@ -805,7 +753,8 @@ function(paddle_protobuf_generate_cpp SRCS HDRS)
      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
      -I${CMAKE_CURRENT_SOURCE_DIR}
      --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL}
-      DEPENDS ${ABS_FIL} protoc
+      # Set `EXTERN_PROTOBUF_DEPEND` only if need to compile `protoc.exe`.
+      DEPENDS ${ABS_FIL} ${EXTERN_PROTOBUF_DEPEND}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM )
  endforeach()
@ -854,8 +803,7 @@ function(py_test TARGET_NAME)
    else()
      add_test(NAME ${TARGET_NAME}
               COMMAND ${CMAKE_COMMAND} -E env FLAGS_init_allocated_mem=true FLAGS_cudnn_deterministic=true
-               FLAGS_cpu_deterministic=true
-               PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_ENVS}
+               FLAGS_cpu_deterministic=true ${py_test_ENVS}
               ${PYTHON_EXECUTABLE} -u ${py_test_SRCS} ${py_test_ARGS}
               WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
    endif()
--- a/cmake/hip.cmake
+++ b/cmake/hip.cmake
@ -1,104 +1,87 @@
-if(NOT WITH_ROCM_PLATFORM)
+if(NOT WITH_ROCM)
    return()
 endif()

-include_directories("${ROCM_PATH}/include")
-include_directories("${ROCM_PATH}/hip/include")
-include_directories("${ROCM_PATH}/miopen/include")
-include_directories("${ROCM_PATH}/hipblas/include")
-include_directories("${ROCM_PATH}/rocblas/include")
-include_directories("${ROCM_PATH}/hiprand/include")
-include_directories("${ROCM_PATH}/rocrand/include")
-include_directories("${ROCM_PATH}/rccl/include")
-
-include_directories("${ROCM_PATH}/rocthrust/include/")
-include_directories("${ROCM_PATH}/hipcub/include/")
-include_directories("${ROCM_PATH}/rocprim/include/")
-include_directories("${ROCM_PATH}/hipsparse/include/")
-include_directories("${ROCM_PATH}/rocsparse/include/")
-include_directories("${ROCM_PATH}/rocfft/include/")
-
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
-set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
-# now default is clang
-set(HIP_COMPILER "clang")
-
-list(APPEND EXTERNAL_LIBS "-L${ROCM_PATH}/lib/ -lhip_hcc")
-set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -fPIC -DPADDLE_WITH_HIP -DEIGEN_USE_HIP -DEIGEN_USE_GPU -D__HIP_NO_HALF_CONVERSIONS__ -std=c++11 --amdgpu-target=gfx906" )
-
-if(WITH_RCCL)
-  set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_RCCL")
+if(NOT DEFINED ENV{ROCM_PATH})
+    set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCm has been installed")
+    set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed")
+    set(HIP_CLANG_PATH ${ROCM_PATH}/llvm/bin CACHE PATH "Path to which clang has been installed")
+else()
+    set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCm has been installed")
+    set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed")
+    set(HIP_CLANG_PATH ${ROCM_PATH}/llvm/bin CACHE PATH "Path to which clang has been installed")
 endif()
-
-if(NOT WITH_PYTHON)
-  set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_NO_PYTHON")
-endif(NOT WITH_PYTHON)
-
-if(WITH_DSO)
-  set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_USE_DSO")
-endif(WITH_DSO)
-
-if(WITH_TESTING)
-  set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_TESTING")
-endif(WITH_TESTING)
-
-if(WITH_DISTRIBUTE)
-  set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_DISTRIBUTE")
-endif(WITH_DISTRIBUTE)
-
-if(WITH_GRPC)
-  set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_GRPC")
-endif(WITH_GRPC)
-
-if(WITH_MKLDNN)
-  set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_MKLDNN")
-endif(WITH_MKLDNN)
-
-set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DANY_IMPL_ANY_CAST_MOVEABLE")
-
-if(CMAKE_BUILD_TYPE  STREQUAL "Debug")
-    list(APPEND HIP_HIPCC_FLAGS  ${CMAKE_CXX_FLAGS_DEBUG})
-elseif(CMAKE_BUILD_TYPE  STREQUAL "RelWithDebInfo")
-    list(APPEND HIP_HIPCC_FLAGS  ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
-elseif(CMAKE_BUILD_TYPE  STREQUAL "MinSizeRel")
-    list(APPEND HIP_HIPCC_FLAGS  ${CMAKE_CXX_FLAGS_MINSIZEREL})
+set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
+
+find_package(HIP REQUIRED)
+include_directories(${ROCM_PATH}/include)
+message(STATUS "HIP version: ${HIP_VERSION}")
+message(STATUS "HIP_CLANG_PATH: ${HIP_CLANG_PATH}")
+
+macro(find_package_and_include PACKAGE_NAME)
+  find_package("${PACKAGE_NAME}" REQUIRED)
+  include_directories("${ROCM_PATH}/${PACKAGE_NAME}/include")
+  message(STATUS "${PACKAGE_NAME} version: ${${PACKAGE_NAME}_VERSION}")
+endmacro()
+
+find_package_and_include(miopen)
+find_package_and_include(rocblas)
+find_package_and_include(hiprand)
+find_package_and_include(rocrand)
+find_package_and_include(rccl)
+find_package_and_include(rocthrust)
+find_package_and_include(hipcub)
+find_package_and_include(rocprim)
+find_package_and_include(hipsparse)
+find_package_and_include(rocsparse)
+find_package_and_include(rocfft)
+
+# set CXX flags for HIP
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__HIP_PLATFORM_HCC__")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__HIP_PLATFORM_HCC__")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP")
+set(THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_HIP)
+
+# define HIP_CXX_FLAGS
+list(APPEND HIP_CXX_FLAGS -fPIC)
+list(APPEND HIP_CXX_FLAGS -D__HIP_PLATFORM_HCC__=1)
+# Note(qili93): HIP has compile conflicts of float16.h as platform::float16 overload std::is_floating_point and std::is_integer
+list(APPEND HIP_CXX_FLAGS -D__HIP_NO_HALF_CONVERSIONS__=1)
+list(APPEND HIP_CXX_FLAGS -Wno-macro-redefined)
+list(APPEND HIP_CXX_FLAGS -Wno-inconsistent-missing-override)
+list(APPEND HIP_CXX_FLAGS -Wno-exceptions)
+list(APPEND HIP_CXX_FLAGS -Wno-shift-count-negative)
+list(APPEND HIP_CXX_FLAGS -Wno-shift-count-overflow)
+list(APPEND HIP_CXX_FLAGS -Wno-unused-command-line-argument)
+list(APPEND HIP_CXX_FLAGS -Wno-duplicate-decl-specifier)
+list(APPEND HIP_CXX_FLAGS -Wno-implicit-int-float-conversion)
+list(APPEND HIP_CXX_FLAGS -Wno-pass-failed)
+list(APPEND HIP_CXX_FLAGS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP)
+list(APPEND HIP_CXX_FLAGS -std=c++14)
+
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+  list(APPEND HIP_CXX_FLAGS -g2)
+  list(APPEND HIP_CXX_FLAGS -O0)
+  list(APPEND HIP_HIPCC_FLAGS -fdebug-info-for-profiling)
+endif(CMAKE_BUILD_TYPE MATCHES Debug)
+
+set(HIP_HCC_FLAGS ${HIP_CXX_FLAGS})
+set(HIP_CLANG_FLAGS ${HIP_CXX_FLAGS})
+# Ask hcc to generate device code during compilation so we can use
+# host linker to link.
+list(APPEND HIP_HCC_FLAGS -fno-gpu-rdc)
+list(APPEND HIP_HCC_FLAGS --amdgpu-target=gfx906)
+list(APPEND HIP_CLANG_FLAGS -fno-gpu-rdc)
+list(APPEND HIP_CLANG_FLAGS --amdgpu-target=gfx906)
+
+
+if(HIP_COMPILER STREQUAL clang)
+  set(hip_library_name amdhip64)
+else()
+  set(hip_library_name hip_hcc)
 endif()
+message(STATUS "HIP library name: ${hip_library_name}")

-if("${HIP_COMPILER}" STREQUAL "hcc")
-    if("x${HCC_HOME}" STREQUAL "x")
-      set(HCC_HOME "${ROCM_PATH}/hcc")
-    endif()
-
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES> -ldl --amdgpu-target=gfx906 ")
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES> -shared --amdgpu-target=gfx906")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES> -shared --amdgpu-target=gfx906")
-
-elseif("${HIP_COMPILER}" STREQUAL "clang")
-    
-    if("x${HIP_CLANG_PATH}" STREQUAL "x")
-        set(HIP_CLANG_PATH  "${ROCM_PATH}/llvm/bin")
-    endif()
-
-    #Number of parallel jobs by default is 1
-    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
-      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
-    endif()
-    #Add support for parallel build and link
-    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
-      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
-    endif()
-    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
-      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
-        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS} -Wno-format-nonliteral")
-        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-      else()
-        message("clang compiler doesn't support parallel jobs")
-      endif()
-    endif()
-
-
-    # Set the CMake Flags to use the HIP-Clang Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES> --amdgpu-target=gfx906")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared --amdgpu-target=gfx906" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES> -ldl --amdgpu-target=gfx906")
-endif()
+# set HIP link libs
+find_library(ROCM_HIPRTC_LIB ${hip_library_name} HINTS ${HIP_PATH}/lib)
+message(STATUS "ROCM_HIPRTC_LIB: ${ROCM_HIPRTC_LIB}")
--- a/cmake/inference_lib.cmake
+++ b/cmake/inference_lib.cmake
@ -137,7 +137,7 @@ function(copy_part_of_thrid_party TARGET DST)
 endfunction()

 # inference library for only inference
-set(inference_lib_deps third_party paddle_fluid paddle_fluid_c paddle_fluid_shared paddle_fluid_c_shared)
+set(inference_lib_deps third_party paddle_inference paddle_inference_c paddle_inference_shared paddle_inference_c_shared)
 add_custom_target(inference_lib_dist DEPENDS ${inference_lib_deps})


@ -164,20 +164,20 @@ copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_INSTALL_DIR})
 set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid")
 if(WIN32)
    if(WITH_STATIC_LIB)
-        set(paddle_fluid_lib ${PADDLE_BINARY_DIR}/paddle/fluid/inference/${CMAKE_BUILD_TYPE}/libpaddle_fluid.lib
-                             ${PADDLE_BINARY_DIR}/paddle/fluid/inference/${CMAKE_BUILD_TYPE}/paddle_fluid.*)
+        set(paddle_inference_lib $<TARGET_FILE_DIR:paddle_inference>/libpaddle_inference.lib
+                             $<TARGET_FILE_DIR:paddle_inference>/paddle_inference.*)
    else()
-        set(paddle_fluid_lib ${PADDLE_BINARY_DIR}/paddle/fluid/inference/${CMAKE_BUILD_TYPE}/paddle_fluid.dll
-                             ${PADDLE_BINARY_DIR}/paddle/fluid/inference/${CMAKE_BUILD_TYPE}/paddle_fluid.lib)
+        set(paddle_inference_lib $<TARGET_FILE_DIR:paddle_inference_shared>/paddle_inference.dll
+                             $<TARGET_FILE_DIR:paddle_inference_shared>/paddle_inference.lib)
    endif()
    copy(inference_lib_dist
-            SRCS  ${src_dir}/inference/api/paddle_*.h ${paddle_fluid_lib}
+            SRCS  ${src_dir}/inference/api/paddle_*.h ${paddle_inference_lib}
            DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/lib
            ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/lib)
 else(WIN32)
-    set(paddle_fluid_lib ${PADDLE_BINARY_DIR}/paddle/fluid/inference/libpaddle_fluid.*)
+    set(paddle_inference_lib ${PADDLE_BINARY_DIR}/paddle/fluid/inference/libpaddle_inference.*)
    copy(inference_lib_dist
-                SRCS  ${src_dir}/inference/api/paddle_*.h ${paddle_fluid_lib}
+                SRCS  ${src_dir}/inference/api/paddle_*.h ${paddle_inference_lib}
                DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/lib)
 endif(WIN32)

@ -189,6 +189,19 @@ copy(inference_lib_dist
        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/crypto/)
 include_directories(${CMAKE_BINARY_DIR}/../paddle/fluid/framework/io)

+copy(inference_lib_dist
+        SRCS  ${PADDLE_SOURCE_DIR}/paddle/fluid/extension/include/*
+        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/)
+copy(inference_lib_dist
+        SRCS  ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/complex64.h
+        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/)
+copy(inference_lib_dist
+        SRCS  ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/complex128.h
+        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/)
+copy(inference_lib_dist
+        SRCS  ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/float16.h
+        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/)
+
 # CAPI inference library for only inference
 set(PADDLE_INFERENCE_C_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_c_install_dir" CACHE STRING
 "A path setting CAPI paddle inference shared")
@ -196,13 +209,13 @@ copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_C_INSTALL_DIR})

 set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid")
 if(WIN32)
-  set(paddle_fluid_c_lib ${PADDLE_BINARY_DIR}/paddle/fluid/inference/capi/${CMAKE_BUILD_TYPE}/paddle_fluid_c.*)
+  set(paddle_inference_c_lib $<TARGET_FILE_DIR:paddle_inference_c>/paddle_inference_c.*)
 else(WIN32)
-  set(paddle_fluid_c_lib ${PADDLE_BINARY_DIR}/paddle/fluid/inference/capi/libpaddle_fluid_c.*)
+  set(paddle_inference_c_lib ${PADDLE_BINARY_DIR}/paddle/fluid/inference/capi/libpaddle_inference_c.*)
 endif(WIN32)

 copy(inference_lib_dist
-      SRCS  ${src_dir}/inference/capi/paddle_c_api.h  ${paddle_fluid_c_lib}
+      SRCS  ${src_dir}/inference/capi/paddle_c_api.h  ${paddle_inference_c_lib}
      DSTS  ${PADDLE_INFERENCE_C_INSTALL_DIR}/paddle/include ${PADDLE_INFERENCE_C_INSTALL_DIR}/paddle/lib)

 # fluid library for both train and inference
@ -213,12 +226,12 @@ set(dst_dir "${PADDLE_INSTALL_DIR}/paddle/fluid")
 set(module "inference")
 if(WIN32)
        copy(fluid_lib_dist
-                SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h ${paddle_fluid_lib}
+                SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h ${paddle_inference_lib}
                DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module}
                )
        else()
        copy(fluid_lib_dist
-                SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h ${paddle_fluid_lib}
+                SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h ${paddle_inference_lib}
                DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module} 
                )
 endif()
--- a/cmake/init.cmake
+++ b/cmake/init.cmake
@ -18,6 +18,10 @@ if(NOT WIN32)
    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
    set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG")
 else()
+    # It has not been used now, it can specify CUDA compile flag manualy,
+    # its use is to remvoe /Zi to reduce GPU static library size. But it's dangerous
+    # because CUDA will update by nvidia, then error will occur.
+    # Now, it's used in CUDA:[10.0, 10.2]
    set(WIN_PROPS ${CMAKE_SOURCE_DIR}/cmake/paddle_win.props)
 endif()

--- a/cmake/operators.cmake
+++ b/cmake/operators.cmake
@ -7,13 +7,16 @@ function(op_library TARGET)
    # for ops.
    set(cc_srcs)
    set(cu_srcs)
-    set(hip_cu_srcs)
-    set(miopen_hip_cc_srcs)
+    set(hip_srcs)
    set(cu_cc_srcs)
+    set(hip_cc_srcs)
    set(xpu_cc_srcs)
    set(cudnn_cu_cc_srcs)
+    set(miopen_cu_cc_srcs)
    set(cudnn_cu_srcs)
+    set(miopen_cu_srcs)
    set(CUDNN_FILE)
+    set(MIOPEN_FILE)
    set(mkldnn_cc_srcs)
    set(MKLDNN_FILE)
    set(op_common_deps operator op_registry math_function layer common_infer_shape_functions)
@ -30,46 +33,44 @@ function(op_library TARGET)
        if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cc)
            list(APPEND cc_srcs ${TARGET}.cc)
        endif()
-        if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc)
-            list(APPEND cu_cc_srcs ${TARGET}.cu.cc)
-        endif()
-        if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu)
-            list(APPEND cu_srcs ${TARGET}.cu)
-        endif()
-        if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
-            set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu
-                    ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE)
-            list(APPEND cu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
-        endif()
-
-        if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.hip.cu)
-            list(APPEND hip_cu_srcs ${TARGET}.hip.cu)
-        endif()
-        string(REPLACE "_op" "_cudnn_op" CUDNN_FILE "${TARGET}")
-        if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu.cc)
-            list(APPEND cudnn_cu_cc_srcs ${CUDNN_FILE}.cu.cc)
-        endif()
-        if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu)
-            list(APPEND cudnn_cu_srcs ${CUDNN_FILE}.cu)
+        if(WITH_GPU)
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc)
+                list(APPEND cu_cc_srcs ${TARGET}.cu.cc)
+            endif()
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu)
+                list(APPEND cu_srcs ${TARGET}.cu)
+            endif()
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
+                set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu
+                        ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE)
+                list(APPEND cu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
+            endif()
+            string(REPLACE "_op" "_cudnn_op" CUDNN_FILE "${TARGET}")
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu.cc)
+                list(APPEND cudnn_cu_cc_srcs ${CUDNN_FILE}.cu.cc)
+            endif()
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu)
+                list(APPEND cudnn_cu_srcs ${CUDNN_FILE}.cu)
+            endif()
        endif()
-        if(WITH_ROCM_PLATFORM)
-            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.hip.cu.cc)
-                list(APPEND hip_cu_cc_srcs ${TARGET}.hip.cu.cc)
+        if(WITH_ROCM)
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc)
+                list(APPEND hip_cc_srcs ${TARGET}.cu.cc)
            endif()
-            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.hip.cu)
-                list(APPEND hip_cu_srcs ${TARGET}.hip.cu)
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu)
+                list(APPEND hip_srcs ${TARGET}.cu)
            endif()
-            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.hip.cu)
-                set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.hip.cu
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
+                set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu
                        ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE)
-                list(APPEND hip_cu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.hip.cu)
+                list(APPEND hip_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
            endif()
-            string(REPLACE "_op" "_miopen_op" MIOPEN_FILE "${TARGET}")
-            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.hip.cu.cc)
-                list(APPEND miopen_hip_cu_cc_srcs ${MIOPEN_FILE}.hip.cu.cc)
+            string(REPLACE "_op" "_cudnn_op" MIOPEN_FILE "${TARGET}")
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.cu.cc)
+                list(APPEND miopen_cu_cc_srcs ${MIOPEN_FILE}.cu.cc)
            endif()
-            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.hip.cu)
-                list(APPEND miopen_hip_cu_srcs ${MIOPEN_FILE}.hip.cu)
+            if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.cu)
+                list(APPEND miopen_cu_srcs ${MIOPEN_FILE}.cu)
            endif()
        endif()
        if(WITH_MKLDNN)
@ -86,20 +87,20 @@ function(op_library TARGET)
        endif()
    else()
        foreach(src ${op_library_SRCS})
-            if (WITH_ROCM_PLATFORM AND ${src} MATCHES ".*\\.hip.cu$")
-                list(APPEND hip_cu_srcs ${src})
-            elseif(WITH_ROCM_PLATFORM AND ${src} MATCHES ".*\\.hip.cu.cc$")
-                list(APPEND hip_cu_cc_srcs ${src})
+            if(WITH_ROCM AND ${src} MATCHES ".*_cudnn_op.cu$")
+                list(APPEND miopen_cu_srcs ${src})
+            elseif(WITH_ROCM AND ${src} MATCHES ".*\\.cu$")
+                list(APPEND hip_srcs ${src})
+            elseif(WITH_ROCM AND ${src} MATCHES ".*_cudnn_op.cu.cc$")
+                list(APPEND miopen_cu_cc_srcs ${src})
+            elseif(WITH_ROCM AND ${src} MATCHES ".*\\.cu.cc$")
+                list(APPEND hip_cc_srcs ${src})
            elseif(${src} MATCHES ".*_cudnn_op.cu$")
                list(APPEND cudnn_cu_srcs ${src})
            elseif (${src} MATCHES ".*\\.cu$")
                list(APPEND cu_srcs ${src})
            elseif(${src} MATCHES ".*_cudnn_op.cu.cc$")
                list(APPEND cudnn_cu_cc_srcs ${src})
-            elseif(WITH_ROCM_PLATFORM AND ${src} MATCHES ".*_miopen_op.hip.cc$")
-                list(APPEND miopen_hip_cc_srcs ${src})
-            elseif(WITH_ROCM_PLATFORM AND ${src} MATCHES ".*_miopen_op.hip.cu$")
-                list(APPEND miopen_hip_cu_srcs ${src})
            elseif(WITH_MKLDNN AND ${src} MATCHES ".*_mkldnn_op.cc$")
                list(APPEND mkldnn_cc_srcs ${src})
            elseif(${src} MATCHES ".*\\.cu.cc$")
@ -163,8 +164,13 @@ function(op_library TARGET)
            nv_library(${TARGET} SRCS ${cc_srcs} ${cu_cc_srcs} ${cudnn_cu_cc_srcs} ${cudnn_cu_srcs} ${mkldnn_cc_srcs} ${cu_srcs} DEPS ${op_library_DEPS}
                ${op_common_deps})
        endif()
-    elseif (WITH_ROCM_PLATFORM)
-        hip_library_ops(${TARGET} SRCS ${cc_srcs} ${hip_cu_cc_srcs} ${hip_cu_srcs} ${miopen_hip_cu_cc_srcs} ${miopen_hip_cu_srcs} ${mkldnn_cc_srcs} DEPS ${op_library_DEPS}
+    elseif (WITH_ROCM)
+        list(REMOVE_ITEM miopen_cu_cc_srcs "affine_grid_cudnn_op.cu.cc")
+        list(REMOVE_ITEM miopen_cu_cc_srcs "grid_sampler_cudnn_op.cu.cc")
+        list(REMOVE_ITEM hip_srcs "cholesky_op.cu")
+        list(REMOVE_ITEM hip_srcs "correlation_op.cu")
+        list(REMOVE_ITEM hip_srcs "multinomial_op.cu")
+        hip_library(${TARGET} SRCS ${cc_srcs} ${hip_cc_srcs} ${miopen_cu_cc_srcs} ${miopen_cu_srcs} ${mkldnn_cc_srcs} ${hip_srcs} DEPS ${op_library_DEPS}
                ${op_common_deps})
    else()
        # Unity Build relies on global option `WITH_UNITY_BUILD` and local option `UNITY`.
@ -191,7 +197,7 @@ function(op_library TARGET)
 "tensor_array_read_write_op" "tensorrt_engine_op" "conv_fusion_op"
 "fusion_transpose_flatten_concat_op" "fusion_conv_inception_op"
 "sync_batch_norm_op" "dgc_op" "fused_fc_elementwise_layernorm_op"
-"skip_layernorm_op" "multihead_matmul_op" "fusion_group_op" "fused_bn_activation_op" "fused_embedding_eltwise_layernorm_op" "fusion_gru_op"
+"skip_layernorm_op" "multihead_matmul_op" "fusion_group_op" "fused_bn_activation_op" "fused_embedding_eltwise_layernorm_op" "fusion_gru_op" "fusion_lstm_op"
 "fused_bn_add_activation_op")
        if ("${TARGET}" STREQUAL "${manual_pybind_op}")
            set(pybind_flag 1)
@ -227,13 +233,14 @@ function(op_library TARGET)

    # pybind USE_CPU_ONLY_OP
    list(LENGTH cu_srcs cu_srcs_len)
+    list(LENGTH hip_srcs hip_srcs_len)
    list(LENGTH cu_cc_srcs cu_cc_srcs_len)
+    list(LENGTH hip_cc_srcs hip_cc_srcs_len)
    list(LENGTH mkldnn_cc_srcs mkldnn_cc_srcs_len)
    list(LENGTH xpu_cc_srcs xpu_cc_srcs_len)
-    list(LENGTH hip_cu_srcs hip_cu_srcs_len)
-    list(LENGTH miopen_hip_cc_srcs miopen_hip_cc_srcs_len)
+    list(LENGTH miopen_cu_cc_srcs miopen_cu_cc_srcs_len)
    if (${pybind_flag} EQUAL 0 AND ${mkldnn_cc_srcs_len} EQUAL 0 AND ${cu_srcs_len} EQUAL 0 AND ${cu_cc_srcs_len} EQUAL 0 AND
-        ${hip_cu_srcs_len} EQUAL 0 AND ${miopen_hip_cc_srcs_len} EQUAL 0 AND ${xpu_cc_srcs_len} EQUAL 0)
+        ${hip_srcs_len} EQUAL 0 AND ${hip_cc_srcs_len} EQUAL 0 AND ${miopen_cu_cc_srcs_len} EQUAL 0 AND ${xpu_cc_srcs_len} EQUAL 0)
        file(APPEND ${pybind_file} "USE_CPU_ONLY_OP(${TARGET});\n")
        set(pybind_flag 1)
    endif()
@ -248,26 +255,26 @@ function(op_library TARGET)
      endif()
    endif()

-    # pybind USE_OP_DEVICE_KERNEL for CUDNN
-    list(LENGTH cudnn_cu_srcs cudnn_cu_srcs_len)
-    if (WITH_GPU AND ${cudnn_cu_srcs_len} GREATER 0)
-        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n")
-    endif()
-
    # pybind USE_OP_DEVICE_KERNEL for MIOPEN
-    list(LENGTH miopen_hip_cu_cc_srcs miopen_hip_cu_cc_srcs_len)
-    if (WITH_ROCM_PLATFORM AND ${miopen_hip_cu_cc_srcs_len} GREATER 0)
+    list(LENGTH miopen_cu_cc_srcs miopen_cu_cc_srcs_len)
+    if (WITH_ROCM AND ${miopen_cu_cc_srcs_len} GREATER 0)
      if(${TARGET} STREQUAL "activation")
        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(relu, CUDNN);\n")
      else()
-       file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n")
+        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n")
      endif()
    endif()

+    # pybind USE_OP_DEVICE_KERNEL for CUDNN
+    list(LENGTH cudnn_cu_srcs cudnn_cu_srcs_len)
+    if (WITH_GPU AND ${cudnn_cu_srcs_len} GREATER 0)
+        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n")
+    endif()
+
    # pybind USE_OP_DEVICE_KERNEL for MIOPEN
-    list(LENGTH miopen_hip_cu_srcs miopen_hip_cu_srcs_len)
-    if (WITH_ROCM_PLATFORM AND ${miopen_hip_cu_srcs_len} GREATER 0)
-       file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n")
+    list(LENGTH miopen_cu_srcs miopen_cu_srcs_len)
+    if (WITH_ROCM AND ${miopen_cu_srcs_len} GREATER 0)
+        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n")
    endif()

    if (WITH_XPU AND ${xpu_cc_srcs_len} GREATER 0)
--- a/cmake/paddle_win.props
+++ b/cmake/paddle_win.props
@ -15,7 +15,7 @@
            <Warning>InheritFromHost</Warning>

            <BaseCommandLineTemplate>-ccbin "%(VCBinDir)" -x cu [GenerateRelocatableDeviceCode] [Include] [RequiredIncludes] [InterleaveSourceInPTX] [GPUDebugInfo] [GenerateLineInfo] [Keep] [KeepDir] [MaxRegCount] [PtxAsOptionV] [TargetMachinePlatform] [NvccCompilation] [CudaRuntime] [AdditionalOptions]</BaseCommandLineTemplate>
-            <BuildCommandLineTemplate>--use-local-env --cl-version $(CudaClVersion)</BuildCommandLineTemplate>
+            <BuildCommandLineTemplate>--use-local-env $(CudaClVersion)</BuildCommandLineTemplate>
            <BuildDynamicCommandLineTemplate>[CodeGeneration]</BuildDynamicCommandLineTemplate>
            <CleanCommandLineTemplate>-clean</CleanCommandLineTemplate>
            <!-- <HostCommandLineTemplate>-Xcompiler &quot;/EHsc [Warning] /nologo [Optimization] $(CudaForceSynchronousPdbWrites) /Zi [RuntimeChecks] [Runtime] [TypeInfo]&quot;</HostCommandLineTemplate> -->
--- a/cmake/rccl.cmake
+++ b/cmake/rccl.cmake
@ -0,0 +1,28 @@
+if(NOT WITH_ROCM)
+    return()
+endif()
+
+# Now we don't support RCCL on windows
+if(WIN32)
+    return()
+endif()
+
+if(WITH_RCCL)
+    set(RCCL_ROOT ${ROCM_PATH}/rccl CACHE PATH "RCCL ROOT")
+    find_path(RCCL_INCLUDE_DIR rccl.h
+        PATHS ${RCCL_ROOT} ${RCCL_ROOT}/include ${RCCL_ROOT}/local/include
+        $ENV{RCCL_ROOT} $ENV{RCCL_ROOT}/include $ENV{RCCL_ROOT}/local/include
+        NO_DEFAULT_PATH
+    )
+
+    file(READ ${RCCL_INCLUDE_DIR}/rccl.h RCCL_VERSION_FILE_CONTENTS)
+
+    string(REGEX MATCH "define NCCL_VERSION_CODE +([0-9]+)"
+        RCCL_VERSION "${RCCL_VERSION_FILE_CONTENTS}")
+    string(REGEX REPLACE "define NCCL_VERSION_CODE +([0-9]+)" "\\1"
+        RCCL_VERSION "${RCCL_VERSION}")
+
+    # 2604 for ROCM3.5 and 2708 for ROCM 3.9
+    message(STATUS "Current RCCL header is ${RCCL_INCLUDE_DIR}/rccl.h. "
+            "Current RCCL version is v${RCCL_VERSION}. ")
+endif()
--- a/Show More
+++ b/Show More