diff --git a/paddle/fluid/platform/cuda_helper.h b/paddle/fluid/platform/cuda_helper.h
index 2a1f0b9ac5..9357d5db17 100644
--- a/paddle/fluid/platform/cuda_helper.h
+++ b/paddle/fluid/platform/cuda_helper.h
@@ -86,7 +86,7 @@ class CublasHandleHolder {
           dynload::cublasSetMathMode(handle_, CUBLAS_TENSOR_OP_MATH));
 #if CUDA_VERSION >= 11000
     } else if (math_type == CUBLAS_TF32_TENSOR_OP_MATH) {
-      PADDLE_ENFORCE_CUDA_SUCCESS(
+      PADDLE_RETRY_CUDA_SUCCESS(
           dynload::cublasSetMathMode(handle_, CUBLAS_TF32_TENSOR_OP_MATH));
 #endif  // CUDA_VERSION >= 11000
     }