|
|
|
@ -152,14 +152,12 @@ static void PrintNanInf(const T* value, const size_t numel, int print_num,
|
|
|
|
|
static_cast<uint64_t>(i), static_cast<float>(value[i]));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
bool has_nan_inf = true;
|
|
|
|
|
printf("In cpu, there has %lu,%lu,%lu nan,inf,num\n",
|
|
|
|
|
static_cast<uint64_t>(nan_count), static_cast<uint64_t>(inf_count),
|
|
|
|
|
static_cast<uint64_t>(num_count));
|
|
|
|
|
PADDLE_ENFORCE_EQ(has_nan_inf, false,
|
|
|
|
|
platform::errors::PreconditionNotMet(
|
|
|
|
|
"===ERROR: in [op=%s] [tensor=%s] find nan or inf===",
|
|
|
|
|
op_type, var_name));
|
|
|
|
|
PADDLE_THROW(platform::errors::PreconditionNotMet(
|
|
|
|
|
"There are `nan` or `inf` in tensor (%s) of operator (%s).", var_name,
|
|
|
|
|
op_type));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// openmp 4.0, reduction with fp16
|
|
|
|
@ -231,14 +229,25 @@ template <>
|
|
|
|
|
void CheckNanInf<paddle::platform::complex64>(
|
|
|
|
|
const paddle::platform::complex64* value, const size_t numel, int print_num,
|
|
|
|
|
const std::string& op_type, const std::string& var_name) {
|
|
|
|
|
paddle::platform::complex64 sum(0.0, 0.0);
|
|
|
|
|
#pragma omp parallel for reduction(+ : sum)
|
|
|
|
|
float real_sum = 0.0f;
|
|
|
|
|
#pragma omp parallel for reduction(+ : real_sum)
|
|
|
|
|
for (size_t i = 0; i < numel; ++i) {
|
|
|
|
|
sum += (value[i] - value[i]);
|
|
|
|
|
real_sum += (value[i].real - value[i].real);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (std::isnan(sum) || std::isinf(sum)) {
|
|
|
|
|
PrintNanInf(value, numel, print_num, op_type, var_name);
|
|
|
|
|
float imag_sum = 0.0f;
|
|
|
|
|
#pragma omp parallel for reduction(+ : imag_sum)
|
|
|
|
|
for (size_t i = 0; i < numel; ++i) {
|
|
|
|
|
imag_sum += (value[i].imag - value[i].imag);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (std::isnan(real_sum) || std::isinf(real_sum) || std::isnan(imag_sum) ||
|
|
|
|
|
std::isinf(imag_sum)) {
|
|
|
|
|
// hot fix for compile failed in gcc4.8
|
|
|
|
|
// here also need print detail info of nan or inf later
|
|
|
|
|
PADDLE_THROW(platform::errors::PreconditionNotMet(
|
|
|
|
|
"There are `nan` or `inf` in tensor (%s) of operator (%s).", var_name,
|
|
|
|
|
op_type));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -246,17 +255,27 @@ template <>
|
|
|
|
|
void CheckNanInf<paddle::platform::complex128>(
|
|
|
|
|
const paddle::platform::complex128* value, const size_t numel,
|
|
|
|
|
int print_num, const std::string& op_type, const std::string& var_name) {
|
|
|
|
|
paddle::platform::complex128 sum(0.0, 0.0);
|
|
|
|
|
#pragma omp parallel for reduction(+ : sum)
|
|
|
|
|
double real_sum = 0.0;
|
|
|
|
|
#pragma omp parallel for reduction(+ : real_sum)
|
|
|
|
|
for (size_t i = 0; i < numel; ++i) {
|
|
|
|
|
sum += (value[i] - value[i]);
|
|
|
|
|
real_sum += (value[i].real - value[i].real);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (std::isnan(sum) || std::isinf(sum)) {
|
|
|
|
|
PrintNanInf(value, numel, print_num, op_type, var_name);
|
|
|
|
|
}
|
|
|
|
|
double imag_sum = 0.0;
|
|
|
|
|
#pragma omp parallel for reduction(+ : imag_sum)
|
|
|
|
|
for (size_t i = 0; i < numel; ++i) {
|
|
|
|
|
imag_sum += (value[i].imag - value[i].imag);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (std::isnan(real_sum) || std::isinf(real_sum) || std::isnan(imag_sum) ||
|
|
|
|
|
std::isinf(imag_sum)) {
|
|
|
|
|
// hot fix for compile failed in gcc4.8
|
|
|
|
|
// here also need print detail info of nan or inf later
|
|
|
|
|
PADDLE_THROW(platform::errors::PreconditionNotMet(
|
|
|
|
|
"There are `nan` or `inf` in tensor (%s) of operator (%s).", var_name,
|
|
|
|
|
op_type));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
template <>
|
|
|
|
|