fix trt int8 calib precision bug. test=develop (#23036)

revert-23830-2.0-beta
Pei Yang 5 years ago committed by GitHub
parent 1dc1f9270e
commit 24db750386
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -37,7 +37,8 @@ TRTInt8Calibrator::TRTInt8Calibrator(
temp_tensor.Resize(data_shape);
data_tensors_.push_back(temp_tensor);
data_buffers_[input_name] = std::pair<void*, size_t>(
static_cast<void*>(temp_tensor.mutable_data<int16_t>(place)), num_ele);
static_cast<void*>(temp_tensor.mutable_data<int16_t>(place)),
data_size);
i += 1;
}
}

Loading…
Cancel
Save