Repair nccl op test (#8575)
* fix nccl op unit test * fix build error * format code * refine nccl related unit test * fix build error * add setGPUData * clean up * follow comments * rm test_nccl.cu * follow comment * rm waitshanyi15-patch-2
QI JUN
7 years ago
committed by
GitHub
parent
ada82a3e24
commit
7287630e83
File diff suppressed because it is too large
Load Diff
@ -1,153 +0,0 @@
|
|||||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License. */
|
|
||||||
|
|
||||||
#include <thrust/device_vector.h>
|
|
||||||
#include <memory>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "glog/logging.h"
|
|
||||||
#include "gtest/gtest.h"
|
|
||||||
|
|
||||||
#include "paddle/fluid/framework/init.h"
|
|
||||||
#include "paddle/fluid/platform/device_context.h"
|
|
||||||
#include "paddle/fluid/platform/dynload/nccl.h"
|
|
||||||
#include "paddle/fluid/platform/enforce.h"
|
|
||||||
#include "paddle/fluid/platform/gpu_info.h"
|
|
||||||
|
|
||||||
static int dev_count = 0;
|
|
||||||
|
|
||||||
namespace paddle {
|
|
||||||
namespace platform {
|
|
||||||
|
|
||||||
TEST(NCCL, init) {
|
|
||||||
std::vector<ncclComm_t> comms;
|
|
||||||
comms.resize(dev_count);
|
|
||||||
PADDLE_ENFORCE(dynload::ncclCommInitAll(comms.data(), dev_count, nullptr));
|
|
||||||
|
|
||||||
for (int i = 0; i < dev_count; ++i) {
|
|
||||||
dynload::ncclCommDestroy(comms[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename T>
|
|
||||||
struct PerThreadData {
|
|
||||||
thrust::device_vector<T> send_buff;
|
|
||||||
thrust::device_vector<T> recv_buff;
|
|
||||||
CUDADeviceContext dev_ctx;
|
|
||||||
|
|
||||||
T* SendBuff() { return thrust::raw_pointer_cast(send_buff.data()); }
|
|
||||||
|
|
||||||
T* RecvBuff() { return thrust::raw_pointer_cast(recv_buff.data()); }
|
|
||||||
|
|
||||||
PerThreadData(int gpu_id, size_t size) : dev_ctx(CUDAPlace(gpu_id)) {
|
|
||||||
send_buff.resize(size);
|
|
||||||
for (size_t i = 0; i < size; ++i) {
|
|
||||||
send_buff[i] = static_cast<T>(i);
|
|
||||||
}
|
|
||||||
recv_buff.resize(size);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
static constexpr int ELEM_COUNT = 10000;
|
|
||||||
|
|
||||||
TEST(NCCL, all_reduce) {
|
|
||||||
std::vector<ncclComm_t> comms;
|
|
||||||
comms.resize(dev_count);
|
|
||||||
VLOG(1) << "Initializing ncclComm";
|
|
||||||
dynload::ncclCommInitAll(comms.data(), dev_count, nullptr);
|
|
||||||
VLOG(1) << "ncclComm initialized";
|
|
||||||
VLOG(1) << "Creating thread data";
|
|
||||||
std::vector<std::unique_ptr<PerThreadData<double>>> data;
|
|
||||||
data.reserve(dev_count);
|
|
||||||
for (int i = 0; i < dev_count; ++i) {
|
|
||||||
VLOG(1) << "Creating thread data for device " << i;
|
|
||||||
SetDeviceId(i);
|
|
||||||
data.emplace_back(new PerThreadData<double>(i, ELEM_COUNT));
|
|
||||||
}
|
|
||||||
VLOG(1) << "Thread data created";
|
|
||||||
|
|
||||||
VLOG(1) << "Check send_buf data";
|
|
||||||
for (int i = 0; i < dev_count; ++i) {
|
|
||||||
VLOG(1) << "Check on device " << i;
|
|
||||||
SetDeviceId(i);
|
|
||||||
thrust::host_vector<double> tmp = data[i]->send_buff;
|
|
||||||
for (size_t j = 0; j < tmp.size(); ++j) {
|
|
||||||
ASSERT_NEAR(static_cast<double>(j), tmp[j], 1e-5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
VLOG(1) << "Invoking ncclAllReduce";
|
|
||||||
|
|
||||||
dynload::ncclGroupStart();
|
|
||||||
for (int i = 0; i < dev_count; ++i) {
|
|
||||||
VLOG(1) << "Invoking ncclAllReduce with device " << i;
|
|
||||||
SetDeviceId(i);
|
|
||||||
PADDLE_ENFORCE(dynload::ncclAllReduce(
|
|
||||||
data[i]->SendBuff(), data[i]->RecvBuff(), ELEM_COUNT, ncclDouble,
|
|
||||||
ncclSum, comms[i], data[i]->dev_ctx.stream()));
|
|
||||||
VLOG(1) << "Invoked ncclAllReduce for device " << i;
|
|
||||||
}
|
|
||||||
dynload::ncclGroupEnd();
|
|
||||||
|
|
||||||
VLOG(1) << "Invoked ncclAllReduce";
|
|
||||||
|
|
||||||
VLOG(1) << "Sync devices";
|
|
||||||
for (int i = 0; i < dev_count; ++i) {
|
|
||||||
VLOG(1) << "Sync device " << i;
|
|
||||||
SetDeviceId(i);
|
|
||||||
data[i]->dev_ctx.Wait();
|
|
||||||
}
|
|
||||||
VLOG(1) << "device synced";
|
|
||||||
|
|
||||||
for (int i = 0; i < dev_count; ++i) {
|
|
||||||
SetDeviceId(i);
|
|
||||||
VLOG(1) << "Checking vector on device " << i;
|
|
||||||
thrust::host_vector<double> tmp = data[i]->recv_buff;
|
|
||||||
for (size_t j = 0; j < tmp.size(); ++j) {
|
|
||||||
auto elem = static_cast<double>(j);
|
|
||||||
elem *= dev_count;
|
|
||||||
ASSERT_NEAR(tmp[j], elem, 1e-4);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < dev_count; ++i) {
|
|
||||||
dynload::ncclCommDestroy(comms[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // namespace platform
|
|
||||||
} // namespace paddle
|
|
||||||
|
|
||||||
int main(int argc, char** argv) {
|
|
||||||
dev_count = paddle::platform::GetCUDADeviceCount();
|
|
||||||
if (dev_count <= 1) {
|
|
||||||
LOG(WARNING)
|
|
||||||
<< "Cannot test multi-gpu nccl, because the CUDA device count is "
|
|
||||||
<< dev_count;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<paddle::platform::Place> places;
|
|
||||||
|
|
||||||
places.emplace_back(paddle::platform::CPUPlace());
|
|
||||||
int count = paddle::platform::GetCUDADeviceCount();
|
|
||||||
for (int i = 0; i < count; ++i) {
|
|
||||||
places.emplace_back(paddle::platform::CUDAPlace(i));
|
|
||||||
}
|
|
||||||
|
|
||||||
VLOG(0) << " DeviceCount " << count;
|
|
||||||
paddle::platform::DeviceContextPool::Init(places);
|
|
||||||
|
|
||||||
testing::InitGoogleTest(&argc, argv);
|
|
||||||
return RUN_ALL_TESTS();
|
|
||||||
}
|
|
||||||
Loading…
Reference in new issue