gongweibao
4 years ago
committed by
GitHub
parent
1882f2ce2d
commit
f9c97dd728
@ -0,0 +1,35 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
#include <glog/logging.h>
|
||||
#include "acl/acl_rt.h"
|
||||
#include "paddle/fluid/platform/ascend_npu_info.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace platform {
|
||||
namespace ascend{
|
||||
|
||||
int NPUDevice::GetDeviceCount() {
|
||||
uint32_t count = 0;
|
||||
aclError status = aclrtGetDeviceCount(&count);
|
||||
if(status != 0){
|
||||
LOG(ERROR) << "aclrtGetDeviceCount error code:" << status;
|
||||
return -1;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
} // namespace ascend
|
||||
} // namespace platform
|
||||
} // namespace paddle
|
||||
|
||||
|
||||
@ -0,0 +1,28 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
#pragma once
|
||||
|
||||
#ifdef PADDLE_WITH_ASCEND
|
||||
|
||||
namespace paddle {
|
||||
namespace platform {
|
||||
namespace ascend{
|
||||
|
||||
class NPUDevice{
|
||||
public:
|
||||
//! Get the total number of XPU devices in system.
|
||||
static int GetDeviceCount();
|
||||
};
|
||||
|
||||
} // namespace ascend
|
||||
} // namespace platform
|
||||
} // namespace paddle
|
||||
#endif
|
||||
@ -0,0 +1,37 @@
|
||||
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
def train(prefix):
|
||||
selected_accelerators = os.getenv("FLAGS_selected_accelerators")
|
||||
trainer_id = int(os.getenv("PADDLE_TRAINER_ID"))
|
||||
worker_endpoints_env = os.getenv("PADDLE_TRAINER_ENDPOINTS")
|
||||
current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT")
|
||||
worker_endpoints = worker_endpoints_env
|
||||
trainers_num = len(worker_endpoints.split(','))
|
||||
|
||||
details = "selected_accelerators:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\
|
||||
.format(selected_accelerators, worker_endpoints, trainers_num, current_endpoint,trainer_id)
|
||||
|
||||
print(details)
|
||||
with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), "w") as f:
|
||||
f.write(details)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
prefix = sys.argv[1]
|
||||
train(prefix)
|
||||
@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
|
||||
# use paddlecloud
|
||||
echo "begin test use paddlecloud"
|
||||
cluster_node_ips="127.0.0.1,127.0.0.2"
|
||||
export PADDLE_TRAINERS_NUM=2
|
||||
export POD_IP=127.0.0.1
|
||||
export PADDLE_TRAINERS=127.0.0.1,127.0.0.2
|
||||
export PADDLE_TRAINER_ID=0
|
||||
|
||||
export PADDLE_PORT=35789
|
||||
export TRAINER_PORTS_NUM=2
|
||||
|
||||
distributed_args="--ips=${cluster_node_ips} --ascend_npus=0,1 --log_dir=testlog"
|
||||
python -m paddle.distributed.fleet.launch ${distributed_args} ascend_multi_process_collective.py fleetlaunchascend
|
||||
|
||||
str1="selected_accelerators:0 worker_endpoints:127.0.0.1:35789,127.0.0.1:35790,127.0.0.2:35789,127.0.0.2:35790 trainers_num:4 current_endpoint:127.0.0.1:35789 trainer_id:0"
|
||||
str2="selected_accelerators:1 worker_endpoints:127.0.0.1:35789,127.0.0.1:35790,127.0.0.2:35789,127.0.0.2:35790 trainers_num:4 current_endpoint:127.0.0.1:35790 trainer_id:1"
|
||||
file_0="multi_process_fleetlaunchascend.check_0.log"
|
||||
file_1="multi_process_fleetlaunchascend.check_1.log"
|
||||
|
||||
echo "paddlecloud params test"
|
||||
if grep -q "$str1" "$file_0"; then
|
||||
echo "find trainer 0"
|
||||
else
|
||||
echo "not find trainer 0"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
if grep -q "$str2" "$file_1"; then
|
||||
echo "find trainer 1"
|
||||
else
|
||||
echo "not find trainer 1"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
# test async poll process
|
||||
if [ -f $file_0 ]; then
|
||||
rm $file_0
|
||||
fi
|
||||
if [ -f $file_1 ]; then
|
||||
rm $file_1
|
||||
fi
|
||||
@ -0,0 +1,64 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from . import collective
|
||||
from .. import core
|
||||
OpRole = core.op_proto_and_checker_maker.OpRole
|
||||
|
||||
class AscendTranspiler(collective.Collective):
|
||||
def __init__(self, startup_program, main_program):
|
||||
self.nrings = 1
|
||||
super(AscendTranspiler, self).__init__(self.nrings)
|
||||
self._startup_program = startup_program
|
||||
self._main_program = main_program
|
||||
|
||||
|
||||
def _insert_allreduce_ops(self):
|
||||
block = self._main_program.global_block()
|
||||
ring_id = -1
|
||||
grad = None
|
||||
for idx, op in reversed(list(enumerate(block.ops))):
|
||||
if self._is_backward_op(op) and \
|
||||
self.op_role_var_key in op.attr_names:
|
||||
op_role_var = op.all_attrs()[self.op_role_var_key]
|
||||
|
||||
if len(op_role_var) == 0:
|
||||
continue
|
||||
assert len(op_role_var) % 2 == 0
|
||||
|
||||
offset = idx
|
||||
for i in range(0, len(op_role_var), 2):
|
||||
param = block.vars[op_role_var[i]]
|
||||
grad = block.vars[op_role_var[i + 1]]
|
||||
if param.is_distributed:
|
||||
continue
|
||||
|
||||
# As we search ops reversedly, we should insert c_allreduce_sum
|
||||
# op in the same way to keep the ring_id alternate
|
||||
ring_id = (ring_id + 1) % self.nrings
|
||||
block._insert_op(
|
||||
offset + 1,
|
||||
type='allreduce',
|
||||
inputs={'X': grad},
|
||||
outputs={'Out': grad},
|
||||
attrs={
|
||||
'ring_id': ring_id,
|
||||
self.op_role_key: OpRole.Backward
|
||||
})
|
||||
|
||||
if grad is None:
|
||||
return
|
||||
|
||||
def transpile(self):
|
||||
self._insert_allreduce_ops()
|
||||
Loading…
Reference in new issue