|
|
@ -800,7 +800,7 @@ class TestDistBase(unittest.TestCase):
|
|
|
|
if self.__use_cuda:
|
|
|
|
if self.__use_cuda:
|
|
|
|
tr_cmd += " --use_cuda"
|
|
|
|
tr_cmd += " --use_cuda"
|
|
|
|
env.update({
|
|
|
|
env.update({
|
|
|
|
"CUDA_VISIBLE_DEVICES": "{}".format(trainer_id),
|
|
|
|
"CUDA_VISIBLE_DEVICES": "{}".format(trainer_id % 2),
|
|
|
|
"PADDLE_TRAINERS_NUM": "{}".format(trainer_num),
|
|
|
|
"PADDLE_TRAINERS_NUM": "{}".format(trainer_num),
|
|
|
|
"PADDLE_TRAINER_ID": "{}".format(trainer_id),
|
|
|
|
"PADDLE_TRAINER_ID": "{}".format(trainer_id),
|
|
|
|
"PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints,
|
|
|
|
"PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints,
|
|
|
@ -813,7 +813,7 @@ class TestDistBase(unittest.TestCase):
|
|
|
|
tr_cmd += " --use_dgc"
|
|
|
|
tr_cmd += " --use_dgc"
|
|
|
|
|
|
|
|
|
|
|
|
if self._mp_mode:
|
|
|
|
if self._mp_mode:
|
|
|
|
env = {"FLAGS_selected_gpus": "{}".format(trainer_id)}
|
|
|
|
env = {"FLAGS_selected_gpus": "{}".format(trainer_id % 2)}
|
|
|
|
|
|
|
|
|
|
|
|
if self._nccl_comm_num > 1:
|
|
|
|
if self._nccl_comm_num > 1:
|
|
|
|
tr_cmd += " --nccl_comm_num {}".format(self._nccl_comm_num)
|
|
|
|
tr_cmd += " --nccl_comm_num {}".format(self._nccl_comm_num)
|
|
|
|