|
|
@ -50,7 +50,7 @@ echo "RANK_TABLE_FILE=${RANK_TABLE_FILE}"
|
|
|
|
|
|
|
|
|
|
|
|
export SERVER_ID=0
|
|
|
|
export SERVER_ID=0
|
|
|
|
rank_start=$((DEVICE_NUM * SERVER_ID))
|
|
|
|
rank_start=$((DEVICE_NUM * SERVER_ID))
|
|
|
|
for((i=0; i<${SERVER_NUM}; i++))
|
|
|
|
for((i=0; i<${DEVICE_NUM}; i++))
|
|
|
|
do
|
|
|
|
do
|
|
|
|
export DEVICE_ID=$i
|
|
|
|
export DEVICE_ID=$i
|
|
|
|
export RANK_ID=$((rank_start + i))
|
|
|
|
export RANK_ID=$((rank_start + i))
|
|
|
@ -59,7 +59,7 @@ do
|
|
|
|
cp -r ../src ./train_parallel$i
|
|
|
|
cp -r ../src ./train_parallel$i
|
|
|
|
cp ../train.py ./train_parallel$i
|
|
|
|
cp ../train.py ./train_parallel$i
|
|
|
|
echo "start training for rank $RANK_ID, device $DEVICE_ID, $dataset_type"
|
|
|
|
echo "start training for rank $RANK_ID, device $DEVICE_ID, $dataset_type"
|
|
|
|
cd .train_parallel$i || exit
|
|
|
|
cd ./train_parallel$i || exit
|
|
|
|
env > env.log
|
|
|
|
env > env.log
|
|
|
|
python train.py --device_id=$i --dataset_name=$dataset_type> log 2>&1 &
|
|
|
|
python train.py --device_id=$i --dataset_name=$dataset_type> log 2>&1 &
|
|
|
|
cd ..
|
|
|
|
cd ..
|
|
|
|