!10290 add fasttext model to model zoo
From: @zhaojichen Reviewed-by: Signed-off-by:pull/10290/MERGE
mindspore-ci-bot
4 years ago
committed by
Gitee
commit
1150ae3376
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,118 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
"""FastText for Evaluation"""
|
||||||
|
import argparse
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
import mindspore.nn as nn
|
||||||
|
import mindspore.common.dtype as mstype
|
||||||
|
import mindspore.ops.operations as P
|
||||||
|
from mindspore.common.tensor import Tensor
|
||||||
|
from mindspore.train.model import Model
|
||||||
|
from mindspore.train.serialization import load_checkpoint, load_param_into_net
|
||||||
|
import mindspore.dataset.engine as de
|
||||||
|
import mindspore.dataset.transforms.c_transforms as deC
|
||||||
|
from mindspore import context
|
||||||
|
from src.fasttext_model import FastText
|
||||||
|
parser = argparse.ArgumentParser(description='fasttext')
|
||||||
|
parser.add_argument('--data_path', type=str, help='infer dataset path..')
|
||||||
|
parser.add_argument('--data_name', type=str, required=True, default='ag',
|
||||||
|
help='dataset name. eg. ag, dbpedia')
|
||||||
|
parser.add_argument("--model_ckpt", type=str, required=True,
|
||||||
|
help="existed checkpoint address.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.data_name == "ag":
|
||||||
|
from src.config import config_ag as config
|
||||||
|
target_label1 = ['0', '1', '2', '3']
|
||||||
|
elif args.data_name == 'dbpedia':
|
||||||
|
from src.config import config_db as config
|
||||||
|
target_label1 = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13']
|
||||||
|
elif args.data_name == 'yelp_p':
|
||||||
|
from src.config import config_yelpp as config
|
||||||
|
target_label1 = ['0', '1']
|
||||||
|
context.set_context(
|
||||||
|
mode=context.GRAPH_MODE,
|
||||||
|
save_graphs=False,
|
||||||
|
device_target="Ascend")
|
||||||
|
|
||||||
|
class FastTextInferCell(nn.Cell):
|
||||||
|
"""
|
||||||
|
Encapsulation class of FastText network infer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
network (nn.Cell): FastText model.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple[Tensor, Tensor], predicted_ids
|
||||||
|
"""
|
||||||
|
def __init__(self, network):
|
||||||
|
super(FastTextInferCell, self).__init__(auto_prefix=False)
|
||||||
|
self.network = network
|
||||||
|
self.argmax = P.ArgMaxWithValue(axis=1, keep_dims=True)
|
||||||
|
self.log_softmax = nn.LogSoftmax(axis=1)
|
||||||
|
|
||||||
|
def construct(self, src_tokens, src_tokens_lengths):
|
||||||
|
"""construct fasttext infer cell"""
|
||||||
|
prediction = self.network(src_tokens, src_tokens_lengths)
|
||||||
|
predicted_idx = self.log_softmax(prediction)
|
||||||
|
predicted_idx, _ = self.argmax(predicted_idx)
|
||||||
|
|
||||||
|
return predicted_idx
|
||||||
|
|
||||||
|
def load_infer_dataset(batch_size, datafile):
|
||||||
|
"""data loader for infer"""
|
||||||
|
ds = de.MindDataset(datafile, columns_list=['src_tokens', 'src_tokens_length', 'label_idx'])
|
||||||
|
|
||||||
|
type_cast_op = deC.TypeCast(mstype.int32)
|
||||||
|
ds = ds.map(operations=type_cast_op, input_columns="src_tokens")
|
||||||
|
ds = ds.map(operations=type_cast_op, input_columns="src_tokens_length")
|
||||||
|
ds = ds.map(operations=type_cast_op, input_columns="label_idx")
|
||||||
|
ds = ds.batch(batch_size=batch_size, drop_remainder=True)
|
||||||
|
|
||||||
|
return ds
|
||||||
|
|
||||||
|
def run_fasttext_infer():
|
||||||
|
"""run infer with FastText"""
|
||||||
|
dataset = load_infer_dataset(batch_size=config.batch_size, datafile=args.data_path)
|
||||||
|
fasttext_model = FastText(config.vocab_size, config.embedding_dims, config.num_class)
|
||||||
|
|
||||||
|
parameter_dict = load_checkpoint(args.model_ckpt)
|
||||||
|
load_param_into_net(fasttext_model, parameter_dict=parameter_dict)
|
||||||
|
|
||||||
|
ft_infer = FastTextInferCell(fasttext_model)
|
||||||
|
|
||||||
|
model = Model(ft_infer)
|
||||||
|
|
||||||
|
predictions = []
|
||||||
|
target_sens = []
|
||||||
|
|
||||||
|
for batch in dataset.create_dict_iterator(output_numpy=True, num_epochs=1):
|
||||||
|
target_sens.append(batch['label_idx'])
|
||||||
|
src_tokens = Tensor(batch['src_tokens'], mstype.int32)
|
||||||
|
src_tokens_length = Tensor(batch['src_tokens_length'], mstype.int32)
|
||||||
|
predicted_idx = model.predict(src_tokens, src_tokens_length)
|
||||||
|
predictions.append(predicted_idx.asnumpy())
|
||||||
|
|
||||||
|
from sklearn.metrics import accuracy_score, classification_report
|
||||||
|
target_sens = np.array(target_sens).flatten()
|
||||||
|
predictions = np.array(predictions).flatten()
|
||||||
|
acc = accuracy_score(target_sens, predictions)
|
||||||
|
|
||||||
|
result_report = classification_report(target_sens, predictions, target_names=target_label1)
|
||||||
|
print("********Accuracy: ", acc)
|
||||||
|
print(result_report)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
run_fasttext_infer()
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
spacy
|
||||||
|
sklearn
|
||||||
|
en_core_web_lg
|
||||||
@ -0,0 +1,83 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
echo "Please run the scipt as: "
|
||||||
|
echo "sh create_dataset.sh SOURCE_DATASET_PATH DATASET_NAME"
|
||||||
|
echo "for example: sh create_dataset.sh /home/workspace/ag_news_csv ag"
|
||||||
|
echo "DATASET_NAME including ag, dbpedia, and yelp_p"
|
||||||
|
echo "It is better to use absolute path."
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
ulimit -u unlimited
|
||||||
|
get_real_path(){
|
||||||
|
if [ "${1:0:1}" == "/" ]; then
|
||||||
|
echo "$1"
|
||||||
|
else
|
||||||
|
echo "$(realpath -m $PWD/$1)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
SOURCE_DATASET_PATH=$(get_real_path $1)
|
||||||
|
DATASET_NAME=$2
|
||||||
|
|
||||||
|
export DEVICE_NUM=1
|
||||||
|
export DEVICE_ID=5
|
||||||
|
export RANK_ID=0
|
||||||
|
export RANK_SIZE=1
|
||||||
|
|
||||||
|
if [ $DATASET_NAME == 'ag' ];
|
||||||
|
then
|
||||||
|
echo "Begin to process ag news data"
|
||||||
|
if [ -d "ag" ];
|
||||||
|
then
|
||||||
|
rm -rf ./ag
|
||||||
|
fi
|
||||||
|
mkdir ./ag
|
||||||
|
cd ./ag || exit
|
||||||
|
echo "start data preprocess for device $DEVICE_ID"
|
||||||
|
python ../../src/dataset.py --train_file $SOURCE_DATASET_PATH/train.csv --test_file $SOURCE_DATASET_PATH/test.csv --class_num 4 --max_len 467 --bucket [64,128,467] --test_bucket [467]
|
||||||
|
cd ..
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $DATASET_NAME == 'dbpedia' ];
|
||||||
|
then
|
||||||
|
echo "Begin to process dbpedia data"
|
||||||
|
if [ -d "dbpedia" ];
|
||||||
|
then
|
||||||
|
rm -rf ./dbpedia
|
||||||
|
fi
|
||||||
|
mkdir ./dbpedia
|
||||||
|
cd ./dbpedia || exit
|
||||||
|
echo "start data preprocess for device $DEVICE_ID"
|
||||||
|
python ../../src/dataset.py --train_file $SOURCE_DATASET_PATH/train.csv --test_file $SOURCE_DATASET_PATH/test.csv --class_num 14 --max_len 3013 --bucket [128,512,3013] --test_bucket [1120]
|
||||||
|
cd ..
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $DATASET_NAME == 'yelp_p' ];
|
||||||
|
then
|
||||||
|
echo "Begin to process ag news data"
|
||||||
|
if [ -d "yelp_p" ];
|
||||||
|
then
|
||||||
|
rm -rf ./yelp_p
|
||||||
|
fi
|
||||||
|
mkdir ./yelp_p
|
||||||
|
cd ./yelp_p || exit
|
||||||
|
echo "start data preprocess for device $DEVICE_ID"
|
||||||
|
python ../../src/dataset.py --train_file $SOURCE_DATASET_PATH/train.csv --test_file $SOURCE_DATASET_PATH/test.csv --class_num 2 --max_len 2955 --bucket [64,128,256,512,2955] --test_bucket [2955]
|
||||||
|
cd ..
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,66 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
echo "Please run the scipt as: "
|
||||||
|
echo "sh run_distributed_train.sh DATASET_PATH RANK_TABLE_PATH"
|
||||||
|
echo "for example: sh run_distributed_train.sh /home/workspace/ag /home/workspace/rank_table_file.json"
|
||||||
|
echo "It is better to use absolute path."
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
get_real_path(){
|
||||||
|
if [ "${1:0:1}" == "/" ]; then
|
||||||
|
echo "$1"
|
||||||
|
else
|
||||||
|
echo "$(realpath -m $PWD/$1)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
DATASET=$(get_real_path $1)
|
||||||
|
echo $DATASET
|
||||||
|
DATANAME=$(basename $DATASET)
|
||||||
|
RANK_TABLE_PATH=$(get_real_path $2)
|
||||||
|
echo $DATANAME
|
||||||
|
if [ ! -d $DATASET ]
|
||||||
|
then
|
||||||
|
echo "Error: DATA_PATH=$DATASET is not a file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
current_exec_path=$(pwd)
|
||||||
|
echo ${current_exec_path}
|
||||||
|
|
||||||
|
export RANK_TABLE_FILE=$RANK_TABLE_PATH
|
||||||
|
|
||||||
|
|
||||||
|
echo $RANK_TABLE_FILE
|
||||||
|
export RANK_SIZE=8
|
||||||
|
export DEVICE_NUM=8
|
||||||
|
|
||||||
|
|
||||||
|
for((i=0;i<=7;i++));
|
||||||
|
do
|
||||||
|
rm -rf ${current_exec_path}/device$i
|
||||||
|
mkdir ${current_exec_path}/device$i
|
||||||
|
cd ${current_exec_path}/device$i
|
||||||
|
cp ../../*.py ./
|
||||||
|
cp -r ../../src ./
|
||||||
|
cp -r ../*.sh ./
|
||||||
|
export RANK_ID=$i
|
||||||
|
export DEVICE_ID=$i
|
||||||
|
echo "start training for rank $i, device $DEVICE_ID"
|
||||||
|
python ../../train.py --data_path $DATASET --data_name $DATANAME > log_fasttext.log 2>&1 &
|
||||||
|
cd ${current_exec_path}
|
||||||
|
done
|
||||||
|
cd ${current_exec_path}
|
||||||
@ -0,0 +1,54 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
echo "Please run the scipt as: "
|
||||||
|
echo "sh run_eval.sh DATASET_PATH DATASET_NAME MODEL_CKPT"
|
||||||
|
echo "for example: sh run_eval.sh /home/workspace/ag/test*.mindrecord ag device0/ckpt0/fasttext-5-118.ckpt"
|
||||||
|
echo "It is better to use absolute path."
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
|
||||||
|
get_real_path(){
|
||||||
|
if [ "${1:0:1}" == "/" ]; then
|
||||||
|
echo "$1"
|
||||||
|
else
|
||||||
|
echo "$(realpath -m $PWD/$1)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
DATASET=$(get_real_path $1)
|
||||||
|
echo $DATASET
|
||||||
|
DATANAME=$2
|
||||||
|
MODEL_CKPT=$(get_real_path $3)
|
||||||
|
ulimit -u unlimited
|
||||||
|
export DEVICE_NUM=1
|
||||||
|
export DEVICE_ID=5
|
||||||
|
export RANK_ID=0
|
||||||
|
export RANK_SIZE=1
|
||||||
|
|
||||||
|
|
||||||
|
if [ -d "eval" ];
|
||||||
|
then
|
||||||
|
rm -rf ./eval
|
||||||
|
fi
|
||||||
|
mkdir ./eval
|
||||||
|
cp ../*.py ./eval
|
||||||
|
cp -r ../src ./eval
|
||||||
|
cp -r ../scripts/*.sh ./eval
|
||||||
|
cd ./eval || exit
|
||||||
|
echo "start training for device $DEVICE_ID"
|
||||||
|
env > env.log
|
||||||
|
python ../../eval.py --data_path $DATASET --data_name $DATANAME --model_ckpt $MODEL_CKPT> log_fasttext.log 2>&1 &
|
||||||
|
cd ..
|
||||||
@ -0,0 +1,55 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
echo "Please run the scipt as: "
|
||||||
|
echo "sh run_standalone_train.sh DATASET_PATH"
|
||||||
|
echo "for example: sh run_standalone_train.sh /home/workspace/ag"
|
||||||
|
echo "It is better to use absolute path."
|
||||||
|
echo "=============================================================================================================="
|
||||||
|
|
||||||
|
get_real_path(){
|
||||||
|
if [ "${1:0:1}" == "/" ]; then
|
||||||
|
echo "$1"
|
||||||
|
else
|
||||||
|
echo "$(realpath -m $PWD/$1)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
DATASET=$(get_real_path $1)
|
||||||
|
echo $DATASET
|
||||||
|
DATANAME=$(basename $DATASET)
|
||||||
|
echo $DATANAME
|
||||||
|
|
||||||
|
ulimit -u unlimited
|
||||||
|
export DEVICE_NUM=1
|
||||||
|
export DEVICE_ID=0
|
||||||
|
export RANK_ID=0
|
||||||
|
export RANK_SIZE=1
|
||||||
|
|
||||||
|
|
||||||
|
if [ -d "train" ];
|
||||||
|
then
|
||||||
|
rm -rf ./train
|
||||||
|
fi
|
||||||
|
mkdir ./train
|
||||||
|
cp ../*.py ./train
|
||||||
|
cp -r ../src ./train
|
||||||
|
cp -r ../scripts/*.sh ./train
|
||||||
|
cd ./train || exit
|
||||||
|
echo "start training for device $DEVICE_ID"
|
||||||
|
env > env.log
|
||||||
|
python train.py --data_path $DATASET --data_name $DATANAME > log_fasttext.log 2>&1 &
|
||||||
|
cd ..
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#" :===========================================================================
|
||||||
|
"""
|
||||||
|
network config setting, will be used in train.py and eval.py
|
||||||
|
"""
|
||||||
|
from easydict import EasyDict as ed
|
||||||
|
|
||||||
|
config_yelpp = ed({
|
||||||
|
'vocab_size': 6414979,
|
||||||
|
'buckets': [64, 128, 256, 512, 2955],
|
||||||
|
'batch_size': 128,
|
||||||
|
'embedding_dims': 16,
|
||||||
|
'num_class': 2,
|
||||||
|
'epoch': 5,
|
||||||
|
'lr': 0.02,
|
||||||
|
'min_lr': 1e-6,
|
||||||
|
'decay_steps': 549,
|
||||||
|
'warmup_steps': 400000,
|
||||||
|
'poly_lr_scheduler_power': 0.5,
|
||||||
|
'epoch_count': 1,
|
||||||
|
'pretrain_ckpt_dir': None,
|
||||||
|
'save_ckpt_steps': 549,
|
||||||
|
'keep_ckpt_max': 10,
|
||||||
|
})
|
||||||
|
|
||||||
|
config_db = ed({
|
||||||
|
'vocab_size': 6596536,
|
||||||
|
'buckets': [128, 512, 3013],
|
||||||
|
'batch_size': 128,
|
||||||
|
'embedding_dims': 16,
|
||||||
|
'num_class': 14,
|
||||||
|
'epoch': 5,
|
||||||
|
'lr': 0.05,
|
||||||
|
'min_lr': 1e-6,
|
||||||
|
'decay_steps': 549,
|
||||||
|
'warmup_steps': 400000,
|
||||||
|
'poly_lr_scheduler_power': 0.5,
|
||||||
|
'epoch_count': 1,
|
||||||
|
'pretrain_ckpt_dir': None,
|
||||||
|
'save_ckpt_steps': 548,
|
||||||
|
'keep_ckpt_max': 10,
|
||||||
|
})
|
||||||
|
|
||||||
|
config_ag = ed({
|
||||||
|
'vocab_size': 1383812,
|
||||||
|
'buckets': [64, 128, 467],
|
||||||
|
'batch_size': 128,
|
||||||
|
'embedding_dims': 16,
|
||||||
|
'num_class': 4,
|
||||||
|
'epoch': 5,
|
||||||
|
'lr': 0.05,
|
||||||
|
'min_lr': 1e-6,
|
||||||
|
'decay_steps': 115,
|
||||||
|
'warmup_steps': 400000,
|
||||||
|
'poly_lr_scheduler_power': 0.5,
|
||||||
|
'epoch_count': 1,
|
||||||
|
'pretrain_ckpt_dir': None,
|
||||||
|
'save_ckpt_steps': 116,
|
||||||
|
'keep_ckpt_max': 10,
|
||||||
|
})
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,70 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
"""FastText model."""
|
||||||
|
from mindspore import nn
|
||||||
|
from mindspore.ops import operations as P
|
||||||
|
from mindspore.common.initializer import XavierUniform
|
||||||
|
from mindspore.common import dtype as mstype
|
||||||
|
|
||||||
|
class FastText(nn.Cell):
|
||||||
|
"""
|
||||||
|
FastText model
|
||||||
|
Args:
|
||||||
|
|
||||||
|
vocab_size: vocabulary size
|
||||||
|
embedding_dims: The size of each embedding vector
|
||||||
|
num_class: number of labels
|
||||||
|
"""
|
||||||
|
def __init__(self, vocab_size, embedding_dims, num_class):
|
||||||
|
super(FastText, self).__init__()
|
||||||
|
self.vocab_size = vocab_size
|
||||||
|
self.embeding_dims = embedding_dims
|
||||||
|
self.num_class = num_class
|
||||||
|
self.embeding_func = nn.Embedding(vocab_size=self.vocab_size,
|
||||||
|
embedding_size=self.embeding_dims,
|
||||||
|
padding_idx=0, embedding_table='Zeros')
|
||||||
|
self.fc = nn.Dense(self.embeding_dims, out_channels=self.num_class,
|
||||||
|
weight_init=XavierUniform(1)).to_float(mstype.float16)
|
||||||
|
self.reducesum = P.ReduceSum()
|
||||||
|
self.expand_dims = P.ExpandDims()
|
||||||
|
self.squeeze = P.Squeeze(axis=1)
|
||||||
|
self.cast = P.Cast()
|
||||||
|
self.tile = P.Tile()
|
||||||
|
self.realdiv = P.RealDiv()
|
||||||
|
self.fill = P.Fill()
|
||||||
|
self.log_softmax = nn.LogSoftmax(axis=1)
|
||||||
|
def construct(self, src_tokens, src_token_length):
|
||||||
|
"""
|
||||||
|
construct network
|
||||||
|
Args:
|
||||||
|
|
||||||
|
src_tokens: source sentences
|
||||||
|
src_token_length: source sentences length
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple[Tensor], network outputs
|
||||||
|
"""
|
||||||
|
src_tokens = self.embeding_func(src_tokens)
|
||||||
|
embeding = self.reducesum(src_tokens, 1)
|
||||||
|
|
||||||
|
length_tiled = self.tile(src_token_length, (1, self.embeding_dims))
|
||||||
|
|
||||||
|
embeding = self.realdiv(embeding, length_tiled)
|
||||||
|
|
||||||
|
embeding = self.cast(embeding, mstype.float16)
|
||||||
|
classifer = self.fc(embeding)
|
||||||
|
classifer = self.cast(classifer, mstype.float32)
|
||||||
|
|
||||||
|
return classifer
|
||||||
@ -0,0 +1,142 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
"""FastText for train"""
|
||||||
|
import mindspore.common.dtype as mstype
|
||||||
|
from mindspore.ops import operations as P
|
||||||
|
from mindspore.ops import functional as F
|
||||||
|
from mindspore.ops import composite as C
|
||||||
|
from mindspore.common.parameter import ParameterTuple
|
||||||
|
from mindspore.context import ParallelMode
|
||||||
|
from mindspore import nn
|
||||||
|
from mindspore.communication.management import get_group_size
|
||||||
|
from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
|
||||||
|
from mindspore import context
|
||||||
|
from src.fasttext_model import FastText
|
||||||
|
|
||||||
|
|
||||||
|
GRADIENT_CLIP_TYPE = 1
|
||||||
|
GRADIENT_CLIP_VALUE = 1.0
|
||||||
|
|
||||||
|
clip_grad = C.MultitypeFuncGraph("clip_grad")
|
||||||
|
|
||||||
|
|
||||||
|
@clip_grad.register("Number", "Number", "Tensor")
|
||||||
|
def _clip_grad(clip_type, clip_value, grad):
|
||||||
|
"""
|
||||||
|
Clip gradients.
|
||||||
|
|
||||||
|
Inputs:
|
||||||
|
clip_type (int): The way to clip, 0 for 'value', 1 for 'norm'.
|
||||||
|
clip_value (float): Specifies how much to clip.
|
||||||
|
grad (tuple[Tensor]): Gradients.
|
||||||
|
|
||||||
|
Outputs:
|
||||||
|
tuple[Tensor], clipped gradients.
|
||||||
|
"""
|
||||||
|
if clip_type not in (0, 1):
|
||||||
|
return grad
|
||||||
|
dt = F.dtype(grad)
|
||||||
|
if clip_type == 0:
|
||||||
|
new_grad = C.clip_by_value(grad, F.cast(F.tuple_to_array((-clip_value,)), dt),
|
||||||
|
F.cast(F.tuple_to_array((clip_value,)), dt))
|
||||||
|
else:
|
||||||
|
new_grad = nn.ClipByNorm()(grad, F.cast(F.tuple_to_array((clip_value,)), dt))
|
||||||
|
return new_grad
|
||||||
|
|
||||||
|
class FastTextNetWithLoss(nn.Cell):
|
||||||
|
"""
|
||||||
|
Provide FastText training loss
|
||||||
|
|
||||||
|
Args:
|
||||||
|
vocab_size: vocabulary size
|
||||||
|
embedding_dims: The size of each embedding vector
|
||||||
|
num_class: number of labels
|
||||||
|
"""
|
||||||
|
def __init__(self, vocab_size, embedding_dims, num_class):
|
||||||
|
super(FastTextNetWithLoss, self).__init__()
|
||||||
|
self.fasttext = FastText(vocab_size, embedding_dims, num_class)
|
||||||
|
self.loss_func = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
|
||||||
|
self.squeeze = P.Squeeze(axis=1)
|
||||||
|
self.print = P.Print()
|
||||||
|
|
||||||
|
def construct(self, src_tokens, src_tokens_lengths, label_idx):
|
||||||
|
"""
|
||||||
|
FastText network with loss.
|
||||||
|
"""
|
||||||
|
predict_score = self.fasttext(src_tokens, src_tokens_lengths)
|
||||||
|
label_idx = self.squeeze(label_idx)
|
||||||
|
predict_score = self.loss_func(predict_score, label_idx)
|
||||||
|
|
||||||
|
return predict_score
|
||||||
|
|
||||||
|
|
||||||
|
class FastTextTrainOneStepCell(nn.Cell):
|
||||||
|
"""
|
||||||
|
Encapsulation class of fasttext network training.
|
||||||
|
|
||||||
|
Append an optimizer to the training network after that the construct
|
||||||
|
function can be called to create the backward graph.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
network (Cell): The training network. Note that loss function should have been added.
|
||||||
|
optimizer (Optimizer): Optimizer for updating the weights.
|
||||||
|
sens (Number): The adjust parameter. Default: 1.0.
|
||||||
|
"""
|
||||||
|
def __init__(self, network, optimizer, sens=1.0):
|
||||||
|
super(FastTextTrainOneStepCell, self).__init__(auto_prefix=False)
|
||||||
|
self.network = network
|
||||||
|
self.weights = ParameterTuple(network.trainable_params())
|
||||||
|
self.optimizer = optimizer
|
||||||
|
self.grad = C.GradOperation(get_by_list=True, sens_param=True)
|
||||||
|
self.sens = sens
|
||||||
|
self.reducer_flag = False
|
||||||
|
self.parallel_mode = context.get_auto_parallel_context("parallel_mode")
|
||||||
|
if self.parallel_mode not in ParallelMode.MODE_LIST:
|
||||||
|
raise ValueError("Parallel mode does not support: ", self.parallel_mode)
|
||||||
|
if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]:
|
||||||
|
self.reducer_flag = True
|
||||||
|
self.grad_reducer = None
|
||||||
|
if self.reducer_flag:
|
||||||
|
mean = context.get_auto_parallel_context("gradients_mean")
|
||||||
|
degree = get_group_size()
|
||||||
|
self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree)
|
||||||
|
|
||||||
|
self.hyper_map = C.HyperMap()
|
||||||
|
self.cast = P.Cast()
|
||||||
|
|
||||||
|
def set_sens(self, value):
|
||||||
|
self.sens = value
|
||||||
|
|
||||||
|
def construct(self,
|
||||||
|
src_token_text,
|
||||||
|
src_tokens_text_length,
|
||||||
|
label_idx_tag):
|
||||||
|
"""Defines the computation performed."""
|
||||||
|
weights = self.weights
|
||||||
|
loss = self.network(src_token_text,
|
||||||
|
src_tokens_text_length,
|
||||||
|
label_idx_tag)
|
||||||
|
grads = self.grad(self.network, weights)(src_token_text,
|
||||||
|
src_tokens_text_length,
|
||||||
|
label_idx_tag,
|
||||||
|
self.cast(F.tuple_to_array((self.sens,)),
|
||||||
|
mstype.float32))
|
||||||
|
grads = self.hyper_map(F.partial(clip_grad, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE), grads)
|
||||||
|
if self.reducer_flag:
|
||||||
|
# apply grad reducer on grads
|
||||||
|
grads = self.grad_reducer(grads)
|
||||||
|
|
||||||
|
succ = self.optimizer(grads)
|
||||||
|
return F.depend(loss, succ)
|
||||||
@ -0,0 +1,62 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
"""FastText data loader"""
|
||||||
|
import mindspore.common.dtype as mstype
|
||||||
|
import mindspore.dataset.engine as de
|
||||||
|
import mindspore.dataset.transforms.c_transforms as deC
|
||||||
|
|
||||||
|
def load_dataset(dataset_path,
|
||||||
|
batch_size,
|
||||||
|
epoch_count=1,
|
||||||
|
rank_size=1,
|
||||||
|
rank_id=0,
|
||||||
|
bucket=None,
|
||||||
|
shuffle=True):
|
||||||
|
"""dataset loader"""
|
||||||
|
def batch_per_bucket(bucket_length, input_file):
|
||||||
|
input_file = input_file +'/train_dataset_bs_' + str(bucket_length) + '.mindrecord'
|
||||||
|
if not input_file:
|
||||||
|
raise FileNotFoundError("input file parameter must not be empty.")
|
||||||
|
|
||||||
|
ds = de.MindDataset(input_file,
|
||||||
|
columns_list=['src_tokens', 'src_tokens_length', 'label_idx'],
|
||||||
|
shuffle=shuffle,
|
||||||
|
num_shards=rank_size,
|
||||||
|
shard_id=rank_id,
|
||||||
|
num_parallel_workers=8)
|
||||||
|
ori_dataset_size = ds.get_dataset_size()
|
||||||
|
print(f"Dataset size: {ori_dataset_size}")
|
||||||
|
repeat_count = epoch_count
|
||||||
|
type_cast_op = deC.TypeCast(mstype.int32)
|
||||||
|
ds = ds.map(operations=type_cast_op, input_columns="src_tokens")
|
||||||
|
ds = ds.map(operations=type_cast_op, input_columns="src_tokens_length")
|
||||||
|
ds = ds.map(operations=type_cast_op, input_columns="label_idx")
|
||||||
|
|
||||||
|
ds = ds.rename(input_columns=['src_tokens', 'src_tokens_length', 'label_idx'],
|
||||||
|
output_columns=['src_token_text', 'src_tokens_text_length', 'label_idx_tag'])
|
||||||
|
ds = ds.batch(batch_size, drop_remainder=False)
|
||||||
|
ds = ds.repeat(repeat_count)
|
||||||
|
return ds
|
||||||
|
for i, _ in enumerate(bucket):
|
||||||
|
bucket_len = bucket[i]
|
||||||
|
ds_per = batch_per_bucket(bucket_len, dataset_path)
|
||||||
|
if i == 0:
|
||||||
|
ds = ds_per
|
||||||
|
else:
|
||||||
|
ds = ds + ds_per
|
||||||
|
ds = ds.shuffle(ds.get_dataset_size())
|
||||||
|
ds.channel_name = 'fasttext'
|
||||||
|
|
||||||
|
return ds
|
||||||
@ -0,0 +1,54 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
"""Learning rate utilities."""
|
||||||
|
from math import ceil
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
def polynomial_decay_scheduler(lr, min_lr, decay_steps, total_update_num, warmup_steps=1000, power=1.0):
|
||||||
|
"""
|
||||||
|
Implements of polynomial decay learning rate scheduler which cycles by default.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
lr (float): Initial learning rate.
|
||||||
|
warmup_steps (int): Warmup steps.
|
||||||
|
decay_steps (int): Decay steps.
|
||||||
|
total_update_num (int): Total update steps.
|
||||||
|
min_lr (float): Min learning.
|
||||||
|
power (float): Power factor.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
np.ndarray, learning rate of each step.
|
||||||
|
"""
|
||||||
|
lrs = np.zeros(shape=total_update_num, dtype=np.float32)
|
||||||
|
|
||||||
|
if decay_steps <= 0:
|
||||||
|
raise ValueError("`decay_steps` must larger than 1.")
|
||||||
|
|
||||||
|
_start_step = 0
|
||||||
|
if 0 < warmup_steps < total_update_num:
|
||||||
|
warmup_end_lr = lr
|
||||||
|
warmup_init_lr = 0 if warmup_steps > 0 else warmup_end_lr
|
||||||
|
lrs[:warmup_steps] = np.linspace(warmup_init_lr, warmup_end_lr, warmup_steps)
|
||||||
|
_start_step = warmup_steps
|
||||||
|
|
||||||
|
decay_steps = decay_steps
|
||||||
|
for step in range(_start_step, total_update_num):
|
||||||
|
_step = step - _start_step
|
||||||
|
ratio = ceil(_step / decay_steps)
|
||||||
|
ratio = 1 if ratio < 1 else ratio
|
||||||
|
_decay_steps = decay_steps * ratio
|
||||||
|
lrs[step] = (lr - min_lr) * pow(1 - _step / _decay_steps, power) + min_lr
|
||||||
|
|
||||||
|
return lrs
|
||||||
@ -0,0 +1,202 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
"""FastText for train"""
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import argparse
|
||||||
|
from mindspore import context
|
||||||
|
from mindspore.nn.optim import Adam
|
||||||
|
from mindspore.common import set_seed
|
||||||
|
from mindspore.train.model import Model
|
||||||
|
import mindspore.common.dtype as mstype
|
||||||
|
from mindspore.common.tensor import Tensor
|
||||||
|
from mindspore.context import ParallelMode
|
||||||
|
from mindspore.train.callback import Callback, TimeMonitor
|
||||||
|
from mindspore.communication import management as MultiAscend
|
||||||
|
from mindspore.train.callback import CheckpointConfig, ModelCheckpoint
|
||||||
|
from mindspore.train.serialization import load_checkpoint, load_param_into_net
|
||||||
|
from src.load_dataset import load_dataset
|
||||||
|
from src.lr_schedule import polynomial_decay_scheduler
|
||||||
|
from src.fasttext_train import FastTextTrainOneStepCell, FastTextNetWithLoss
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--data_path', type=str, required=True, help='FastText input data file path.')
|
||||||
|
parser.add_argument('--data_name', type=str, required=True, default='ag', help='dataset name. eg. ag, dbpedia')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.data_name == "ag":
|
||||||
|
from src.config import config_ag as config
|
||||||
|
elif args.data_name == 'dbpedia':
|
||||||
|
from src.config import config_db as config
|
||||||
|
elif args.data_name == 'yelp_p':
|
||||||
|
from src.config import config_yelpp as config
|
||||||
|
|
||||||
|
def get_ms_timestamp():
|
||||||
|
t = time.time()
|
||||||
|
return int(round(t * 1000))
|
||||||
|
set_seed(5)
|
||||||
|
time_stamp_init = False
|
||||||
|
time_stamp_first = 0
|
||||||
|
rank_id = os.getenv('DEVICE_ID')
|
||||||
|
context.set_context(
|
||||||
|
mode=context.GRAPH_MODE,
|
||||||
|
save_graphs=False,
|
||||||
|
device_target="Ascend")
|
||||||
|
|
||||||
|
class LossCallBack(Callback):
|
||||||
|
"""
|
||||||
|
Monitor the loss in training.
|
||||||
|
|
||||||
|
If the loss is NAN or INF terminating training.
|
||||||
|
|
||||||
|
Note:
|
||||||
|
If per_print_times is 0 do not print loss.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
per_print_times (int): Print loss every times. Default: 1.
|
||||||
|
"""
|
||||||
|
def __init__(self, per_print_times=1, rank_ids=0):
|
||||||
|
super(LossCallBack, self).__init__()
|
||||||
|
if not isinstance(per_print_times, int) or per_print_times < 0:
|
||||||
|
raise ValueError("print_step must be int and >= 0.")
|
||||||
|
self._per_print_times = per_print_times
|
||||||
|
self.rank_id = rank_ids
|
||||||
|
global time_stamp_init, time_stamp_first
|
||||||
|
if not time_stamp_init:
|
||||||
|
time_stamp_first = get_ms_timestamp()
|
||||||
|
time_stamp_init = True
|
||||||
|
|
||||||
|
def step_end(self, run_context):
|
||||||
|
"""Monitor the loss in training."""
|
||||||
|
global time_stamp_first
|
||||||
|
time_stamp_current = get_ms_timestamp()
|
||||||
|
cb_params = run_context.original_args()
|
||||||
|
print("time: {}, epoch: {}, step: {}, outputs are {}".format(time_stamp_current - time_stamp_first,
|
||||||
|
cb_params.cur_epoch_num,
|
||||||
|
cb_params.cur_step_num,
|
||||||
|
str(cb_params.net_outputs)))
|
||||||
|
with open("./loss_{}.log".format(self.rank_id), "a+") as f:
|
||||||
|
f.write("time: {}, epoch: {}, step: {}, loss: {}".format(
|
||||||
|
time_stamp_current - time_stamp_first,
|
||||||
|
cb_params.cur_epoch_num,
|
||||||
|
cb_params.cur_step_num,
|
||||||
|
str(cb_params.net_outputs.asnumpy())))
|
||||||
|
f.write('\n')
|
||||||
|
|
||||||
|
|
||||||
|
def _build_training_pipeline(pre_dataset):
|
||||||
|
"""
|
||||||
|
Build training pipeline
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pre_dataset: preprocessed dataset
|
||||||
|
"""
|
||||||
|
net_with_loss = FastTextNetWithLoss(config.vocab_size, config.embedding_dims, config.num_class)
|
||||||
|
net_with_loss.init_parameters_data()
|
||||||
|
if config.pretrain_ckpt_dir:
|
||||||
|
parameter_dict = load_checkpoint(config.pretrain_ckpt_dir)
|
||||||
|
load_param_into_net(net_with_loss, parameter_dict)
|
||||||
|
if pre_dataset is None:
|
||||||
|
raise ValueError("pre-process dataset must be provided")
|
||||||
|
|
||||||
|
#get learning rate
|
||||||
|
update_steps = config.epoch * pre_dataset.get_dataset_size()
|
||||||
|
decay_steps = pre_dataset.get_dataset_size()
|
||||||
|
rank_size = os.getenv("RANK_SIZE")
|
||||||
|
if isinstance(rank_size, int):
|
||||||
|
raise ValueError("RANK_SIZE must be integer")
|
||||||
|
if rank_size is not None and int(rank_size) > 1:
|
||||||
|
base_lr = config.lr
|
||||||
|
else:
|
||||||
|
base_lr = config.lr / 10
|
||||||
|
print("+++++++++++Total update steps ", update_steps)
|
||||||
|
lr = Tensor(polynomial_decay_scheduler(lr=base_lr,
|
||||||
|
min_lr=config.min_lr,
|
||||||
|
decay_steps=decay_steps,
|
||||||
|
total_update_num=update_steps,
|
||||||
|
warmup_steps=config.warmup_steps,
|
||||||
|
power=config.poly_lr_scheduler_power), dtype=mstype.float32)
|
||||||
|
optimizer = Adam(net_with_loss.trainable_params(), lr, beta1=0.9, beta2=0.999)
|
||||||
|
|
||||||
|
net_with_grads = FastTextTrainOneStepCell(net_with_loss, optimizer=optimizer)
|
||||||
|
net_with_grads.set_train(True)
|
||||||
|
model = Model(net_with_grads)
|
||||||
|
loss_monitor = LossCallBack(rank_ids=rank_id)
|
||||||
|
dataset_size = pre_dataset.get_dataset_size()
|
||||||
|
time_monitor = TimeMonitor(data_size=dataset_size)
|
||||||
|
ckpt_config = CheckpointConfig(save_checkpoint_steps=decay_steps,
|
||||||
|
keep_checkpoint_max=config.keep_ckpt_max)
|
||||||
|
callbacks = [time_monitor, loss_monitor]
|
||||||
|
if rank_size is None or int(rank_size) == 1:
|
||||||
|
ckpt_callback = ModelCheckpoint(prefix='fasttext',
|
||||||
|
directory=os.path.join('./', 'ckpe_{}'.format(os.getenv("DEVICE_ID"))),
|
||||||
|
config=ckpt_config)
|
||||||
|
callbacks.append(ckpt_callback)
|
||||||
|
if rank_size is not None and int(rank_size) > 1 and MultiAscend.get_rank() % 8 == 0:
|
||||||
|
ckpt_callback = ModelCheckpoint(prefix='fasttext',
|
||||||
|
directory=os.path.join('./', 'ckpe_{}'.format(os.getenv("DEVICE_ID"))),
|
||||||
|
config=ckpt_config)
|
||||||
|
callbacks.append(ckpt_callback)
|
||||||
|
print("Prepare to Training....")
|
||||||
|
epoch_size = pre_dataset.get_repeat_count()
|
||||||
|
print("Epoch size ", epoch_size)
|
||||||
|
if os.getenv("RANK_SIZE") is not None and int(os.getenv("RANK_SIZE")) > 1:
|
||||||
|
print(f" | Rank {MultiAscend.get_rank()} Call model train.")
|
||||||
|
model.train(epoch=config.epoch, train_dataset=pre_dataset, callbacks=callbacks, dataset_sink_mode=False)
|
||||||
|
|
||||||
|
|
||||||
|
def train_single(input_file_path):
|
||||||
|
"""
|
||||||
|
Train model on single device
|
||||||
|
Args:
|
||||||
|
input_file_path: preprocessed dataset path
|
||||||
|
"""
|
||||||
|
print("Staring training on single device.")
|
||||||
|
preprocessed_data = load_dataset(dataset_path=input_file_path,
|
||||||
|
batch_size=config.batch_size,
|
||||||
|
epoch_count=config.epoch_count,
|
||||||
|
bucket=config.buckets)
|
||||||
|
_build_training_pipeline(preprocessed_data)
|
||||||
|
|
||||||
|
|
||||||
|
def set_parallel_env():
|
||||||
|
context.reset_auto_parallel_context()
|
||||||
|
MultiAscend.init()
|
||||||
|
context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL,
|
||||||
|
device_num=MultiAscend.get_group_size(),
|
||||||
|
gradients_mean=True)
|
||||||
|
def train_paralle(input_file_path):
|
||||||
|
"""
|
||||||
|
Train model on multi device
|
||||||
|
Args:
|
||||||
|
input_file_path: preprocessed dataset path
|
||||||
|
"""
|
||||||
|
set_parallel_env()
|
||||||
|
print("Starting traning on mutiple devices. |~ _ ~| |~ _ ~| |~ _ ~| |~ _ ~|")
|
||||||
|
preprocessed_data = load_dataset(dataset_path=input_file_path,
|
||||||
|
batch_size=config.batch_size,
|
||||||
|
epoch_count=config.epoch_count,
|
||||||
|
rank_size=MultiAscend.get_group_size(),
|
||||||
|
rank_id=MultiAscend.get_rank(),
|
||||||
|
bucket=config.buckets,
|
||||||
|
shuffle=False)
|
||||||
|
_build_training_pipeline(preprocessed_data)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
_rank_size = os.getenv("RANK_SIZE")
|
||||||
|
if _rank_size is not None and int(_rank_size) > 1:
|
||||||
|
train_paralle(args.data_path)
|
||||||
|
else:
|
||||||
|
train_single(args.data_path)
|
||||||
Loading…
Reference in new issue