!8727 Clean code for gnmt_v2 network
From: @gaojing22 Reviewed-by: @yingjy,@guoqi1024 Signed-off-by: @yingjypull/8727/MERGE
mindspore-ci-bot
4 years ago
committed by
Gitee
commit
7689062c7d
@ -0,0 +1,109 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""Train and eval api."""
|
||||
import os
|
||||
import argparse
|
||||
import pickle
|
||||
import datetime
|
||||
|
||||
import mindspore.common.dtype as mstype
|
||||
from mindspore.common import set_seed
|
||||
|
||||
from config import GNMTConfig
|
||||
from train import train_parallel
|
||||
from src.gnmt_model import infer
|
||||
from src.gnmt_model.bleu_calculate import bleu_calculate
|
||||
from src.dataset.tokenizer import Tokenizer
|
||||
|
||||
parser = argparse.ArgumentParser(description='GNMT train and eval.')
|
||||
# train
|
||||
parser.add_argument("--config_train", type=str, required=True,
|
||||
help="model config json file path.")
|
||||
parser.add_argument("--dataset_schema_train", type=str, required=True,
|
||||
help="dataset schema for train.")
|
||||
parser.add_argument("--pre_train_dataset", type=str, required=True,
|
||||
help="pre-train dataset address.")
|
||||
# eval
|
||||
parser.add_argument("--config_test", type=str, required=True,
|
||||
help="model config json file path.")
|
||||
parser.add_argument("--dataset_schema_test", type=str, required=True,
|
||||
help="dataset schema for evaluation.")
|
||||
parser.add_argument("--test_dataset", type=str, required=True,
|
||||
help="test dataset address.")
|
||||
parser.add_argument("--existed_ckpt", type=str, required=True,
|
||||
help="existed checkpoint address.")
|
||||
parser.add_argument("--vocab", type=str, required=True,
|
||||
help="Vocabulary to use.")
|
||||
parser.add_argument("--bpe_codes", type=str, required=True,
|
||||
help="bpe codes to use.")
|
||||
parser.add_argument("--test_tgt", type=str, required=True,
|
||||
default=None,
|
||||
help="data file of the test target")
|
||||
parser.add_argument("--output", type=str, required=False,
|
||||
default="./output.npz",
|
||||
help="result file path.")
|
||||
|
||||
|
||||
def get_config(config):
|
||||
config = GNMTConfig.from_json_file(config)
|
||||
config.compute_type = mstype.float16
|
||||
config.dtype = mstype.float32
|
||||
return config
|
||||
|
||||
|
||||
def _check_args(config):
|
||||
if not os.path.exists(config):
|
||||
raise FileNotFoundError("`config` is not existed.")
|
||||
if not isinstance(config, str):
|
||||
raise ValueError("`config` must be type of str.")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_time = datetime.datetime.now()
|
||||
_rank_size = os.getenv('RANK_SIZE')
|
||||
args, _ = parser.parse_known_args()
|
||||
# train
|
||||
_check_args(args.config_train)
|
||||
_config_train = get_config(args.config_train)
|
||||
_config_train.dataset_schema = args.dataset_schema_train
|
||||
_config_train.pre_train_dataset = args.pre_train_dataset
|
||||
set_seed(_config_train.random_seed)
|
||||
assert _rank_size is not None and int(_rank_size) > 1
|
||||
if _rank_size is not None and int(_rank_size) > 1:
|
||||
train_parallel(_config_train)
|
||||
# eval
|
||||
_check_args(args.config_test)
|
||||
_config_test = get_config(args.config_test)
|
||||
_config_test.dataset_schema = args.dataset_schema_test
|
||||
_config_test.test_dataset = args.test_dataset
|
||||
_config_test.existed_ckpt = args.existed_ckpt
|
||||
result = infer(_config_test)
|
||||
|
||||
with open(args.output, "wb") as f:
|
||||
pickle.dump(result, f, 1)
|
||||
|
||||
result_npy_addr = args.output
|
||||
vocab = args.vocab
|
||||
bpe_codes = args.bpe_codes
|
||||
test_tgt = args.test_tgt
|
||||
tokenizer = Tokenizer(vocab, bpe_codes, 'en', 'de')
|
||||
scores = bleu_calculate(tokenizer, result_npy_addr, test_tgt)
|
||||
print(f"BLEU scores is :{scores}")
|
||||
end_time = datetime.datetime.now()
|
||||
cost_time = (end_time - start_time).seconds
|
||||
print(f"Cost time is {cost_time}s.")
|
||||
assert scores >= 23.8
|
||||
assert cost_time < 10800.0
|
||||
print("----done!----")
|
||||
@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
|
||||
echo "=============================================================================================================="
|
||||
echo "Please run the scipt as: "
|
||||
echo "sh run_distributed_train_ascend.sh \
|
||||
GNMT_ADDR RANK_TABLE_ADDR \
|
||||
DATASET_SCHEMA_TRAIN PRE_TRAIN_DATASET \
|
||||
DATASET_SCHEMA_TEST TEST_DATASET EXISTED_CKPT_PATH \
|
||||
VOCAB_ADDR BPE_CODE_ADDR TEST_TARGET"
|
||||
echo "for example:"
|
||||
echo "sh run_distributed_train_ascend.sh \
|
||||
/home/workspace/gnmt_v2 \
|
||||
/home/workspace/rank_table_8p.json \
|
||||
/home/workspace/dataset_menu/train.tok.clean.bpe.32000.en.json \
|
||||
/home/workspace/dataset_menu/train.tok.clean.bpe.32000.en.tfrecord-001-of-001 \
|
||||
/home/workspace/dataset_menu/newstest2014.en.json \
|
||||
/home/workspace/dataset_menu/newstest2014.en.tfrecord-001-of-001 \
|
||||
/home/workspace/gnmt_v2/gnmt-6_3452.ckpt \
|
||||
/home/workspace/wmt16_de_en/vocab.bpe.32000 \
|
||||
/home/workspace/wmt16_de_en/bpe.32000 \
|
||||
/home/workspace/wmt16_de_en/newstest2014.de"
|
||||
echo "It is better to use absolute path."
|
||||
echo "=============================================================================================================="
|
||||
|
||||
GNMT_ADDR=$1
|
||||
RANK_TABLE_ADDR=$2
|
||||
# train dataset addr
|
||||
DATASET_SCHEMA_TRAIN=$3
|
||||
PRE_TRAIN_DATASET=$4
|
||||
# eval dataset addr
|
||||
DATASET_SCHEMA_TEST=$5
|
||||
TEST_DATASET=$6
|
||||
EXISTED_CKPT_PATH=$7
|
||||
VOCAB_ADDR=$8
|
||||
BPE_CODE_ADDR=$9
|
||||
TEST_TARGET=${10}
|
||||
|
||||
current_exec_path=$(pwd)
|
||||
echo ${current_exec_path}
|
||||
|
||||
export RANK_TABLE_FILE=$RANK_TABLE_ADDR
|
||||
export MINDSPORE_HCCL_CONFIG_PATH=$RANK_TABLE_ADDR
|
||||
|
||||
echo $RANK_TABLE_FILE
|
||||
export RANK_SIZE=8
|
||||
export GLOG_v=2
|
||||
|
||||
for((i=0;i<=7;i++));
|
||||
do
|
||||
rm -rf ${current_exec_path}/device$i
|
||||
mkdir ${current_exec_path}/device$i
|
||||
cd ${current_exec_path}/device$i || exit
|
||||
cp ${current_exec_path}/*.py .
|
||||
cp ${GNMT_ADDR}/*.py .
|
||||
cp -r ${GNMT_ADDR}/src .
|
||||
cp -r ${GNMT_ADDR}/config .
|
||||
export RANK_ID=$i
|
||||
export DEVICE_ID=$i
|
||||
python test_gnmt_v2.py \
|
||||
--config_train=${GNMT_ADDR}/config/config.json \
|
||||
--dataset_schema_train=$DATASET_SCHEMA_TRAIN \
|
||||
--pre_train_dataset=$PRE_TRAIN_DATASET \
|
||||
--config_test=${GNMT_ADDR}/config/config_test.json \
|
||||
--dataset_schema_test=$DATASET_SCHEMA_TEST \
|
||||
--test_dataset=$TEST_DATASET \
|
||||
--existed_ckpt=$EXISTED_CKPT_PATH \
|
||||
--vocab=$VOCAB_ADDR \
|
||||
--bpe_codes=$BPE_CODE_ADDR \
|
||||
--test_tgt=$TEST_TARGET > log_gnmt_network${i}.log 2>&1 &
|
||||
cd ${current_exec_path} || exit
|
||||
done
|
||||
cd ${current_exec_path} || exit
|
||||
Loading…
Reference in new issue