From 748e07eb9e35cec25d95b9e67a1dc2e9e47b47f9 Mon Sep 17 00:00:00 2001 From: liyong Date: Sun, 28 Jun 2020 17:09:56 +0800 Subject: [PATCH] adjust model zoo utils --- example/nlp_to_mindrecord/CLUERNER2020/run.sh | 40 ------------------- model_zoo/gat/scripts/run_process_data.sh | 2 +- model_zoo/gcn/scripts/run_process_data.sh | 2 +- .../Caltech-UCSD-Birds-200-2011/README.md | 0 .../create_dataset.py | 0 .../data/README.md | 0 .../gen_mindrecord.py | 0 .../output/README.md | 0 .../Caltech-UCSD-Birds-200-2011/run.sh | 0 .../Caltech-UCSD-Birds-200-2011/run_read.sh | 0 .../ImageNet_Similar_Perf/README.md | 0 .../imagenet/__init__.py | 0 .../ImageNet_Similar_Perf/imagenet/mr_api.py | 0 .../ImageNet_Similar_Perf/run_imagenet.sh | 0 .../ImageNet_Similar_Perf/run_template.sh | 0 .../template/__init__.py | 0 .../ImageNet_Similar_Perf/template/mr_api.py | 0 .../ImageNet_Similar_Perf/writer.py | 0 .../utils}/cv_to_mindrecord/README.md | 0 .../utils}/graph_to_mindrecord/README.md | 0 .../graph_to_mindrecord/citeseer/__init__.py | 0 .../graph_to_mindrecord/citeseer/mr_api.py | 0 .../graph_to_mindrecord/cora/__init__.py | 0 .../utils}/graph_to_mindrecord/cora/mr_api.py | 0 .../graph_to_mindrecord/graph_map_schema.py | 0 .../graph_to_mindrecord/read_citeseer.sh | 0 .../utils}/graph_to_mindrecord/read_cora.sh | 0 .../utils}/graph_to_mindrecord/reader.py | 0 .../graph_to_mindrecord/sns/__init__.py | 0 .../utils}/graph_to_mindrecord/sns/mr_api.py | 0 .../graph_to_mindrecord/write_citeseer.sh | 0 .../utils}/graph_to_mindrecord/write_cora.sh | 0 .../utils}/graph_to_mindrecord/write_sns.sh | 0 .../utils}/graph_to_mindrecord/writer.py | 0 .../nlp_to_mindrecord/CLUERNER2020/README.md | 0 .../CLUERNER2020/create_dataset.py | 0 .../CLUERNER2020/data/.gitignore | 0 .../CLUERNER2020/data/README.md | 0 .../CLUERNER2020/output/README.md | 0 .../nlp_to_mindrecord/CLUERNER2020/run.sh | 40 +++++++++++++++++++ .../CLUERNER2020/run_read.sh | 0 .../utils}/nlp_to_mindrecord/README.md | 0 .../nlp_to_mindrecord/aclImdb/README.md | 0 .../aclImdb/create_dataset.py | 0 .../nlp_to_mindrecord/aclImdb/data/README.md | 0 .../aclImdb/gen_mindrecord.py | 0 .../aclImdb/output/README.md | 0 .../utils}/nlp_to_mindrecord/aclImdb/run.sh | 0 .../nlp_to_mindrecord/aclImdb/run_read.sh | 0 .../aclImdb_preprocess/README.md | 0 .../aclImdb_preprocess/create_dataset.py | 0 .../aclImdb_preprocess/data/README.md | 0 .../aclImdb_preprocess/gen_mindrecord.py | 0 .../aclImdb_preprocess/output/README.md | 0 .../aclImdb_preprocess/run.sh | 0 .../aclImdb_preprocess/run_read.sh | 0 .../utils}/nlp_to_mindrecord/enwiki/README.md | 0 .../enwiki/create_dataset.py | 0 .../utils}/nlp_to_mindrecord/enwiki/run.sh | 14 +++---- .../nlp_to_mindrecord/enwiki/run_read.sh | 0 .../utils}/nlp_to_mindrecord/zhwiki/README.md | 2 +- .../zhwiki/create_dataset.py | 0 .../nlp_to_mindrecord/zhwiki/data/.gitignore | 0 .../nlp_to_mindrecord/zhwiki/data/README.md | 0 .../nlp_to_mindrecord/zhwiki/output/README.md | 0 .../utils}/nlp_to_mindrecord/zhwiki/run.sh | 16 ++++---- .../nlp_to_mindrecord/zhwiki/run_read.sh | 0 .../zhwiki/run_read_simple.sh | 2 +- .../nlp_to_mindrecord/zhwiki/run_simple.sh | 20 +++++----- 69 files changed, 69 insertions(+), 69 deletions(-) delete mode 100644 example/nlp_to_mindrecord/CLUERNER2020/run.sh rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/README.md (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/README.md (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/citeseer/__init__.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/citeseer/mr_api.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/cora/__init__.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/cora/mr_api.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/graph_map_schema.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/read_citeseer.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/read_cora.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/reader.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/sns/__init__.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/sns/mr_api.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/write_citeseer.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/write_cora.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/write_sns.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/writer.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/data/.gitignore (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/output/README.md (100%) create mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/gen_mindrecord.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/output/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/run.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/output/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/run.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/run.sh (83%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/README.md (98%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/data/.gitignore (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/output/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run.sh (80%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run_read_simple.sh (91%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run_simple.sh (53%) diff --git a/example/nlp_to_mindrecord/CLUERNER2020/run.sh b/example/nlp_to_mindrecord/CLUERNER2020/run.sh deleted file mode 100644 index 15c6aa4362..0000000000 --- a/example/nlp_to_mindrecord/CLUERNER2020/run.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/train.mindrecord* -rm -f output/dev.mindrecord* - -if [ ! -d "../../../third_party/to_mindrecord/CLUERNER2020" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/CLUERNER2020 is not exist." - exit 1 -fi - -if [ ! -f "../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch is not exist." - exit 1 -fi - -# patch for data_processor_seq.py -patch -p0 -d ../../../third_party/to_mindrecord/CLUERNER2020/ -o data_processor_seq_patched.py < ../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py failed" - exit 1 -fi - -# use patched script -python ../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq_patched.py \ ---vocab_file=../../../third_party/to_mindrecord/CLUERNER2020/vocab.txt \ ---label2id_file=../../../third_party/to_mindrecord/CLUERNER2020/label2id.json diff --git a/model_zoo/gat/scripts/run_process_data.sh b/model_zoo/gat/scripts/run_process_data.sh index 4501f3c67f..3bf6672301 100755 --- a/model_zoo/gat/scripts/run_process_data.sh +++ b/model_zoo/gat/scripts/run_process_data.sh @@ -42,7 +42,7 @@ MINDRECORD_PATH=`pwd`/data_mr rm -f $MINDRECORD_PATH/* -cd ../../../example/graph_to_mindrecord || exit +cd ../../utils/graph_to_mindrecord || exit python writer.py --mindrecord_script $DATASET_NAME \ --mindrecord_file "$MINDRECORD_PATH/$DATASET_NAME" \ diff --git a/model_zoo/gcn/scripts/run_process_data.sh b/model_zoo/gcn/scripts/run_process_data.sh index d51d915943..013cf2f28c 100755 --- a/model_zoo/gcn/scripts/run_process_data.sh +++ b/model_zoo/gcn/scripts/run_process_data.sh @@ -43,7 +43,7 @@ MINDRECORD_PATH=`pwd`/data_mr rm -f $MINDRECORD_PATH/$DATASET_NAME rm -f $MINDRECORD_PATH/$DATASET_NAME.db -cd ../../../example/graph_to_mindrecord || exit +cd ../../utils/graph_to_mindrecord || exit python writer.py --mindrecord_script $DATASET_NAME \ --mindrecord_file "$MINDRECORD_PATH/$DATASET_NAME" \ diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/README.md b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/README.md similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/README.md rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/README.md diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py diff --git a/example/cv_to_mindrecord/README.md b/model_zoo/utils/cv_to_mindrecord/README.md similarity index 100% rename from example/cv_to_mindrecord/README.md rename to model_zoo/utils/cv_to_mindrecord/README.md diff --git a/example/graph_to_mindrecord/README.md b/model_zoo/utils/graph_to_mindrecord/README.md similarity index 100% rename from example/graph_to_mindrecord/README.md rename to model_zoo/utils/graph_to_mindrecord/README.md diff --git a/example/graph_to_mindrecord/citeseer/__init__.py b/model_zoo/utils/graph_to_mindrecord/citeseer/__init__.py similarity index 100% rename from example/graph_to_mindrecord/citeseer/__init__.py rename to model_zoo/utils/graph_to_mindrecord/citeseer/__init__.py diff --git a/example/graph_to_mindrecord/citeseer/mr_api.py b/model_zoo/utils/graph_to_mindrecord/citeseer/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/citeseer/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/citeseer/mr_api.py diff --git a/example/graph_to_mindrecord/cora/__init__.py b/model_zoo/utils/graph_to_mindrecord/cora/__init__.py similarity index 100% rename from example/graph_to_mindrecord/cora/__init__.py rename to model_zoo/utils/graph_to_mindrecord/cora/__init__.py diff --git a/example/graph_to_mindrecord/cora/mr_api.py b/model_zoo/utils/graph_to_mindrecord/cora/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/cora/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/cora/mr_api.py diff --git a/example/graph_to_mindrecord/graph_map_schema.py b/model_zoo/utils/graph_to_mindrecord/graph_map_schema.py similarity index 100% rename from example/graph_to_mindrecord/graph_map_schema.py rename to model_zoo/utils/graph_to_mindrecord/graph_map_schema.py diff --git a/example/graph_to_mindrecord/read_citeseer.sh b/model_zoo/utils/graph_to_mindrecord/read_citeseer.sh similarity index 100% rename from example/graph_to_mindrecord/read_citeseer.sh rename to model_zoo/utils/graph_to_mindrecord/read_citeseer.sh diff --git a/example/graph_to_mindrecord/read_cora.sh b/model_zoo/utils/graph_to_mindrecord/read_cora.sh similarity index 100% rename from example/graph_to_mindrecord/read_cora.sh rename to model_zoo/utils/graph_to_mindrecord/read_cora.sh diff --git a/example/graph_to_mindrecord/reader.py b/model_zoo/utils/graph_to_mindrecord/reader.py similarity index 100% rename from example/graph_to_mindrecord/reader.py rename to model_zoo/utils/graph_to_mindrecord/reader.py diff --git a/example/graph_to_mindrecord/sns/__init__.py b/model_zoo/utils/graph_to_mindrecord/sns/__init__.py similarity index 100% rename from example/graph_to_mindrecord/sns/__init__.py rename to model_zoo/utils/graph_to_mindrecord/sns/__init__.py diff --git a/example/graph_to_mindrecord/sns/mr_api.py b/model_zoo/utils/graph_to_mindrecord/sns/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/sns/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/sns/mr_api.py diff --git a/example/graph_to_mindrecord/write_citeseer.sh b/model_zoo/utils/graph_to_mindrecord/write_citeseer.sh similarity index 100% rename from example/graph_to_mindrecord/write_citeseer.sh rename to model_zoo/utils/graph_to_mindrecord/write_citeseer.sh diff --git a/example/graph_to_mindrecord/write_cora.sh b/model_zoo/utils/graph_to_mindrecord/write_cora.sh similarity index 100% rename from example/graph_to_mindrecord/write_cora.sh rename to model_zoo/utils/graph_to_mindrecord/write_cora.sh diff --git a/example/graph_to_mindrecord/write_sns.sh b/model_zoo/utils/graph_to_mindrecord/write_sns.sh similarity index 100% rename from example/graph_to_mindrecord/write_sns.sh rename to model_zoo/utils/graph_to_mindrecord/write_sns.sh diff --git a/example/graph_to_mindrecord/writer.py b/model_zoo/utils/graph_to_mindrecord/writer.py similarity index 100% rename from example/graph_to_mindrecord/writer.py rename to model_zoo/utils/graph_to_mindrecord/writer.py diff --git a/example/nlp_to_mindrecord/CLUERNER2020/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/README.md rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md diff --git a/example/nlp_to_mindrecord/CLUERNER2020/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py diff --git a/example/nlp_to_mindrecord/CLUERNER2020/data/.gitignore b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/data/.gitignore rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore diff --git a/example/nlp_to_mindrecord/CLUERNER2020/data/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md diff --git a/example/nlp_to_mindrecord/CLUERNER2020/output/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh new file mode 100644 index 0000000000..9de6f0e9fc --- /dev/null +++ b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +rm -f output/train.mindrecord* +rm -f output/dev.mindrecord* + +if [ ! -d "../../../../third_party/to_mindrecord/CLUERNER2020" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/CLUERNER2020 is not exist." + exit 1 +fi + +if [ ! -f "../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch is not exist." + exit 1 +fi + +# patch for data_processor_seq.py +patch -p0 -d ../../../../third_party/to_mindrecord/CLUERNER2020/ -o data_processor_seq_patched.py < ../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch +if [ $? -ne 0 ]; then + echo "Patch ../../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py failed" + exit 1 +fi + +# use patched script +python ../../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq_patched.py \ +--vocab_file=../../../../third_party/to_mindrecord/CLUERNER2020/vocab.txt \ +--label2id_file=../../../../third_party/to_mindrecord/CLUERNER2020/label2id.json diff --git a/example/nlp_to_mindrecord/CLUERNER2020/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh diff --git a/example/nlp_to_mindrecord/README.md b/model_zoo/utils/nlp_to_mindrecord/README.md similarity index 100% rename from example/nlp_to_mindrecord/README.md rename to model_zoo/utils/nlp_to_mindrecord/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/create_dataset.py diff --git a/example/nlp_to_mindrecord/aclImdb/data/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/data/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/gen_mindrecord.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb/gen_mindrecord.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/gen_mindrecord.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/gen_mindrecord.py diff --git a/example/nlp_to_mindrecord/aclImdb/output/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/output/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/run.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb/run.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/run.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/run.sh diff --git a/example/nlp_to_mindrecord/aclImdb/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/run_read.sh diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/data/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/data/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/output/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/output/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/run.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/run.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run.sh diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh diff --git a/example/nlp_to_mindrecord/enwiki/README.md b/model_zoo/utils/nlp_to_mindrecord/enwiki/README.md similarity index 100% rename from example/nlp_to_mindrecord/enwiki/README.md rename to model_zoo/utils/nlp_to_mindrecord/enwiki/README.md diff --git a/example/nlp_to_mindrecord/enwiki/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/enwiki/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py diff --git a/example/nlp_to_mindrecord/enwiki/run.sh b/model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh similarity index 83% rename from example/nlp_to_mindrecord/enwiki/run.sh rename to model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh index cf66bed0fd..c36b6b9903 100644 --- a/example/nlp_to_mindrecord/enwiki/run.sh +++ b/model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh @@ -66,20 +66,20 @@ getdir "${data_dir}" # echo "The input files: "${file_list[@]} # echo "The output files: "${output_filename[@]} -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." +if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." exit 1 fi -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." +if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." exit 1 fi # patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch +patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" + echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" exit 1 fi @@ -94,7 +94,7 @@ file_list_len=`expr ${#file_list[*]} - 1` for index in $(seq 0 $file_list_len); do echo "Begin preprocess input file: ${file_list[$index]}" echo "Begin output file: ${output_filename[$index]}" - python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ + python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ --input_file=${file_list[$index]} \ --output_file=${output_dir}/${output_filename[$index]} \ --partition_number=1 \ diff --git a/example/nlp_to_mindrecord/enwiki/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/enwiki/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh diff --git a/example/nlp_to_mindrecord/zhwiki/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md similarity index 98% rename from example/nlp_to_mindrecord/zhwiki/README.md rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md index 1a9de05114..ad7afdd662 100644 --- a/example/nlp_to_mindrecord/zhwiki/README.md +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md @@ -26,7 +26,7 @@ This example is based on [zhwiki](https://dumps.wikimedia.org/zhwiki) training d Follow the step: ```bash -bash run_simple.sh # generate output/simple.mindrecord* by ../../../third_party/to_mindrecord/zhwiki/sample_text.txt +bash run_simple.sh # generate output/simple.mindrecord* by ../../../../third_party/to_mindrecord/zhwiki/sample_text.txt bash run_read_simple.sh # use MindDataset to read output/simple.mindrecord* ``` diff --git a/example/nlp_to_mindrecord/zhwiki/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py diff --git a/example/nlp_to_mindrecord/zhwiki/data/.gitignore b/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/data/.gitignore rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore diff --git a/example/nlp_to_mindrecord/zhwiki/data/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md diff --git a/example/nlp_to_mindrecord/zhwiki/output/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md diff --git a/example/nlp_to_mindrecord/zhwiki/run.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh similarity index 80% rename from example/nlp_to_mindrecord/zhwiki/run.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh index a057031e6b..3142056b11 100644 --- a/example/nlp_to_mindrecord/zhwiki/run.sh +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh @@ -45,20 +45,20 @@ getdir "${data_dir}" # echo "The input files: "${file_list[@]} # echo "The output files: "${output_filename[@]} -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." +if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." exit 1 fi -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." +if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." exit 1 fi # patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch +patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" + echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" exit 1 fi @@ -73,11 +73,11 @@ file_list_len=`expr ${#file_list[*]} - 1` for index in $(seq 0 $file_list_len); do echo "Begin preprocess input file: ${file_list[$index]}" echo "Begin output file: ${output_filename[$index]}" - python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ + python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ --input_file=${file_list[$index]} \ --output_file=output/${output_filename[$index]} \ --partition_number=1 \ - --vocab_file=../../../third_party/to_mindrecord/zhwiki/vocab.txt \ + --vocab_file=../../../../third_party/to_mindrecord/zhwiki/vocab.txt \ --do_lower_case=True \ --max_seq_length=128 \ --max_predictions_per_seq=20 \ diff --git a/example/nlp_to_mindrecord/zhwiki/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh diff --git a/example/nlp_to_mindrecord/zhwiki/run_read_simple.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh similarity index 91% rename from example/nlp_to_mindrecord/zhwiki/run_read_simple.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh index 1c26dec449..abecc20187 100644 --- a/example/nlp_to_mindrecord/zhwiki/run_read_simple.sh +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh @@ -15,4 +15,4 @@ # ============================================================================ # create dataset for train -python create_dataset.py --input_file=output/simple.mindrecord0 +python create_dataset.py --input_file=output/simple.mindrecord diff --git a/example/nlp_to_mindrecord/zhwiki/run_simple.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh similarity index 53% rename from example/nlp_to_mindrecord/zhwiki/run_simple.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh index 20c1d98d66..a41df700b4 100644 --- a/example/nlp_to_mindrecord/zhwiki/run_simple.sh +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh @@ -16,29 +16,29 @@ rm -f output/simple.mindrecord* -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." +if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." exit 1 fi -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." +if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." exit 1 fi # patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch +patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" + echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" exit 1 fi # using patched script to generate mindrecord -python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ ---input_file=../../../third_party/to_mindrecord/zhwiki/sample_text.txt \ +python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ +--input_file=../../../../third_party/to_mindrecord/zhwiki/sample_text.txt \ --output_file=output/simple.mindrecord \ ---partition_number=4 \ ---vocab_file=../../../third_party/to_mindrecord/zhwiki/vocab.txt \ +--partition_number=1 \ +--vocab_file=../../../../third_party/to_mindrecord/zhwiki/vocab.txt \ --do_lower_case=True \ --max_seq_length=128 \ --max_predictions_per_seq=20 \