Add grep enforce shell & polish other enforce count shell (#24897)
* add grep enforce shell, test=develop, test=document_fix * add copyright desc, test=develop, test=document_fix * refactor four scripts, test=develop, test=document_fix * change white list var, test=develop, test=document_fix * add args for count_enforce_by_file, test=develop, test=document_fixrevert-24981-add_device_attr_for_regulization
Chen Weihang
5 years ago
committed by
GitHub
parent
b174b99764
commit
8f416eb1b4
@ -0,0 +1,127 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script is used to grep invalid PADDLE checks by directory or file in the paddle/fluid/,
|
||||
# the result show all invalid PADDLE checks in specified directory or file.
|
||||
|
||||
# Usage:
|
||||
# - bash grep_invalid_enforce.sh [target directory or file] (run in tools directory)
|
||||
# - The default check path is paddle/fluid/operators
|
||||
|
||||
# Result Examples:
|
||||
# 1. grep invalid PADDLE checks in directory
|
||||
|
||||
# - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/imperative
|
||||
# - Results:
|
||||
# - paddle/fluid/imperative/gradient_accumulator.cc
|
||||
# PADDLE_ENFORCE_EQ(dst_tensor->numel() == numel, true,
|
||||
# "dst_numel %d vs. src_numel %d", dst_tensor->numel(),
|
||||
# numel);
|
||||
|
||||
# - paddle/fluid/imperative/nccl_context.cc
|
||||
# PADDLE_ENFORCE_EQ(addr.size(), 2UL,
|
||||
# "The endpoint should contain host and port: %s", ep);
|
||||
# PADDLE_THROW("create server fd failed");
|
||||
# PADDLE_THROW("set socket opt failed");
|
||||
# PADDLE_THROW("binding failed on ep: %s", ep);
|
||||
# PADDLE_THROW("listen on server fd failed");
|
||||
# PADDLE_THROW("accept the new socket fd failed");
|
||||
# PADDLE_THROW("reading the ncclUniqueId from socket failed");
|
||||
# PADDLE_ENFORCE_EQ(addr.size(), 2UL,
|
||||
# "The endpoint should contain host and port: %s", ep);
|
||||
# PADDLE_THROW("create socket failed");
|
||||
# PADDLE_THROW("invalied address: %s", ep);
|
||||
|
||||
# - paddle/fluid/imperative/jit/program_desc_tracer.cc
|
||||
# PADDLE_ENFORCE_NOT_NULL(new_var);
|
||||
# PADDLE_ENFORCE_EQ(inner_var.IsInitialized(), true);
|
||||
# PADDLE_THROW("Not support variable type %s",
|
||||
# framework::ToTypeName(inner_var.Type()));
|
||||
|
||||
# 2. grep invalid PADDLE checks in file
|
||||
|
||||
# - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/pybind/reader_py.cc
|
||||
# - Results:
|
||||
# - paddle/fluid/pybind/reader_py.cc
|
||||
# PADDLE_THROW(
|
||||
# "Place cannot be CUDAPlace when use_double_buffer is False");
|
||||
# PADDLE_ENFORCE_NOT_NULL(exceptions_[i]);
|
||||
# PADDLE_ENFORCE_EQ(status, Status::kException);
|
||||
# PADDLE_ENFORCE_EQ(status, Status::kSuccess);
|
||||
|
||||
. ./count_enforce_by_file.sh --source-only
|
||||
|
||||
ROOT_DIR=../paddle/fluid/operators
|
||||
|
||||
if [ "$1" != "" ]; then
|
||||
ROOT_DIR=$1
|
||||
fi
|
||||
|
||||
function enforce_grep(){
|
||||
paddle_check=`grep -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
|
||||
valid_check=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' || true`
|
||||
invalid_check=`echo "$paddle_check" | grep -vxF "$valid_check" || true`
|
||||
if [ "${invalid_check}" != "" ];then
|
||||
file_path=$1
|
||||
echo -e "\n- ${file_path#../}"
|
||||
echo "${invalid_check}"
|
||||
fi
|
||||
}
|
||||
|
||||
function grep_file_recursively(){
|
||||
local i=0
|
||||
local dir_array
|
||||
for file in `ls $1`
|
||||
do
|
||||
if [ -f $1"/"$file ];then
|
||||
in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}")
|
||||
if [[ "$in_white_list" == "" ]];then
|
||||
enforce_grep $1"/"$file
|
||||
fi
|
||||
fi
|
||||
if [ -d $1"/"$file ];then
|
||||
dir_array[$i]=$1"/"$file
|
||||
((i++))
|
||||
fi
|
||||
done
|
||||
for sub_dir_name in ${dir_array[@]}
|
||||
do
|
||||
grep_file_recursively $sub_dir_name
|
||||
done
|
||||
}
|
||||
|
||||
function grep_file(){
|
||||
file_path=$1
|
||||
file_name=`echo ${file_path##*/} `
|
||||
if [ -f $file_path ];then
|
||||
in_white_list=$(echo $FILE_WHITE_LIST | grep "${file_name}")
|
||||
if [[ "$in_white_list" == "" ]];then
|
||||
enforce_grep $file_path
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
if [ -f $ROOT_DIR ];then
|
||||
grep_file $ROOT_DIR
|
||||
else
|
||||
grep_file_recursively $ROOT_DIR
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "${1}" != "--source-only" ]; then
|
||||
main "${@}"
|
||||
fi
|
||||
Loading…
Reference in new issue