Use allow list instead of white list (#25002)

* use allow list instead of white list, test=develop

* reduce include, test=develop
revert-24981-add_device_attr_for_regulization
Leo Chen 5 years ago committed by GitHub
parent 621b638550
commit 25a4dac4c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -12,14 +12,16 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
#include "paddle/fluid/framework/unused_var_check.h"
#include <gflags/gflags.h> #include <gflags/gflags.h>
#include <glog/logging.h> #include <glog/logging.h>
#include <string> #include <string>
#include <unordered_set> #include <unordered_set>
#include <vector> #include <vector>
#include "paddle/fluid/framework/operator.h" #include "paddle/fluid/framework/operator.h"
#include "paddle/fluid/framework/unused_var_check.h"
#include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/enforce.h"
DEFINE_bool(enable_unused_var_check, false, DEFINE_bool(enable_unused_var_check, false,
@ -27,7 +29,7 @@ DEFINE_bool(enable_unused_var_check, false,
"especially for grad operator. It should be in unittest."); "especially for grad operator. It should be in unittest.");
// NOTE(zhiqiu): Currently, there are some operators which involves unused // NOTE(zhiqiu): Currently, there are some operators which involves unused
// inputs and cannot be removed from the white_list below. // inputs and cannot be removed from the allow_list below.
// They can be mainly divided into four categories: // They can be mainly divided into four categories:
// 0: the inputs of which are only used in if branch, or used in cuda kernel but // 0: the inputs of which are only used in if branch, or used in cuda kernel but
// not in cpu kernel; // not in cpu kernel;
@ -35,7 +37,7 @@ DEFINE_bool(enable_unused_var_check, false,
// 2: the inputs of which are used in fused operators. // 2: the inputs of which are used in fused operators.
// The category number is presented in the comments after each operator. // The category number is presented in the comments after each operator.
const std::unordered_set<std::string> op_has_unsed_vars_white_list = { const std::unordered_set<std::string> op_with_unsed_vars_allow_list = {
"batch_norm", // 0 "batch_norm", // 0
"batch_norm_grad", // 0 "batch_norm_grad", // 0
"sync_batch_norm", // 0 "sync_batch_norm", // 0
@ -74,8 +76,8 @@ void LogVarUsageIfUnusedVarCheckEnabled(const std::string &name) {
} }
void CheckUnusedVar(const OperatorBase &op, const Scope &scope) { void CheckUnusedVar(const OperatorBase &op, const Scope &scope) {
// skip op in white list and it should be fixed in the future. // skip op in allow list.
if (op_has_unsed_vars_white_list.count(op.Type()) != 0) { if (op_with_unsed_vars_allow_list.count(op.Type()) != 0) {
return; return;
} }
auto *used_set = GetThreadLocalUsedVarNameSet(); auto *used_set = GetThreadLocalUsedVarNameSet();
@ -116,7 +118,7 @@ void CheckUnusedVar(const OperatorBase &op, const Scope &scope) {
"from inputs of the operator; if yes, register " "from inputs of the operator; if yes, register "
"NoNeedBufferVarsInference or add " "NoNeedBufferVarsInference or add "
"the operator to " "the operator to "
"white list in unused_var_check.cc. See more details at " "allow list in unused_var_check.cc. See more details at "
"[https://github.com/PaddlePaddle/Paddle/wiki/" "[https://github.com/PaddlePaddle/Paddle/wiki/"
"OP-Should-Not-Have-Unused-Input]"; "OP-Should-Not-Have-Unused-Input]";
PADDLE_ENFORCE_EQ(unsed_input_var_names.size(), 0, PADDLE_ENFORCE_EQ(unsed_input_var_names.size(), 0,

@ -19,11 +19,13 @@ limitations under the License. */
#include <string> #include <string>
#include <unordered_set> #include <unordered_set>
#include "paddle/fluid/framework/operator.h"
namespace paddle { namespace paddle {
namespace framework { namespace framework {
class OperatorBase;
class Scope;
std::unordered_set<std::string>* GetThreadLocalUsedVarNameSet(); std::unordered_set<std::string>* GetThreadLocalUsedVarNameSet();
void LogVarUsageIfUnusedVarCheckEnabled(const std::string& name); void LogVarUsageIfUnusedVarCheckEnabled(const std::string& name);

@ -111,7 +111,7 @@ for API_FILE in ${API_FILES[*]}; do
echo_line="You must have one RD (gongweibao or seiriosPlus) approval for the paddle/fluid/operators/distributed/send_recv.proto.in, which manages the environment variables.\n" echo_line="You must have one RD (gongweibao or seiriosPlus) approval for the paddle/fluid/operators/distributed/send_recv.proto.in, which manages the environment variables.\n"
check_approval 1 10721757 5442383 check_approval 1 10721757 5442383
elif [ "${API_FILE}" == "paddle/fluid/framework/unused_var_check.cc" ];then elif [ "${API_FILE}" == "paddle/fluid/framework/unused_var_check.cc" ];then
echo_line="You must have one RD (zhiqiu (Recommend) , sneaxiy or luotao1) approval for the changes of paddle/fluid/framework/unused_var_check.cc, which manages the white list of operators that have unused input variables. Before change the white list, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/OP-Should-Not-Have-Unused-Input] and try to refine code first. \n" echo_line="You must have one RD (zhiqiu (Recommend) , sneaxiy or luotao1) approval for the changes of paddle/fluid/framework/unused_var_check.cc, which manages the allow list of operators that have unused input variables. Before change the allow list, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/OP-Should-Not-Have-Unused-Input] and try to refine code first. \n"
check_approval 1 6888866 32832641 6836917 check_approval 1 6888866 32832641 6836917
elif [ "${API_FILE}" == "paddle/fluid/pybind/op_function_generator.cc" ];then elif [ "${API_FILE}" == "paddle/fluid/pybind/op_function_generator.cc" ];then
echo_line="You must have one RD (zhiqiu (Recommend) , phlrain) approval for the changes of paddle/fluid/pybind/op_function_generator.cc, which manages the logic of automatic generating op functions for dygraph. \n" echo_line="You must have one RD (zhiqiu (Recommend) , phlrain) approval for the changes of paddle/fluid/pybind/op_function_generator.cc, which manages the logic of automatic generating op functions for dygraph. \n"

Loading…
Cancel
Save