# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import sys
SAME = 0
INPUTS = "Inputs"
OUTPUTS = "Outputs"
ATTRS = "Attrs"
ADD = "Add"
DELETE = "Delete"
CHANGE = "Change"
DUPLICABLE = "duplicable"
INTERMEDIATE = "intermediate"
DISPENSABLE = "dispensable"
TYPE = "type"
GENERATED = "generated"
DEFAULT_VALUE = "default_value"
error = False
def diff_vars(origin_vars, new_vars):
global error
var_error = False
var_changed_error_massage = {}
var_added_error_massage = []
var_deleted_error_massage = []
common_vars_name = set(origin_vars.keys()) & set(new_vars.keys())
vars_name_only_in_origin = set(origin_vars.keys()) - set(new_vars.keys())
vars_name_only_in_new = set(new_vars.keys()) - set(origin_vars.keys())
for var_name in common_vars_name:
if cmp(origin_vars.get(var_name), new_vars.get(var_name)) == SAME:
continue
else:
error, var_error = True, True
var_changed_error_massage[var_name] = {}
for arg_name in origin_vars.get(var_name):
new_arg_value = new_vars.get(var_name, {}).get(arg_name)
origin_arg_value = origin_vars.get(var_name, {}).get(arg_name)
if new_arg_value != origin_arg_value:
var_changed_error_massage[var_name][arg_name] = (
origin_arg_value, new_arg_value)
for var_name in vars_name_only_in_origin:
error, var_error = True, True
var_deleted_error_massage.append(var_name)
for var_name in vars_name_only_in_new:
if not new_vars.get(var_name).get(DISPENSABLE):
error, var_error = True, True
var_added_error_massage.append(var_name)
var_diff_message = {}
if var_added_error_massage:
var_diff_message[ADD] = var_added_error_massage
if var_changed_error_massage:
var_diff_message[CHANGE] = var_changed_error_massage
if var_deleted_error_massage:
var_diff_message[DELETE] = var_deleted_error_massage
return var_error, var_diff_message
def diff_attr(ori_attrs, new_attrs):
global error
attr_error = False
attr_changed_error_massage = {}
attr_added_error_massage = []
attr_deleted_error_massage = []
common_attrs = set(ori_attrs.keys()) & set(new_attrs.keys())
attrs_only_in_origin = set(ori_attrs.keys()) - set(new_attrs.keys())
attrs_only_in_new = set(new_attrs.keys()) - set(ori_attrs.keys())
for attr_name in common_attrs:
if cmp(ori_attrs.get(attr_name), new_attrs.get(attr_name)) == SAME:
continue
else:
error, attr_error = True, True
attr_changed_error_massage[attr_name] = {}
for arg_name in ori_attrs.get(attr_name):
new_arg_value = new_attrs.get(attr_name, {}).get(arg_name)
origin_arg_value = ori_attrs.get(attr_name, {}).get(arg_name)
if new_arg_value != origin_arg_value:
attr_changed_error_massage[attr_name][arg_name] = (
origin_arg_value, new_arg_value)
for attr_name in attrs_only_in_origin:
error, attr_error = True, True
attr_deleted_error_massage.append(attr_name)
for attr_name in attrs_only_in_new:
if new_attrs.get(attr_name).get(DEFAULT_VALUE) == None:
error, attr_error = True, True
attr_added_error_massage.append(attr_name)
attr_diff_message = {}
if attr_added_error_massage:
attr_diff_message[ADD] = attr_added_error_massage
if attr_changed_error_massage:
attr_diff_message[CHANGE] = attr_changed_error_massage
if attr_deleted_error_massage:
attr_diff_message[DELETE] = attr_deleted_error_massage
return attr_error, attr_diff_message
def compare_op_desc(origin_op_desc, new_op_desc):
origin = json.loads(origin_op_desc)
new = json.loads(new_op_desc)
error_message = {}
if cmp(origin_op_desc, new_op_desc) == SAME:
return error_message
for op_type in origin:
# no need to compare if the operator is deleted
if op_type not in new:
continue
origin_info = origin.get(op_type, {})
new_info = new.get(op_type, {})
origin_inputs = origin_info.get(INPUTS, {})
new_inputs = new_info.get(INPUTS, {})
ins_error, ins_diff = diff_vars(origin_inputs, new_inputs)
origin_outputs = origin_info.get(OUTPUTS, {})
new_outputs = new_info.get(OUTPUTS, {})
outs_error, outs_diff = diff_vars(origin_outputs, new_outputs)
origin_attrs = origin_info.get(ATTRS, {})
new_attrs = new_info.get(ATTRS, {})
attrs_error, attrs_diff = diff_attr(origin_attrs, new_attrs)
if ins_error:
error_message.setdefault(op_type, {})[INPUTS] = ins_diff
if outs_error:
error_message.setdefault(op_type, {})[OUTPUTS] = outs_diff
if attrs_error:
error_message.setdefault(op_type, {})[ATTRS] = attrs_diff
return error_message
def print_error_message(error_message):
print("Op desc error for the changes of Inputs/Outputs/Attrs of OPs:\n")
for op_name in error_message:
print("For OP '{}':".format(op_name))
# 1. print inputs error message
Inputs_error = error_message.get(op_name, {}).get(INPUTS, {})
for name in Inputs_error.get(ADD, {}):
print(" * The added Input '{}' is not dispensable.".format(name))
for name in Inputs_error.get(DELETE, {}):
print(" * The Input '{}' is deleted.".format(name))
for name in Inputs_error.get(CHANGE, {}):
changed_args = Inputs_error.get(CHANGE, {}).get(name, {})
for arg in changed_args:
ori_value, new_value = changed_args.get(arg)
print(
" * The arg '{}' of Input '{}' is changed: from '{}' to '{}'.".
format(arg, name, ori_value, new_value))
# 2. print outputs error message
Outputs_error = error_message.get(op_name, {}).get(OUTPUTS, {})
for name in Outputs_error.get(ADD, {}):
print(" * The added Output '{}' is not dispensable.".format(name))
for name in Outputs_error.get(DELETE, {}):
print(" * The Output '{}' is deleted.".format(name))
for name in Outputs_error.get(CHANGE, {}):
changed_args = Outputs_error.get(CHANGE, {}).get(name, {})
for arg in changed_args:
ori_value, new_value = changed_args.get(arg)
print(
" * The arg '{}' of Output '{}' is changed: from '{}' to '{}'.".
format(arg, name, ori_value, new_value))
# 3. print attrs error message
attrs_error = error_message.get(op_name, {}).get(ATTRS, {})
for name in attrs_error.get(ADD, {}):
print(" * The added attr '{}' doesn't set default value.".format(
name))
for name in attrs_error.get(DELETE, {}):
print(" * The attr '{}' is deleted.".format(name))
for name in attrs_error.get(CHANGE, {}):
changed_args = attrs_error.get(CHANGE, {}).get(name, {})
for arg in changed_args:
ori_value, new_value = changed_args.get(arg)
print(
" * The arg '{}' of attr '{}' is changed: from '{}' to '{}'.".
format(arg, name, ori_value, new_value))
def print_repeat_process():
print(
"Tips:"
" If you want to repeat the process, please follow these steps:\n"
"\t1. Compile and install paddle from develop branch \n"
"\t2. Run: python tools/print_op_desc.py > OP_DESC_DEV.spec \n"
"\t3. Compile and install paddle from PR branch \n"
"\t4. Run: python tools/print_op_desc.py > OP_DESC_PR.spec \n"
"\t5. Run: python tools/check_op_desc.py OP_DESC_DEV.spec OP_DESC_PR.spec"
)
if len(sys.argv) == 3:
'''
Compare op_desc files generated by branch DEV and branch PR.
And print error message.
'''
with open(sys.argv[1], 'r') as f:
origin_op_desc = f.read()
with open(sys.argv[2], 'r') as f:
new_op_desc = f.read()
error_message = compare_op_desc(origin_op_desc, new_op_desc)
if error:
print("-" * 30)
print_error_message(error_message)
print("-" * 30)
else:
print("Usage: python check_op_desc.py OP_DESC_DEV.spec OP_DESC_PR.spec")