!14447 show accurate error info and error lines when use unsupported builtin function

From: @zhangbuxue
Reviewed-by: @ginfung,@ginfung,@zh_qh
Signed-off-by: @zh_qh

mindspore/_extends/parse/__init__.py

@@ -21,13 +21,16 @@ from .parser import (Parser, create_obj_instance, generate_scope,
                      get_class_member_namespace_symbol, create_slice_obj,
                      get_dataclass_attributes, get_dataclass_methods, get_obj_id,
                      get_module_namespace, get_obj_type, get_object_key,
+                     get_ast_type, get_node_type, get_args, get_args_default_values,
+                     get_ast_namespace_symbol, get_operation_namespace_symbol,
                      get_parse_method_of_class, get_scope_name, expand_expr_statement,
                      is_class_member, parse_cb, resolve_symbol, convert_to_ms_tensor, get_object_description)
 from .serialize import *
 
 __all__ = ['parse_cb', 'get_parse_method_of_class', 'get_bprop_method_of_class', 'resolve_symbol',
-           'get_object_key', 'get_class_instance_type', 'is_class_member',
-           'get_obj_type', 'get_obj_id', 'create_obj_instance', 'get_module_namespace',
+           'get_object_key', 'get_class_instance_type', 'is_class_member', 'get_ast_type', 'get_node_type',
+           'get_args_default_values', 'get_ast_namespace_symbol', 'get_operation_namespace_symbol',
+           'get_args', 'get_obj_type', 'get_obj_id', 'create_obj_instance', 'get_module_namespace',
            'get_class_member_namespace_symbol', 'get_obj_id', 'Parser', 'get_dataclass_attributes',
            'get_dataclass_methods', 'dump_obj', 'load_obj', 'get_dataclass_methods', 'get_scope_name',
            'create_slice_obj', 'convert_to_ms_tensor', 'get_object_description', 'expand_expr_statement']

mindspore/_extends/parse/parser.py

@@ -371,6 +371,89 @@ def expand_expr_statement(node):
     return (False,)
 
 
+def get_ast_namespace_symbol(obj):
+    """Get obj type and namespace and symbol."""
+    # step 1:get symbol from object map
+    ops_info = parse_object_map.get(type(obj), SYMBOL_UNDEFINE)
+    logger.debug("ops info = %r", ops_info)
+    return ops_info
+
+
+def get_operation_namespace_symbol(var: str):
+    """Get operation namespace and symbol."""
+    ops_info = (trope_ns, var)
+    logger.debug("get operation ops info = %r", ops_info)
+    return ops_info
+
+
+def get_ast_type(node):
+    """Get the ast type."""
+    ast_type = AST_SUB_TYPE_UNKNOWN
+    if isinstance(node, ast.And):
+        ast_type = AST_SUB_TYPE_AND
+    elif isinstance(node, ast.Or):
+        ast_type = AST_SUB_TYPE_OR
+    elif isinstance(node, ast.Name):
+        ast_type = AST_SUB_TYPE_NAME
+    elif isinstance(node, ast.Tuple):
+        ast_type = AST_SUB_TYPE_TUPLE
+    elif isinstance(node, ast.Subscript):
+        ast_type = AST_SUB_TYPE_SUBSCRIPT
+    elif isinstance(node, ast.Starred):
+        ast_type = AST_SUB_TYPE_STARRED
+    elif isinstance(node, ast.Attribute):
+        ast_type = AST_SUB_TYPE_ATTRIBUTE
+    else:
+        ast_type = AST_SUB_TYPE_UNKNOWN
+    return ast_type
+
+
+def get_node_type(node):
+    """Process an ast node."""
+    method_name = f'{node.__class__.__name__}'
+    node_type = [method_name]
+    # judge the ast main type
+    if isinstance(node, ast.stmt):
+        node_type.append(AST_MAIN_TYPE_STMT)
+    elif isinstance(node, (ast.expr, ast.slice)) or node is None:
+        # ast.slice and ast.expr should be expr
+        node_type.append(AST_MAIN_TYPE_EXPR)
+    else:
+        node_type.append(AST_MAIN_TYPE_UNKNOWN)
+    return node_type
+
+
+def get_args_default_values(node):
+    """get the args'default values of parse object."""
+    nondefaults = [None] * (len(node.args.args) - len(node.args.defaults))
+    defaults = nondefaults + node.args.defaults + node.args.kw_defaults
+    if node.args.vararg:
+        defaults.append(None)
+    if node.args.kwarg:
+        defaults.append(None)
+    return defaults
+
+
+def get_args(node):
+    """Get the arg of parse object."""
+    args = []
+    # process position args
+    for arg in node.args.args:
+        args.append(arg)
+
+    # process kwonlyargs: kwonlyargs is append after position args
+    if node.args.kwonlyargs:
+        for kwarg in node.args.kwonlyargs:
+            args.append(kwarg)
+    # process vararg: vararg is append after kwonlyargs
+    if node.args.vararg:
+        args.append(node.args.vararg)
+    # process kwarg: kwarg is append after vararg
+    if node.args.kwarg:
+        args.append(node.args.kwarg)
+    return args
+
+
 class Parser:
     """
     Parser python code to ast tree.
@@ -416,102 +499,28 @@ class Parser:
                     idt_err.filename = self.filename
                     idt_err.lineno = self.line_offset
                     idt_err.msg = f"There are incorrect indentations in definition or comment of function: " \
-                                 f"'{self.fn.__qualname__}'."
+                                  f"'{self.fn.__qualname__}'."
                     raise idt_err
                 Parser.ast_cache[hexstr] = tree
         else:
             logger.error("Fn type is invalid")
         return tree
 
-    def get_args(self, node):
-        """Get the arg of parse object."""
-        args = []
-        # process position args
-        for arg in node.args.args:
-            args.append(arg)
-
-        # process kwonlyargs: kwonlyargs is append after position args
-        if node.args.kwonlyargs:
-            for kwarg in node.args.kwonlyargs:
-                args.append(kwarg)
-        # process vararg: vararg is append after kwonlyargs
-        if node.args.vararg:
-            args.append(node.args.vararg)
-        # process kwarg: kwarg is append after vararg
-        if node.args.kwarg:
-            args.append(node.args.kwarg)
-        return args
-
-    def get_args_default_values(self, node):
-        """get the args'default values of parse object."""
-        nondefaults = [None] * (len(node.args.args) - len(node.args.defaults))
-        defaults = nondefaults + node.args.defaults + node.args.kw_defaults
-        if node.args.vararg:
-            defaults.append(None)
-        if node.args.kwarg:
-            defaults.append(None)
-        return defaults
-
-    def get_node_type(self, node):
-        """Process an ast node."""
-        method_name = f'{node.__class__.__name__}'
-        node_type = [method_name]
-        # judge the ast main type
-        if isinstance(node, ast.stmt):
-            node_type.append(AST_MAIN_TYPE_STMT)
-        elif isinstance(node, (ast.expr, ast.slice)) or node is None:
-            # ast.slice and ast.expr should be expr
-            node_type.append(AST_MAIN_TYPE_EXPR)
-        else:
-            node_type.append(AST_MAIN_TYPE_UNKNOWN)
-        return node_type
-
-    def get_ast_type(self, node):
-        """Get the ast type."""
-        ast_type = AST_SUB_TYPE_UNKNOWN
-        if isinstance(node, ast.And):
-            ast_type = AST_SUB_TYPE_AND
-        elif isinstance(node, ast.Or):
-            ast_type = AST_SUB_TYPE_OR
-        elif isinstance(node, ast.Name):
-            ast_type = AST_SUB_TYPE_NAME
-        elif isinstance(node, ast.Tuple):
-            ast_type = AST_SUB_TYPE_TUPLE
-        elif isinstance(node, ast.Subscript):
-            ast_type = AST_SUB_TYPE_SUBSCRIPT
-        elif isinstance(node, ast.Starred):
-            ast_type = AST_SUB_TYPE_STARRED
-        elif isinstance(node, ast.Attribute):
-            ast_type = AST_SUB_TYPE_ATTRIBUTE
-        else:
-            ast_type = AST_SUB_TYPE_UNKNOWN
-        return ast_type
-
     def get_namespace_symbol(self, var: str):
+
         """Get symbol type and namespace and symbol."""
         if var in self.closure_namespace:
-            ops_info = (self.closure_namespace, var)
             logger.debug("in closure_namespace")
-        elif var in self.global_namespace:
-            ops_info = (self.global_namespace, var)
+            return self.closure_namespace, var
+        if var in self.global_namespace:
             logger.debug("in global_namespace")
-        else:
-            ops_info = parse_object_map.get(SYMBOL_UNDEFINE)
-            ops_info = [ops_info[0], var]
-        return ops_info
-
-    def get_operation_namespace_symbol(self, var: str):
-        """Get operation namespace and symbol."""
-        ops_info = (trope_ns, var)
-        logger.debug("get operation ops info = %r", ops_info)
-        return ops_info
-
-    def get_ast_namespace_symbol(self, obj):
-        """Get obj type and namespace and symbol."""
-        # step 1:get symbol from object map
-        ops_info = parse_object_map.get(type(obj), SYMBOL_UNDEFINE)
-        logger.debug("ops info = %r", ops_info)
-        return ops_info
+            value = self.global_namespace[var]
+            if isinstance(value, type(abs)) and self.global_namespace[var] not in convert_object_map:
+                error_info = f"The builtin function '{var}' is not supported in graph mode."
+                return None, var, error_info
+            return self.global_namespace, var
+        error_info = f"The name '{var}' is not defined."
+        return None, var, error_info
 
     def analyze_super(self, class_type_node, subclass_instance):
         """Analyze super and return a class instance."""

mindspore/ccsrc/pipeline/jit/parse/function_block.cc

@@ -139,7 +139,7 @@ AnfNodePtr FunctionBlock::MakeResolveAstOp(const py::object &op) {
   auto ast = parser_.ast();
   MS_EXCEPTION_IF_NULL(ast);
   TraceGuard trace_guard(parser_.GetLocation(op));
-  py::tuple namespace_var = ast->CallParserObjMethod(PYTHON_PARSE_GET_AST_NAMESPACE_SYMBOL, op);
+  py::tuple namespace_var = ast->CallParseModFunction(PYTHON_PARSE_GET_AST_NAMESPACE_SYMBOL, op);
   if (namespace_var.size() != 2) {
     MS_LOG(EXCEPTION) << "Resolve ast op failed, get namespace tuple size=" << namespace_var.size();
   }
@@ -170,6 +170,9 @@ AnfNodePtr FunctionBlock::MakeResolveSymbol(const std::string &value) {
   }
   py::tuple namespace_var = parser_.ast()->CallParserObjMethod(PYTHON_PARSE_GET_NAMESPACE_SYMBOL, value);
   if (namespace_var[0].is_none()) {
+    if (namespace_var.size() >= 3) {
+      MS_EXCEPTION(NameError) << namespace_var[2].cast<std::string>();
+    }
     MS_EXCEPTION(NameError) << "The name \'" << value << "\' is not defined.";
   }
 
@@ -179,7 +182,7 @@ AnfNodePtr FunctionBlock::MakeResolveSymbol(const std::string &value) {
 }
 
 AnfNodePtr FunctionBlock::MakeResolveOperation(const std::string &value) {
-  py::tuple namespace_var = parser_.ast()->CallParserObjMethod(PYTHON_PARSE_GET_OPERATION_NAMESPACE_SYMBOL, value);
+  py::tuple namespace_var = parser_.ast()->CallParseModFunction(PYTHON_PARSE_GET_OPERATION_NAMESPACE_SYMBOL, value);
   NameSpacePtr name_space = std::make_shared<NameSpace>(RESOLVE_NAMESPACE_NAME_COMMON_OPS, namespace_var[0]);
   SymbolPtr symbol = std::make_shared<Symbol>(namespace_var[1].cast<std::string>());
   return MakeResolve(name_space, symbol);

mindspore/ccsrc/pipeline/jit/parse/parse.cc

@@ -619,7 +619,7 @@ AnfNodePtr Parser::ParseSuper(const FunctionBlockPtr &block, const py::list &arg
     father_class = py::none();
   } else if (args.size() == 2) {
     father_class = args[0];
-    auto arg_type = AstSubType(py::cast<int32_t>(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, args[1])));
+    auto arg_type = AstSubType(py::cast<int32_t>(ast_->CallParseModFunction(PYTHON_PARSE_GET_AST_TYPE, args[1])));
     if (arg_type != AST_SUB_TYPE_NAME || py::cast<std::string>(python_adapter::GetPyObjAttr(args[1], "id")) != "self") {
       MS_EXCEPTION(ArgumentError) << "When call 'super', the second arg should be 'self'.";
     }
@@ -641,7 +641,7 @@ AnfNodePtr Parser::ParseCall(const FunctionBlockPtr &block, const py::object &no
   py::list args = python_adapter::GetPyObjAttr(node, "args");
 
   auto arg_type =
-    AstSubType(py::cast<int32_t>(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, function_ast_node)));
+    AstSubType(py::cast<int32_t>(ast_->CallParseModFunction(PYTHON_PARSE_GET_AST_TYPE, function_ast_node)));
   if (arg_type == AST_SUB_TYPE_NAME) {
     auto name_id = py::cast<std::string>(python_adapter::GetPyObjAttr(function_ast_node, "id"));
     if (name_id == "super") {
@@ -694,7 +694,7 @@ bool Parser::ParseArgsInCall(const FunctionBlockPtr &block, const py::list &args
                              std::vector<AnfNodePtr> *packed_arguments, std::vector<AnfNodePtr> *group_arguments) {
   bool need_unpack = false;
   for (size_t i = 0; i < args.size(); i++) {
-    auto arg_node = AstSubType(py::cast<int32_t>(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, args[i])));
+    auto arg_node = AstSubType(py::cast<int32_t>(ast_->CallParseModFunction(PYTHON_PARSE_GET_AST_TYPE, args[i])));
     if (arg_node == AST_SUB_TYPE_STARRED) {
       if (!group_arguments->empty()) {
         packed_arguments->push_back(GenerateMakeTuple(block, *group_arguments));
@@ -1058,7 +1058,7 @@ FunctionBlockPtr Parser::ParseAugAssign(const FunctionBlockPtr &block, const py:
   AnfNodePtr target_node = nullptr;
   AnfNodePtr op_node = block->MakeResolveAstOp(op_obj);
   AnfNodePtr value_node = ParseExprNode(block, value_obj);
-  auto ast_type = AstSubType(py::cast<int32_t>(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, target_obj)));
+  auto ast_type = AstSubType(py::cast<int32_t>(ast_->CallParseModFunction(PYTHON_PARSE_GET_AST_TYPE, target_obj)));
 
   if (ast_type == AST_SUB_TYPE_NAME) {
     target_node = ParseName(block, target_obj);
@@ -1575,7 +1575,7 @@ void Parser::HandleAssignSubscript(const FunctionBlockPtr &block, const py::obje
     block->WriteVariable(var_name, setitem_app);
     return;
   }
-  if (AstSubType(py::cast<int32_t>(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, value_obj))) ==
+  if (AstSubType(py::cast<int32_t>(ast_->CallParseModFunction(PYTHON_PARSE_GET_AST_TYPE, value_obj))) ==
       AST_SUB_TYPE_SUBSCRIPT) {
     HandleAssignSubscript(block, value_obj, setitem_app);
     return;
@@ -1590,7 +1590,7 @@ void Parser::HandleAssignSubscript(const FunctionBlockPtr &block, const py::obje
 void Parser::WriteAssignVars(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &value_node) {
   MS_EXCEPTION_IF_NULL(value_node);
   MS_LOG(DEBUG) << "Process WriteAssignVars";
-  auto ast_type = AstSubType(py::cast<int32_t>(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, targ)));
+  auto ast_type = AstSubType(py::cast<int32_t>(ast_->CallParseModFunction(PYTHON_PARSE_GET_AST_TYPE, targ)));
   if (ast_type == AST_SUB_TYPE_NAME) {
     HandleAssignName(block, targ, value_node);
   } else if (ast_type == AST_SUB_TYPE_TUPLE) {
@@ -1778,17 +1778,17 @@ py::object ParseAst::GetAstNode() {
 }
 
 py::list ParseAst::GetArgs(const py::object &func_node) {
-  py::list ret = python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_ARGS, func_node);
+  py::list ret = python_adapter::CallPyModFn(module_, PYTHON_PARSE_GET_ARGS, func_node);
   return ret;
 }
 
 py::list ParseAst::GetArgsDefaultValues(const py::object &func_node) {
-  py::list ret = python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_ARGS_DEFAULT_VALUES, func_node);
+  py::list ret = python_adapter::CallPyModFn(module_, PYTHON_PARSE_GET_ARGS_DEFAULT_VALUES, func_node);
   return ret;
 }
 
 AstNodeTypePtr ParseAst::GetNodeType(const py::object &node) {
-  py::list list_value = python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_NODE_TYPE, node);
+  py::list list_value = python_adapter::CallPyModFn(module_, PYTHON_PARSE_GET_NODE_TYPE, node);
   if (list_value.size() < 2) {
     MS_LOG(ERROR) << "The node of python method must has 2 values.";
     return nullptr;
@@ -1799,7 +1799,7 @@ AstNodeTypePtr ParseAst::GetNodeType(const py::object &node) {
 }
 
 AstSubType ParseAst::GetOpType(const py::object &node) {
-  auto op_type = AstSubType(python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_AST_TYPE, node).cast<int32_t>());
+  auto op_type = AstSubType(python_adapter::CallPyModFn(module_, PYTHON_PARSE_GET_AST_TYPE, node).cast<int32_t>());
   return op_type;
 }