[Dy2Stat] Add debugging and logging mechanism for dygraph to static (#26457)
* [Dy2Stat] Add debugging and logging mechanism for dygraph to static. * Remove TransformerError temporarily. * import mock in PY2, from unittest import mock in PY3. test=develop * Expose interfaces set_code_level and set_verbosity in paddle.jit, fix doc of the two interface. * polish doc of set_verbosity and set_code_level.revert-26856-strategy_example2
liym27
5 years ago
committed by
GitHub
parent
e3f8e5cf5c
commit
b1f9ed6006
@ -0,0 +1,211 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
||||
import six
|
||||
from paddle.fluid import log_helper
|
||||
from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code
|
||||
|
||||
__all__ = ["TranslatorLogger", "set_verbosity", "set_code_level"]
|
||||
|
||||
VERBOSITY_ENV_NAME = 'TRANSLATOR_VERBOSITY'
|
||||
CODE_LEVEL_ENV_NAME = 'TRANSLATOR_CODE_LEVEL'
|
||||
DEFAULT_VERBOSITY = -1
|
||||
DEFAULT_CODE_LEVEL = -1
|
||||
|
||||
|
||||
def synchronized(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
with threading.Lock():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class TranslatorLogger(object):
|
||||
"""
|
||||
class for Logging and debugging during the tranformation from dygraph to static graph.
|
||||
The object of this class is a singleton.
|
||||
"""
|
||||
|
||||
@synchronized
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if not hasattr(cls, '_instance'):
|
||||
cls._instance = object.__new__(cls, *args, **kwargs)
|
||||
cls._instance._initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
self._initialized = True
|
||||
self._logger = log_helper.get_logger(
|
||||
__name__, 1, fmt='%(asctime)s-%(levelname)s: %(message)s')
|
||||
self._verbosity_level = None
|
||||
self._transformed_code_level = None
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
return self._logger
|
||||
|
||||
@property
|
||||
def verbosity_level(self):
|
||||
if self._verbosity_level is not None:
|
||||
return self._verbosity_level
|
||||
else:
|
||||
return int(os.getenv(VERBOSITY_ENV_NAME, DEFAULT_VERBOSITY))
|
||||
|
||||
@verbosity_level.setter
|
||||
def verbosity_level(self, level):
|
||||
self.check_level(level)
|
||||
self._verbosity_level = level
|
||||
|
||||
@property
|
||||
def transformed_code_level(self):
|
||||
if self._transformed_code_level is not None:
|
||||
return self._transformed_code_level
|
||||
else:
|
||||
return int(os.getenv(CODE_LEVEL_ENV_NAME, DEFAULT_CODE_LEVEL))
|
||||
|
||||
@transformed_code_level.setter
|
||||
def transformed_code_level(self, level):
|
||||
self.check_level(level)
|
||||
self._transformed_code_level = level
|
||||
|
||||
def check_level(self, level):
|
||||
if isinstance(level, (six.integer_types, type(None))):
|
||||
rv = level
|
||||
else:
|
||||
raise TypeError("Level is not an integer: {}".format(level))
|
||||
return rv
|
||||
|
||||
def has_code_level(self, level):
|
||||
level = self.check_level(level)
|
||||
return level == self.transformed_code_level
|
||||
|
||||
def has_verbosity(self, level):
|
||||
level = self.check_level(level)
|
||||
return level >= self.verbosity_level
|
||||
|
||||
def error(self, msg, *args, **kwargs):
|
||||
self.logger.error(msg, *args, **kwargs)
|
||||
|
||||
def warn(self, msg, *args, **kwargs):
|
||||
self.logger.warn(msg, *args, **kwargs)
|
||||
|
||||
def log(self, level, msg, *args, **kwargs):
|
||||
if self.has_verbosity(level):
|
||||
self.logger.log(level, msg, *args, **kwargs)
|
||||
|
||||
def log_transformed_code(self, level, ast_node, transformer_name, *args,
|
||||
**kwargs):
|
||||
if self.has_code_level(level):
|
||||
source_code = ast_to_source_code(ast_node)
|
||||
header_msg = "After the level {} ast transformer: '{}', the transformed code:\n"\
|
||||
.format(level, transformer_name)
|
||||
|
||||
msg = header_msg + source_code
|
||||
self.logger.info(msg, *args, **kwargs)
|
||||
|
||||
|
||||
_TRANSLATOR_LOGGER = TranslatorLogger()
|
||||
|
||||
|
||||
def set_verbosity(level=0):
|
||||
"""
|
||||
Sets the verbosity level of log for dygraph to static graph.
|
||||
There are two means to set the logging verbosity:
|
||||
1. Call function `set_verbosity`
|
||||
2. Set environment variable `TRANSLATOR_VERBOSITY`
|
||||
|
||||
**Note**:
|
||||
`set_verbosity` has a higher priority than the environment variable.
|
||||
|
||||
Args:
|
||||
level(int): The verbosity level. The larger value idicates more verbosity.
|
||||
The default value is 0, which means no logging.
|
||||
|
||||
Examples:
|
||||
.. code-block:: python
|
||||
|
||||
import os
|
||||
import paddle
|
||||
|
||||
paddle.jit.set_verbosity(1)
|
||||
# The verbosity level is now 1
|
||||
|
||||
os.environ['TRANSLATOR_VERBOSITY'] = '3'
|
||||
# The verbosity level is now 3, but it has no effect because it has a lower priority than `set_verbosity`
|
||||
"""
|
||||
_TRANSLATOR_LOGGER.verbosity_level = level
|
||||
|
||||
|
||||
def get_verbosity():
|
||||
return _TRANSLATOR_LOGGER.verbosity_level
|
||||
|
||||
|
||||
LOG_AllTransformer = 100
|
||||
|
||||
|
||||
def set_code_level(level=LOG_AllTransformer):
|
||||
"""
|
||||
Sets the level to print code from specific level of Ast Transformer.
|
||||
There are two means to set the code level:
|
||||
1. Call function `set_code_level`
|
||||
2. Set environment variable `TRANSLATOR_CODE_LEVEL`
|
||||
|
||||
**Note**:
|
||||
`set_code_level` has a higher priority than the environment variable.
|
||||
|
||||
Args:
|
||||
level(int): The level to print code. Default is 100, which means to print the code after all AST Transformers.
|
||||
|
||||
Examples:
|
||||
.. code-block:: python
|
||||
|
||||
import paddle
|
||||
|
||||
paddle.jit.set_code_level(2)
|
||||
# It will print the transformed code at level 2, which means to print the code after second transformer,
|
||||
# as the date of August 28, 2020, it is CastTransformer.
|
||||
|
||||
os.environ['TRANSLATOR_CODE_LEVEL'] = '3'
|
||||
# The code level is now 3, but it has no effect because it has a lower priority than `set_code_level`
|
||||
|
||||
"""
|
||||
_TRANSLATOR_LOGGER.transformed_code_level = level
|
||||
|
||||
|
||||
def get_code_level():
|
||||
return _TRANSLATOR_LOGGER.transformed_code_level
|
||||
|
||||
|
||||
def error(msg, *args, **kwargs):
|
||||
_TRANSLATOR_LOGGER.error(msg, *args, **kwargs)
|
||||
|
||||
|
||||
def warn(msg, *args, **kwargs):
|
||||
_TRANSLATOR_LOGGER.warn(msg, *args, **kwargs)
|
||||
|
||||
|
||||
def log(level, msg, *args, **kwargs):
|
||||
_TRANSLATOR_LOGGER.log(level, msg, *args, **kwargs)
|
||||
|
||||
|
||||
def log_transformed_code(level, ast_node, transformer_name, *args, **kwargs):
|
||||
_TRANSLATOR_LOGGER.log_transformed_code(level, ast_node, transformer_name,
|
||||
*args, **kwargs)
|
||||
@ -0,0 +1,120 @@
|
||||
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
import gast
|
||||
import six
|
||||
|
||||
import paddle
|
||||
from paddle.fluid.dygraph.dygraph_to_static import logging_utils
|
||||
|
||||
# TODO(liym27): library mock needs to be installed separately in PY2,
|
||||
# but CI environment has not installed mock yet.
|
||||
# After discuss with Tian Shuo, now use mock only in PY3, and use it in PY2 after CI installs it.
|
||||
if six.PY3:
|
||||
from unittest import mock
|
||||
# else:
|
||||
# import mock
|
||||
|
||||
|
||||
class TestLoggingUtils(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.verbosity_level = 1
|
||||
self.code_level = 3
|
||||
self.translator_logger = logging_utils._TRANSLATOR_LOGGER
|
||||
|
||||
def test_verbosity(self):
|
||||
paddle.jit.set_verbosity(None)
|
||||
os.environ[logging_utils.VERBOSITY_ENV_NAME] = '3'
|
||||
self.assertEqual(logging_utils.get_verbosity(), 3)
|
||||
|
||||
paddle.jit.set_verbosity(self.verbosity_level)
|
||||
self.assertEqual(self.verbosity_level, logging_utils.get_verbosity())
|
||||
|
||||
# String is not supported
|
||||
with self.assertRaises(TypeError):
|
||||
paddle.jit.set_verbosity("3")
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
paddle.jit.set_verbosity(3.3)
|
||||
|
||||
def test_code_level(self):
|
||||
|
||||
paddle.jit.set_code_level(None)
|
||||
os.environ[logging_utils.CODE_LEVEL_ENV_NAME] = '2'
|
||||
self.assertEqual(logging_utils.get_code_level(), 2)
|
||||
|
||||
paddle.jit.set_code_level(self.code_level)
|
||||
self.assertEqual(logging_utils.get_code_level(), self.code_level)
|
||||
|
||||
paddle.jit.set_code_level(9)
|
||||
self.assertEqual(logging_utils.get_code_level(), 9)
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
paddle.jit.set_code_level(3.3)
|
||||
|
||||
def test_log(self):
|
||||
stream = io.BytesIO() if six.PY2 else io.StringIO()
|
||||
log = self.translator_logger.logger
|
||||
stdout_handler = logging.StreamHandler(stream)
|
||||
log.addHandler(stdout_handler)
|
||||
|
||||
warn_msg = "test_warn"
|
||||
error_msg = "test_error"
|
||||
log_msg_1 = "test_log_1"
|
||||
log_msg_2 = "test_log_2"
|
||||
|
||||
if six.PY3:
|
||||
with mock.patch.object(sys, 'stdout', stream):
|
||||
logging_utils.warn(warn_msg)
|
||||
logging_utils.error(error_msg)
|
||||
self.translator_logger.verbosity_level = 2
|
||||
logging_utils.log(1, log_msg_1)
|
||||
logging_utils.log(2, log_msg_2)
|
||||
|
||||
result_msg = '\n'.join([warn_msg, error_msg, log_msg_2, ""])
|
||||
self.assertEqual(result_msg, stream.getvalue())
|
||||
|
||||
def test_log_transformed_code(self):
|
||||
source_code = "x = 3"
|
||||
ast_code = gast.parse(source_code)
|
||||
|
||||
stream = io.BytesIO() if six.PY2 else io.StringIO()
|
||||
log = self.translator_logger.logger
|
||||
stdout_handler = logging.StreamHandler(stream)
|
||||
log.addHandler(stdout_handler)
|
||||
|
||||
if six.PY3:
|
||||
with mock.patch.object(sys, 'stdout', stream):
|
||||
paddle.jit.set_code_level(1)
|
||||
logging_utils.log_transformed_code(1, ast_code,
|
||||
"BasicApiTransformer")
|
||||
|
||||
paddle.jit.set_code_level()
|
||||
logging_utils.log_transformed_code(
|
||||
logging_utils.LOG_AllTransformer, ast_code,
|
||||
"All Transformers")
|
||||
|
||||
self.assertIn(source_code, stream.getvalue())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue