enhance the op_version_registry, test=develop (#28347)
* enhance the op_version_registry, test=develop * add unittests, test=develop * enhance the op_version_registry, test=develop * fix bugs, test=develop * revert pybind_boost_headers.h, test=develop * fix a attribute bug, test=developTCChenlong-patch-1
石晓伟
5 years ago
committed by
GitHub
parent
c1c3e21726
commit
21a63f6f90
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,42 @@
|
||||
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
REGISTER_OP_VERSION(for_pybind_test__)
|
||||
.AddCheckpoint("Note 0", framework::compatible::OpVersionDesc()
|
||||
.BugfixWithBehaviorChanged(
|
||||
"BugfixWithBehaviorChanged Remark"))
|
||||
.AddCheckpoint("Note 1", framework::compatible::OpVersionDesc()
|
||||
.ModifyAttr("BOOL", "bool", true)
|
||||
.ModifyAttr("FLOAT", "float", 1.23f)
|
||||
.ModifyAttr("INT", "int32", -1)
|
||||
.ModifyAttr("STRING", "std::string",
|
||||
std::string{"hello"}))
|
||||
.AddCheckpoint("Note 2",
|
||||
framework::compatible::OpVersionDesc()
|
||||
.ModifyAttr("BOOLS", "std::vector<bool>",
|
||||
std::vector<bool>{true, false})
|
||||
.ModifyAttr("FLOATS", "std::vector<float>",
|
||||
std::vector<float>{2.56f, 1.28f})
|
||||
.ModifyAttr("INTS", "std::vector<int32>",
|
||||
std::vector<int32_t>{10, 100})
|
||||
.NewAttr("LONGS", "std::vector<int64>",
|
||||
std::vector<int64_t>{10000001, -10000001}))
|
||||
.AddCheckpoint("Note 3", framework::compatible::OpVersionDesc()
|
||||
.NewAttr("STRINGS", "std::vector<std::string>",
|
||||
std::vector<std::string>{"str1", "str2"})
|
||||
.ModifyAttr("LONG", "int64", static_cast<int64_t>(10000001))
|
||||
.NewInput("NewInput", "NewInput_")
|
||||
.NewOutput("NewOutput", "NewOutput_")
|
||||
.BugfixWithBehaviorChanged(
|
||||
"BugfixWithBehaviorChanged_"));
|
||||
@ -0,0 +1,83 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import unittest
|
||||
|
||||
import paddle.utils as utils
|
||||
import paddle.fluid as fluid
|
||||
|
||||
|
||||
class OpLastCheckpointCheckerTest(unittest.TestCase):
|
||||
def __init__(self, methodName='runTest'):
|
||||
super(OpLastCheckpointCheckerTest, self).__init__(methodName)
|
||||
self.checker = utils.OpLastCheckpointChecker()
|
||||
self.fake_op = 'for_pybind_test__'
|
||||
|
||||
def test_op_attr_info(self):
|
||||
update_type = fluid.core.OpUpdateType.kNewAttr
|
||||
info_list = self.checker.filter_updates(self.fake_op, update_type,
|
||||
'STRINGS')
|
||||
self.assertTrue(info_list)
|
||||
self.assertEqual(info_list[0].name(), 'STRINGS')
|
||||
self.assertEqual(info_list[0].default_value(), ['str1', 'str2'])
|
||||
self.assertEqual(info_list[0].remark(), 'std::vector<std::string>')
|
||||
|
||||
def test_op_input_output_info(self):
|
||||
update_type = fluid.core.OpUpdateType.kNewInput
|
||||
info_list = self.checker.filter_updates(self.fake_op, update_type,
|
||||
'NewInput')
|
||||
self.assertTrue(info_list)
|
||||
self.assertEqual(info_list[0].name(), 'NewInput')
|
||||
self.assertEqual(info_list[0].remark(), 'NewInput_')
|
||||
|
||||
def test_op_bug_fix_info(self):
|
||||
update_type = fluid.core.OpUpdateType.kBugfixWithBehaviorChanged
|
||||
info_list = self.checker.filter_updates(self.fake_op, update_type)
|
||||
self.assertTrue(info_list)
|
||||
self.assertEqual(info_list[0].remark(), 'BugfixWithBehaviorChanged_')
|
||||
|
||||
|
||||
class OpVersionTest(unittest.TestCase):
|
||||
def __init__(self, methodName='runTest'):
|
||||
super(OpVersionTest, self).__init__(methodName)
|
||||
self.vmap = fluid.core.get_op_version_map()
|
||||
self.fake_op = 'for_pybind_test__'
|
||||
|
||||
def test_checkpoints(self):
|
||||
version_id = self.vmap[self.fake_op].version_id()
|
||||
checkpoints = self.vmap[self.fake_op].checkpoints()
|
||||
self.assertEqual(version_id, 4)
|
||||
self.assertEqual(len(checkpoints), 4)
|
||||
self.assertEqual(checkpoints[2].note(), 'Note 2')
|
||||
desc_1 = checkpoints[1].version_desc().infos()
|
||||
self.assertEqual(desc_1[0].info().default_value(), True)
|
||||
self.assertAlmostEqual(desc_1[1].info().default_value(), 1.23, 2)
|
||||
self.assertEqual(desc_1[2].info().default_value(), -1)
|
||||
self.assertEqual(desc_1[3].info().default_value(), 'hello')
|
||||
desc_2 = checkpoints[2].version_desc().infos()
|
||||
self.assertEqual(desc_2[0].info().default_value(), [True, False])
|
||||
true_l = [2.56, 1.28]
|
||||
self.assertEqual(len(true_l), len(desc_2[1].info().default_value()))
|
||||
for i in range(len(true_l)):
|
||||
self.assertAlmostEqual(desc_2[1].info().default_value()[i],
|
||||
true_l[i], 2)
|
||||
self.assertEqual(desc_2[2].info().default_value(), [10, 100])
|
||||
self.assertEqual(desc_2[3].info().default_value(),
|
||||
[10000001, -10000001])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,70 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from ..fluid import core
|
||||
|
||||
__all__ = ['OpLastCheckpointChecker']
|
||||
|
||||
|
||||
def Singleton(cls):
|
||||
_instance = {}
|
||||
|
||||
def _singleton(*args, **kargs):
|
||||
if cls not in _instance:
|
||||
_instance[cls] = cls(*args, **kargs)
|
||||
return _instance[cls]
|
||||
|
||||
return _singleton
|
||||
|
||||
|
||||
class OpUpdateInfoHelper(object):
|
||||
def __init__(self, info):
|
||||
self._info = info
|
||||
|
||||
def verify_key_value(self, name=''):
|
||||
result = False
|
||||
key_funcs = {
|
||||
core.OpAttrInfo: 'name',
|
||||
core.OpInputOutputInfo: 'name',
|
||||
}
|
||||
if name == '':
|
||||
result = True
|
||||
elif type(self._info) in key_funcs:
|
||||
if getattr(self._info, key_funcs[type(self._info)])() == name:
|
||||
result = True
|
||||
return result
|
||||
|
||||
|
||||
@Singleton
|
||||
class OpLastCheckpointChecker(object):
|
||||
def __init__(self):
|
||||
self.raw_version_map = core.get_op_version_map()
|
||||
self.checkpoints_map = {}
|
||||
self._construct_map()
|
||||
|
||||
def _construct_map(self):
|
||||
for op_name in self.raw_version_map:
|
||||
last_checkpoint = self.raw_version_map[op_name].checkpoints()[-1]
|
||||
infos = last_checkpoint.version_desc().infos()
|
||||
self.checkpoints_map[op_name] = infos
|
||||
|
||||
def filter_updates(self, op_name, type=core.OpUpdateType.kInvalid, key=''):
|
||||
updates = []
|
||||
if op_name in self.checkpoints_map:
|
||||
for update in self.checkpoints_map[op_name]:
|
||||
if (update.type() == type) or (
|
||||
type == core.OpUpdateType.kInvalid):
|
||||
if OpUpdateInfoHelper(update.info()).verify_key_value(key):
|
||||
updates.append(update.info())
|
||||
return updates
|
||||
Loading…
Reference in new issue