!9544 Fix a possible seg fault in GenerateCRC caused by std::regex

From: @lixiachen
Reviewed-by: @jonyguo,@heleiwang
Signed-off-by: @heleiwang
pull/9544/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
commit aa5d4e08a9

@ -453,8 +453,8 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) {
ss_str = std::regex_replace(ss_str, std::regex("device_id.*\n"), "");
// Filter out the operator id field
ss_str = std::regex_replace(ss_str, std::regex(".*Parent.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex(".*Child.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex(" *Parent.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex(" *Child.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex(R"(\(\s*\d+?\))"), "");
// Doesn't matter whether there is any parent node above CacheOp or not.

@ -324,6 +324,9 @@ HandleRcExit $? 0 0
PytestCmd "test_cache_nomap.py" "test_cache_nomap_get_repeat_count"
HandleRcExit $? 0 0
PytestCmd "test_cache_nomap.py" "test_cache_nomap_long_file_list"
HandleRcExit $? 0 0
for i in $(seq 1 3)
do
test_name="test_cache_nomap_multiple_cache${i}"

@ -1828,9 +1828,7 @@ def test_cache_map_cifar3():
cache
|
Map(resize)
|
Cifar100
Cifar10
"""
logger.info("Test cache map cifar3")
@ -1841,9 +1839,7 @@ def test_cache_map_cifar3():
some_cache = ds.DatasetCache(session_id=session_id, size=1, spilling=False)
ds1 = ds.Cifar10Dataset(CIFAR10_DATA_DIR)
resize_op = c_vision.Resize((224, 224))
ds1 = ds1.map(input_columns=["image"], operations=resize_op, cache=some_cache)
ds1 = ds.Cifar10Dataset(CIFAR10_DATA_DIR, cache=some_cache)
num_epoch = 2
iter1 = ds1.create_dict_iterator(num_epochs=num_epoch)

@ -1775,6 +1775,7 @@ def test_cache_nomap_textfile2():
|
TextFile
"""
def my_tokenizer(line):
words = line.split()
if not words:
@ -1884,6 +1885,34 @@ def test_cache_nomap_get_repeat_count():
num_iter += 1
assert num_iter == 12
@pytest.mark.skipif(os.environ.get('RUN_CACHE_TEST') != 'TRUE', reason="Require to bring up cache server")
def test_cache_nomap_long_file_list():
"""
Test cache after TFRecord with a long list of files as arguments
Cache
|
TFRecord
"""
logger.info("Test cache nomap long file list")
if "SESSION_ID" in os.environ:
session_id = int(os.environ['SESSION_ID'])
else:
raise RuntimeError("Testcase requires SESSION_ID environment variable")
some_cache = ds.DatasetCache(session_id=session_id, size=1, spilling=False)
ds1 = ds.TFRecordDataset([DATA_DIR[0] for _ in range(0, 1000)], SCHEMA_DIR, columns_list=["image"],
cache=some_cache)
with pytest.raises(RuntimeError) as e:
sum([1 for _ in ds1])
assert "Out of memory" in str(e.value)
logger.info("test_cache_nomap_long_file_list Ended.\n")
if __name__ == '__main__':
test_cache_nomap_basic1()
test_cache_nomap_basic2()

Loading…
Cancel
Save