m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!11950 Overwrite default port/hostname from env variables for cache_admin

Browse Source 
From: @lixiachen
Reviewed-by: 
Signed-off-by:
pull/11950/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent 62d272f648 97cd142f8d
commit e8f4cafd02
13 changed files with 126 additions and 15 deletions
Show Stats Download Patch File Download Diff File

7
mindspore/ccsrc/minddata/dataset/core/config_manager.cc
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -57,9 +57,8 @@ ConfigManager::ConfigManager()
char *end = nullptr;
cache_port_ = strtol(env_cache_port, &end, 10);
if (*end != '\0') {
MS_LOG(WARNING) << "\nCache port from env variable MS_CACHE_PORT is invalid, back to use default "
<< kCfgDefaultCachePort << std::endl;
cache_port_ = kCfgDefaultCachePort;
MS_LOG(WARNING) << "Cache port from env variable MS_CACHE_PORT is invalid\n";
cache_port_ = 0; // cause the port range validation to generate an error during the validation checks
}
}
}

15
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -50,6 +50,19 @@ CacheAdminArgHandler::CacheAdminArgHandler()
hostname_(kCfgDefaultCacheHost),
spill_dir_(""),
command_id_(CommandId::kCmdUnknown) {
const char *env_cache_host = std::getenv("MS_CACHE_HOST");
const char *env_cache_port = std::getenv("MS_CACHE_PORT");
if (env_cache_host != nullptr) {
hostname_ = env_cache_host;
}
if (env_cache_port != nullptr) {
char *end = nullptr;
port_ = strtol(env_cache_port, &end, 10);
if (*end != '\0') {
std::cerr << "Cache port from env variable MS_CACHE_PORT is invalid\n";
port_ = 0; // cause the port range validation to generate an error during the validation checks
}
}
// Initialize the command mappings
arg_map_["-h"] = ArgValue::kArgHost;
arg_map_["--hostname"] = ArgValue::kArgHost;

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs vendored
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -307,7 +307,7 @@ Status CacheBase::Prefetcher(int32_t worker_id) {
if (rc == StatusCode::kMDNetWorkError && retry_count < max_retries) {
// If we get some network error, we will attempt some retries
retry_count++;
} else if (rc.IsError()) {
} else if (rc.IsError() && rc.StatusCode() != StatusCode::kMDInterrupted) {
MS_LOG(WARNING) << rc.ToString();
return rc;
}

10
mindspore/dataset/engine/cache_client.py
Unescape View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -38,6 +38,14 @@ class DatasetCache:
num_connections (int, optional): Number of tcp/ip connections (default=12).
prefetch_size (int, optional): Prefetch size (default=20).
Examples:
>>> import mindspore.dataset as ds
>>>
>>> # create a cache instance, in which session_id is generated from command line `cache_admin -g`
>>> some_cache = ds.DatasetCache(session_id=session_id, size=0)
>>>
>>> dataset_dir = "path/to/imagefolder_directory"
>>> ds1 = ds.ImageFolderDataset(dataset_dir, cache=some_cache)
"""
def __init__(self, session_id, size=0, spilling=False, hostname=None, port=None, num_connections=None,

48
tests/ut/python/dataset/test_cache_map.py
Unescape View File

@ -2110,11 +2110,59 @@ def test_cache_map_nested_repeat():
if __name__ == '__main__':
# This is just a list of tests, don't try to run these tests with 'python test_cache_map.py'
# since cache server is required to be brought up first
test_cache_map_basic1()
test_cache_map_basic2()
test_cache_map_basic3()
test_cache_map_basic4()
test_cache_map_basic5()
test_cache_map_failure1()
test_cache_map_failure2()
test_cache_map_failure3()
test_cache_map_failure4()
test_cache_map_failure5()
test_cache_map_failure6()
test_cache_map_failure7()
test_cache_map_failure8()
test_cache_map_failure9()
test_cache_map_failure10()
test_cache_map_failure11()
test_cache_map_split1()
test_cache_map_split2()
test_cache_map_parameter_check()
test_cache_map_running_twice1()
test_cache_map_running_twice2()
test_cache_map_extra_small_size1()
test_cache_map_extra_small_size2()
test_cache_map_no_image()
test_cache_map_parallel_pipeline1(shard=0)
test_cache_map_parallel_pipeline2(shard=1)
test_cache_map_parallel_workers()
test_cache_map_server_workers_1()
test_cache_map_server_workers_100()
test_cache_map_num_connections_1()
test_cache_map_num_connections_100()
test_cache_map_prefetch_size_1()
test_cache_map_prefetch_size_100()
test_cache_map_to_device()
test_cache_map_epoch_ctrl1()
test_cache_map_epoch_ctrl2()
test_cache_map_epoch_ctrl3()
test_cache_map_coco1()
test_cache_map_coco2()
test_cache_map_mnist1()
test_cache_map_mnist2()
test_cache_map_celeba1()
test_cache_map_celeba2()
test_cache_map_manifest1()
test_cache_map_manifest2()
test_cache_map_cifar1()
test_cache_map_cifar2()
test_cache_map_cifar3()
test_cache_map_cifar4()
test_cache_map_voc1()
test_cache_map_voc2()
test_cache_map_python_sampler1()
test_cache_map_python_sampler2()
test_cache_map_nested_repeat()

43
tests/ut/python/dataset/test_cache_nomap.py
Unescape View File

@ -2125,6 +2125,8 @@ def test_cache_nomap_failure5():
if __name__ == '__main__':
# This is just a list of tests, don't try to run these tests with 'python test_cache_nomap.py'
# since cache server is required to be brought up first
test_cache_nomap_basic1()
test_cache_nomap_basic2()
test_cache_nomap_basic3()
@ -2132,8 +2134,49 @@ if __name__ == '__main__':
test_cache_nomap_basic5()
test_cache_nomap_basic6()
test_cache_nomap_basic7()
test_cache_nomap_basic8()
test_cache_nomap_basic9()
test_cache_nomap_allowed_share1()
test_cache_nomap_allowed_share2()
test_cache_nomap_allowed_share3()
test_cache_nomap_allowed_share4()
test_cache_nomap_disallowed_share1()
test_cache_nomap_running_twice1()
test_cache_nomap_running_twice2()
test_cache_nomap_extra_small_size1()
test_cache_nomap_extra_small_size2()
test_cache_nomap_parallel_pipeline1(shard=0)
test_cache_nomap_parallel_pipeline2(shard=1)
test_cache_nomap_parallel_workers()
test_cache_nomap_server_workers_1()
test_cache_nomap_server_workers_100()
test_cache_nomap_num_connections_1()
test_cache_nomap_num_connections_100()
test_cache_nomap_prefetch_size_1()
test_cache_nomap_prefetch_size_100()
test_cache_nomap_to_device()
test_cache_nomap_session_destroy()
test_cache_nomap_server_stop()
test_cache_nomap_epoch_ctrl1()
test_cache_nomap_epoch_ctrl2()
test_cache_nomap_epoch_ctrl3()
test_cache_nomap_epoch_ctrl4()
test_cache_nomap_multiple_cache1()
test_cache_nomap_multiple_cache2()
test_cache_nomap_multiple_cache3()
test_cache_nomap_multiple_cache_train()
test_cache_nomap_multiple_cache_eval()
test_cache_nomap_clue1()
test_cache_nomap_clue2()
test_cache_nomap_csv1()
test_cache_nomap_csv2()
test_cache_nomap_textfile1()
test_cache_nomap_textfile2()
test_cache_nomap_nested_repeat()
test_cache_nomap_get_repeat_count()
test_cache_nomap_long_file_list()
test_cache_nomap_failure1()
test_cache_nomap_failure2()
test_cache_nomap_failure3()
test_cache_nomap_failure4()
test_cache_nomap_failure5()

Write Preview
Loading…
Cancel
Save