!11690 [MS][RDR] support env_config_path to set the config file for DFX
From: @luopengting Reviewed-by: Signed-off-by:pull/11690/MERGE
mindspore-ci-bot
4 years ago
committed by
Gitee
commit
9e2e936806
@ -0,0 +1,160 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#include "debug/env_config_parser.h"
|
||||||
|
#include <fstream>
|
||||||
|
#include "nlohmann/json.hpp"
|
||||||
|
#include "utils/log_adapter.h"
|
||||||
|
#include "debug/common.h"
|
||||||
|
#include "utils/ms_context.h"
|
||||||
|
#include "utils/convert_utils_base.h"
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
constexpr auto kRdrSettings = "rdr";
|
||||||
|
constexpr auto kPath = "path";
|
||||||
|
constexpr auto kEnable = "enable";
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
namespace mindspore {
|
||||||
|
bool EnvConfigParser::CheckJsonStringType(const nlohmann::json &content, const std::string &setting_key,
|
||||||
|
const std::string &key) {
|
||||||
|
if (!content.is_string()) {
|
||||||
|
MS_LOG(WARNING) << "Json Parse Failed. The '" << key << "' in '" << setting_key << "' should be string."
|
||||||
|
<< " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto EnvConfigParser::CheckJsonKeyExist(const nlohmann::json &content, const std::string &setting_key,
|
||||||
|
const std::string &key) {
|
||||||
|
auto iter = content.find(key);
|
||||||
|
if (iter == content.end()) {
|
||||||
|
MS_LOG(WARNING) << "Check json failed, '" << key << "' not found in '" << setting_key << "'."
|
||||||
|
<< " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
}
|
||||||
|
return iter;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string EnvConfigParser::GetIfstreamString(const std::ifstream &ifstream) {
|
||||||
|
std::stringstream buffer;
|
||||||
|
buffer << ifstream.rdbuf();
|
||||||
|
return buffer.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
void EnvConfigParser::Parse() {
|
||||||
|
std::lock_guard<std::mutex> guard(lock_);
|
||||||
|
if (already_parsed_) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
already_parsed_ = true;
|
||||||
|
|
||||||
|
auto context = MsContext::GetInstance();
|
||||||
|
MS_EXCEPTION_IF_NULL(context);
|
||||||
|
auto config_file = context->get_param<std::string>(MS_CTX_ENV_CONFIG_PATH);
|
||||||
|
if (config_file.empty()) {
|
||||||
|
MS_LOG(INFO) << "Get env config file failed. Please check the 'env_config_path' set in context.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
config_file_ = config_file;
|
||||||
|
std::ifstream json_file(config_file_);
|
||||||
|
if (!json_file.is_open()) {
|
||||||
|
MS_LOG(WARNING) << "Env config file:" << config_file_ << " open failed."
|
||||||
|
<< " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
nlohmann::json j;
|
||||||
|
try {
|
||||||
|
json_file >> j;
|
||||||
|
} catch (nlohmann::json::parse_error &e) {
|
||||||
|
MS_LOG(WARNING) << "Env config json contents '" << GetIfstreamString(json_file) << "' in config file '"
|
||||||
|
<< config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// convert json to string
|
||||||
|
std::stringstream ss;
|
||||||
|
ss << j;
|
||||||
|
std::string cfg = ss.str();
|
||||||
|
MS_LOG(INFO) << "Env config json:" << cfg;
|
||||||
|
|
||||||
|
ParseRdrSetting(j);
|
||||||
|
ConfigToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
void EnvConfigParser::ParseRdrSetting(const nlohmann::json &content) {
|
||||||
|
auto rdr_setting = content.find(kRdrSettings);
|
||||||
|
if (rdr_setting == content.end()) {
|
||||||
|
MS_LOG(WARNING) << "The '" << kRdrSettings << "' not exists. Please check the config file '" << config_file_
|
||||||
|
<< "' set by 'env_config_path' in context.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto rdr_enable = CheckJsonKeyExist(*rdr_setting, kRdrSettings, kEnable);
|
||||||
|
auto rdr_path = CheckJsonKeyExist(*rdr_setting, kRdrSettings, kPath);
|
||||||
|
|
||||||
|
ParseRdrEnable(*rdr_enable);
|
||||||
|
ParseRdrPath(*rdr_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
|
||||||
|
if (!CheckJsonStringType(content, kRdrSettings, kPath)) {
|
||||||
|
MS_LOG(WARNING) << "The RDR path will be a default value: '" << rdr_path_ << "'.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
std::string path = content;
|
||||||
|
if (!std::all_of(path.begin(), path.end(),
|
||||||
|
[](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '/'; })) {
|
||||||
|
MS_LOG(WARNING) << "The path in " << kRdrSettings
|
||||||
|
<< " only support alphabets, digit or {'-', '_', '/'}, but got:" << path << "."
|
||||||
|
<< " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (path.empty()) {
|
||||||
|
MS_LOG(WARNING) << "The path in " << kRdrSettings << " is empty."
|
||||||
|
<< " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (path[0] != '/') {
|
||||||
|
MS_LOG(WARNING) << "The path in " << kRdrSettings << " only support absolute path and should start with '/'."
|
||||||
|
<< " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (path.back() != '/') {
|
||||||
|
path += '/';
|
||||||
|
}
|
||||||
|
rdr_path_ = path;
|
||||||
|
}
|
||||||
|
|
||||||
|
void EnvConfigParser::ParseRdrEnable(const nlohmann::json &content) {
|
||||||
|
if (!content.is_boolean()) {
|
||||||
|
MS_LOG(WARNING) << "Json Parse Failed. 'enable' in " << kRdrSettings << " should be boolean."
|
||||||
|
<< " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
|
||||||
|
rdr_enabled_ = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
rdr_enabled_ = content;
|
||||||
|
}
|
||||||
|
|
||||||
|
void EnvConfigParser::ConfigToString() {
|
||||||
|
std::string cur_config;
|
||||||
|
cur_config.append("After parsed, rdr path: ");
|
||||||
|
cur_config.append(rdr_path_);
|
||||||
|
cur_config.append(", rdr_enable: ");
|
||||||
|
cur_config.append(std::to_string(rdr_enabled_));
|
||||||
|
MS_LOG(INFO) << cur_config;
|
||||||
|
}
|
||||||
|
} // namespace mindspore
|
||||||
@ -0,0 +1,60 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#ifndef MINDSPORE_CCSRC_DEBUG_ENV_CONFIG_PARSER_H_
|
||||||
|
#define MINDSPORE_CCSRC_DEBUG_ENV_CONFIG_PARSER_H_
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <map>
|
||||||
|
#include <set>
|
||||||
|
#include <mutex>
|
||||||
|
#include "nlohmann/json.hpp"
|
||||||
|
#include "utils/ms_utils.h"
|
||||||
|
namespace mindspore {
|
||||||
|
class EnvConfigParser {
|
||||||
|
public:
|
||||||
|
static EnvConfigParser &GetInstance() {
|
||||||
|
static EnvConfigParser instance;
|
||||||
|
return instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
void Parse();
|
||||||
|
|
||||||
|
bool rdr_enabled() const { return rdr_enabled_; }
|
||||||
|
std::string rdr_path() const { return rdr_path_; }
|
||||||
|
|
||||||
|
private:
|
||||||
|
EnvConfigParser() {}
|
||||||
|
~EnvConfigParser() {}
|
||||||
|
|
||||||
|
std::mutex lock_;
|
||||||
|
std::string config_file_{""};
|
||||||
|
bool already_parsed_{false};
|
||||||
|
bool rdr_enabled_{false};
|
||||||
|
std::string rdr_path_{"./rdr/"};
|
||||||
|
|
||||||
|
std::string GetIfstreamString(const std::ifstream &ifstream);
|
||||||
|
void ParseRdrSetting(const nlohmann::json &content);
|
||||||
|
|
||||||
|
bool CheckJsonStringType(const nlohmann::json &content, const std::string &setting_key, const std::string &key);
|
||||||
|
auto CheckJsonKeyExist(const nlohmann::json &content, const std::string &setting_key, const std::string &key);
|
||||||
|
|
||||||
|
void ParseRdrPath(const nlohmann::json &content);
|
||||||
|
void ParseRdrEnable(const nlohmann::json &content);
|
||||||
|
|
||||||
|
void ConfigToString();
|
||||||
|
};
|
||||||
|
} // namespace mindspore
|
||||||
|
#endif // MINDSPORE_CCSRC_DEBUG_ENV_CONFIG_PARSER_H_
|
||||||
Loading…
Reference in new issue