diff --git a/mindspore/ccsrc/backend/optimizer/common/pass_manager.cc b/mindspore/ccsrc/backend/optimizer/common/pass_manager.cc index 809441e992..3548555c19 100644 --- a/mindspore/ccsrc/backend/optimizer/common/pass_manager.cc +++ b/mindspore/ccsrc/backend/optimizer/common/pass_manager.cc @@ -67,8 +67,11 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us"; #endif if (save_graphs) { - auto file_name = "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir"; - DumpIR(file_name, func_graph, true); + std::ostringstream oss; + oss << "verbose_ir_files" + << "/"; + oss << "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir"; + DumpIR(oss.str(), func_graph, true); } num++; } diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index f1b5ed9879..b237f64d7e 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -4,6 +4,7 @@ set(_DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/draw.cc" "${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc" "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/common.cc" ) if (ENABLE_DEBUGGER) @@ -12,7 +13,6 @@ if (ENABLE_DEBUGGER) "${CMAKE_CURRENT_SOURCE_DIR}/debugger/grpc_client.cc" "${CMAKE_CURRENT_SOURCE_DIR}/debugger/proto_exporter.cc" "${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/common.cc" ) endif (ENABLE_DEBUGGER) diff --git a/mindspore/ccsrc/debug/anf_ir_dump.cc b/mindspore/ccsrc/debug/anf_ir_dump.cc index 02dd73735c..183deb0fbf 100644 --- a/mindspore/ccsrc/debug/anf_ir_dump.cc +++ b/mindspore/ccsrc/debug/anf_ir_dump.cc @@ -28,6 +28,7 @@ #include "backend/session/anf_runtime_algorithm.h" #include "frontend/parallel/ops_info/operator_info.h" #include "pipeline/jit/base.h" +#include "debug/common.h" namespace mindspore { const std::string ToShortString(const TypeId &typeId) { @@ -483,35 +484,23 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu if (graph == nullptr) { return; } - auto real_filename = pipeline::GetSaveGraphsPathName(AddGlobalId(filename)); - if (real_filename.size() > PATH_MAX) { - MS_LOG(ERROR) << "File path " << real_filename << " is too long."; + auto path = pipeline::GetSaveGraphsPathName(AddGlobalId(filename)); + auto realpath = Common::GetRealPath(path); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed. path=" << path; return; } - char real_path[PATH_MAX] = {0}; -#if defined(_WIN32) || defined(_WIN64) - if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) { - MS_LOG(DEBUG) << "dir " << filename << " does not exit."; - } -#else - if (nullptr == realpath(real_filename.c_str(), real_path)) { - MS_LOG(DEBUG) << "Dir " << real_filename << " does not exit."; - } -#endif - OrderedMap para_map; - std::string path_string = real_path; - ChangeFileMode(path_string, S_IRWXU); - std::ofstream fout(real_path); + ChangeFileMode(realpath.value(), S_IRWXU); + std::ofstream fout(realpath.value()); std::ostringstream buffer; - if (!fout.is_open()) { - MS_LOG(ERROR) << "Open dump file '" << real_path << "' failed!"; + MS_LOG(ERROR) << "Open dump file '" << realpath.value() << "' failed!"; return; } auto nodes = TopoSort(graph->get_return(), SuccDeeperSimple, AlwaysInclude); - + OrderedMap para_map; // dump global info DumpGlobalInfoEntry(graph, buffer); DumpParams(graph, buffer, ¶_map); @@ -528,7 +517,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu fout.close(); // set file mode to read only by user - ChangeFileMode(path_string, S_IRUSR); + ChangeFileMode(realpath.value(), S_IRUSR); } #else void DumpIR(const std::string &, const FuncGraphPtr &, bool) { diff --git a/mindspore/ccsrc/debug/common.cc b/mindspore/ccsrc/debug/common.cc index f3abbbc566..a7b696d03c 100644 --- a/mindspore/ccsrc/debug/common.cc +++ b/mindspore/ccsrc/debug/common.cc @@ -43,11 +43,19 @@ std::optional Common::GetRealPath(const std::string &input_path) { MS_LOG(ERROR) << "CreateNotExistDirs Failed!"; return std::nullopt; } - +#if defined(SYSTEM_ENV_POSIX) if (nullptr == realpath(prefix_path.c_str(), real_path)) { - MS_LOG(ERROR) << "dir " << prefix_path << " does not exit."; + MS_LOG(ERROR) << "dir " << prefix_path << " does not exist."; + return std::nullopt; + } +#elif defined(SYSTEM_ENV_WINDOWS) + if (nullptr == _fullpath(real_path, prefix_path.c_str(), PATH_MAX)) { + MS_LOG(ERROR) << "dir " << prefix_path << " does not exist."; return std::nullopt; } +#else + MS_LOG(EXCEPTION) << "Unsupported platform."; +#endif out_path = std::string(real_path) + last_path; } @@ -56,9 +64,17 @@ std::optional Common::GetRealPath(const std::string &input_path) { MS_LOG(ERROR) << "Prefix path is too longer!"; return std::nullopt; } +#if defined(SYSTEM_ENV_POSIX) if (nullptr == realpath(input_path.c_str(), real_path)) { - MS_LOG(ERROR) << "File " << input_path << " does not exit, it will be created."; + MS_LOG(ERROR) << "File " << input_path << " does not exist, it will be created."; + } +#elif defined(SYSTEM_ENV_WINDOWS) + if (nullptr == _fullpath(real_path, input_path.c_str(), PATH_MAX)) { + MS_LOG(ERROR) << "File " << input_path << " does not exist, it will be created."; } +#else + MS_LOG(EXCEPTION) << "Unsupported platform."; +#endif out_path = std::string(real_path); } return out_path; diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index 0bcc368592..118ac4016c 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include "ir/param_info.h" #include "pipeline/jit/pass.h" @@ -85,7 +86,7 @@ std::unordered_map(); return fs; +#elif defined(SYSTEM_ENV_WINDOWS) + auto fs = std::make_shared(); + return fs; #else MS_LOG(EXCEPTION) << "Now not support the platform."; #endif diff --git a/mindspore/ccsrc/utils/system/file_system.cc b/mindspore/ccsrc/utils/system/file_system.cc index ccb2321f0b..4eaf8e344e 100644 --- a/mindspore/ccsrc/utils/system/file_system.cc +++ b/mindspore/ccsrc/utils/system/file_system.cc @@ -16,14 +16,18 @@ #include "utils/system/file_system.h" +#if defined(SYSTEM_ENV_POSIX) #include #include +#elif defined(SYSTEM_ENV_WINDOWS) +#include +#endif namespace mindspore { namespace system { #if defined(SYSTEM_ENV_POSIX) -// Implement the Posix file systen +// Implement the Posix file system WriteFilePtr PosixFileSystem::CreateWriteFile(const string &file_name) { if (file_name.empty()) { MS_LOG(ERROR) << "Create write file failed because the file name is null."; @@ -49,7 +53,7 @@ bool PosixFileSystem::FileExist(const string &file_name) { } auto result = access(file_name.c_str(), F_OK); if (result != 0) { - MS_LOG(INFO) << "The file(" << file_name << ") not exist."; + MS_LOG(DEBUG) << "The file(" << file_name << ") not exist."; return false; } return true; @@ -62,7 +66,7 @@ bool PosixFileSystem::DeleteFile(const string &file_name) { } auto result = unlink(file_name.c_str()); if (result != 0) { - MS_LOG(ERROR) << "Delete the file(" << file_name << ") is falire, error(" << errno << ")."; + MS_LOG(ERROR) << "Delete the file(" << file_name << ") is failed, error(" << errno << ")."; return false; } return true; @@ -77,7 +81,7 @@ bool PosixFileSystem::CreateDir(const string &dir_name) { auto result = mkdir(dir_name.c_str(), DEFAULT_MKDIR_MODE); if (result != 0) { if (errno != EEXIST) { - MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is falire, error(" << errno << ")."; + MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is failed, error(" << errno << ")."; return false; } } @@ -98,5 +102,147 @@ bool PosixFileSystem::DeleteDir(const string &dir_name) { } #endif +#if defined(SYSTEM_ENV_WINDOWS) +// Implement the Windows file system +WriteFilePtr WinFileSystem::CreateWriteFile(const string &file_name) { + if (file_name.empty()) { + MS_LOG(ERROR) << "Create write file failed because the file name is null."; + return nullptr; + } + auto fp = std::make_shared(file_name); + if (fp == nullptr) { + MS_LOG(ERROR) << "Create write file(" << file_name << ") failed."; + return nullptr; + } + bool result = fp->Open(); + if (!result) { + MS_LOG(ERROR) << "Open the write file(" << file_name << ") failed."; + return nullptr; + } + return fp; +} + +bool WinFileSystem::FileExist(const string &file_name) { + if (file_name.empty()) { + MS_LOG(WARNING) << "The file name is null."; + return false; + } + auto result = access(file_name.c_str(), F_OK); + if (result != 0) { + MS_LOG(DEBUG) << "The file(" << file_name << ") not exist."; + return false; + } + return true; +} + +bool WinFileSystem::CreateDir(const string &dir_name) { + if (dir_name.empty()) { + MS_LOG(WARNING) << "The directory name is null."; + return false; + } + auto result = mkdir(dir_name.c_str()); + if (result != 0) { + MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is failed, error(" << result << ")."; + return false; + } + return true; +} + +bool WinFileSystem::DeleteDir(const string &dir_name) { + if (dir_name.empty()) { + MS_LOG(WARNING) << "The directory name is null."; + return false; + } + auto result = rmdir(dir_name.c_str()); + if (result != 0) { + MS_LOG(ERROR) << "Delete the dir(" << dir_name << ") is failed, error(" << result << ")."; + return false; + } + return true; +} + +bool WinFileSystem::DeleteFile(const string &file_name) { + if (file_name.empty()) { + MS_LOG(WARNING) << "The file name is null."; + return false; + } + auto result = unlink(file_name.c_str()); + if (result != 0) { + MS_LOG(ERROR) << "Delete the file(" << file_name << ") is failed, error(" << errno << ")."; + return false; + } + return true; +} + +bool WinWriteFile::Open() { + if (file_ != nullptr) { + MS_LOG(WARNING) << "The File(" << file_name_ << ") already open."; + return true; + } + // check the path + if (nullptr == file_name_.c_str()) { + MS_LOG(EXCEPTION) << "The file path is null."; + } + char path[PATH_MAX + 1] = {0x00}; + if (file_name_.size() > PATH_MAX || nullptr == _fullpath(path, file_name_.c_str(), PATH_MAX)) { + MS_LOG(EXCEPTION) << "Convert to real path fail, file name is " << file_name_ << "."; + } + + // open the file + file_ = fopen(path, "w+"); + if (file_ == nullptr) { + MS_LOG(ERROR) << "File(" << path << ") IO ERROR: " << errno << "."; + return false; + } + return true; +} + +bool WinWriteFile::Write(const std::string &data) { + MS_LOG(DEBUG) << "Write data(" << data.size() << ") to file(" << this->file_name_ << ")."; + size_t r = fwrite(data.data(), 1, data.size(), file_); + if (r != data.size()) { + MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << "."; + return false; + } + return true; +} + +bool WinWriteFile::Close() { + if (file_ == nullptr) { + MS_LOG(WARNING) << "File(" << file_name_ << ") already close."; + return true; + } + bool result = true; + if (fclose(file_) != 0) { + MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << "."; + result = false; + } + file_ = nullptr; + return result; +} + +bool WinWriteFile::Flush() { + if (fflush(file_) != 0) { + MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << EBADF << "."; + return false; + } + return true; +} + +bool WinWriteFile::Sync() { return Flush(); } + +WinWriteFile::~WinWriteFile() { + try { + if (file_ != nullptr) { + (void)fclose(file_); + file_ = nullptr; + } + } catch (const std::exception &e) { + MS_LOG(ERROR) << "Exception when closing file."; + } catch (...) { + MS_LOG(ERROR) << "Non standard exception when closing file."; + } +} +#endif } // namespace system } // namespace mindspore diff --git a/mindspore/ccsrc/utils/system/file_system.h b/mindspore/ccsrc/utils/system/file_system.h index ed9db874c8..4a08625ccd 100644 --- a/mindspore/ccsrc/utils/system/file_system.h +++ b/mindspore/ccsrc/utils/system/file_system.h @@ -198,6 +198,51 @@ class PosixWriteFile : public WriteFile { }; #endif +#if defined(SYSTEM_ENV_WINDOWS) +// File system of create or delete directory for windows system +class WinFileSystem : public FileSystem { + public: + WinFileSystem() = default; + + ~WinFileSystem() override = default; + + // create a new write file + WriteFilePtr CreateWriteFile(const string &file_name) override; + + // check the file is exist? + bool FileExist(const string &file_name) override; + + // delete the file + bool DeleteFile(const string &file_name) override; + + // Create a Directory + bool CreateDir(const string &dir_name) override; + + // Delete the specified directory. + bool DeleteDir(const string &dir_name) override; +}; + +// A file that can be read and write for windows +class WinWriteFile : public WriteFile { + public: + explicit WinWriteFile(const string &file_name) : WriteFile(file_name), file_(nullptr) {} + + ~WinWriteFile() override; + + bool Open() override; + + bool Write(const std::string &data) override; + + bool Close() override; + + bool Flush() override; + + bool Sync() override; + + private: + FILE *file_; +}; +#endif } // namespace system } // namespace mindspore diff --git a/tests/ut/python/pynative_mode/test_backend.py b/tests/ut/python/pynative_mode/test_backend.py index 0b3e003d78..b0f660fbaf 100644 --- a/tests/ut/python/pynative_mode/test_backend.py +++ b/tests/ut/python/pynative_mode/test_backend.py @@ -14,6 +14,7 @@ # ============================================================================ """ test_backend """ import os +import shutil import pytest import mindspore.nn as nn @@ -86,6 +87,6 @@ def teardown_module(): if not os.path.exists(item_name): continue if os.path.isdir(item_name): - os.rmdir(item_name) + shutil.rmtree(item_name) elif os.path.isfile(item_name): os.remove(item_name) diff --git a/tests/ut/python/pynative_mode/test_context.py b/tests/ut/python/pynative_mode/test_context.py index e2d4e31412..77e71e969b 100644 --- a/tests/ut/python/pynative_mode/test_context.py +++ b/tests/ut/python/pynative_mode/test_context.py @@ -14,6 +14,7 @@ # ============================================================================ """ test_context """ import os +import shutil import pytest from mindspore import context @@ -150,6 +151,6 @@ def teardown_module(): if not os.path.exists(item_name): continue if os.path.isdir(item_name): - os.rmdir(item_name) + shutil.rmtree(item_name) elif os.path.isfile(item_name): os.remove(item_name)