m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

building _ms_mpi with mpi_interface

Browse Source
pull/2428/head
chenjianping 5 years ago
parent c9b8a8da0a
commit 343889cdb7
13 changed files with 161 additions and 125 deletions
Show Stats Download Patch File Download Diff File

5
cmake/package.cmake
Unescape View File

@ -128,6 +128,11 @@ if (ENABLE_MPI)
DESTINATION ${INSTALL_BASE_DIR}
COMPONENT mindspore
)
install(
TARGETS mpi_adapter
DESTINATION ${INSTALL_LIB_DIR}
COMPONENT mindspore
)
endif ()
if (ENABLE_GPU)

5
mindspore/ccsrc/CMakeLists.txt
Unescape View File

@ -126,11 +126,12 @@ endforeach ()
set_property(SOURCE ${SUB_OBJECTS_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ME)
add_library(mindspore STATIC ${SUB_OBJECTS_SRC})
target_link_libraries(mindspore proto_input)
if (ENABLE_CPU AND ENABLE_MPI)
target_link_libraries(mindspore securec mindspore::flatbuffers mindspore::ompi)
if (ENABLE_MPI)
target_link_libraries(mindspore securec mindspore::flatbuffers mpi_adapter)
else ()
target_link_libraries(mindspore securec mindspore::flatbuffers)
endif ()
if (NOT WIN32)
target_link_libraries(mindspore dl)
endif()

17
mindspore/ccsrc/device/CMakeLists.txt
Unescape View File

@ -14,17 +14,22 @@ endif ()
if (ENABLE_CPU)
file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cpu/*.cc")
if (NOT ENABLE_MPI)
list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_adapter.cc")
endif ()
list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_adapter.cc")
list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_interface.cc")
endif ()
if (ENABLE_MPI)
# _ms_mpi
set_property(SOURCE "gpu/mpi/mpi_initializer.cc"
file(GLOB_RECURSE MPI_SRC_LIST "cpu/mpi/mpi_adapter.cc")
set_property(SOURCE ${MPI_SRC_LIST}
PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE)
add_library(mpi_adapter SHARED ${MPI_SRC_LIST})
target_link_libraries(mpi_adapter PRIVATE mindspore::ompi)
set_property(SOURCE "cpu/mpi/mpi_interface.cc"
PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE)
pybind11_add_module(_ms_mpi "gpu/mpi/mpi_initializer.cc")
target_link_libraries(_ms_mpi PRIVATE mindspore::pybind11_module mindspore::ompi)
pybind11_add_module(_ms_mpi "cpu/mpi/mpi_interface.cc")
target_link_libraries(_ms_mpi PRIVATE mindspore::pybind11_module mpi_adapter)
endif ()
# gpu

123
mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc
View File

File diff suppressed because it is too large Load Diff

40
mindspore/ccsrc/device/cpu/mpi/mpi_adapter.h
Unescape View File

@ -22,37 +22,53 @@
#include <map>
#include <string>
#include <mutex>
#endif // ENABLE_MPI
#include <memory>
namespace mindspore {
namespace device {
namespace cpu {
#ifndef FUNC_EXPORT
#define FUNC_EXPORT __attribute__((visibility("default")))
#endif
constexpr auto kOpTypeSum = "sum";
class MPIAdapter {
public:
~MPIAdapter();
static MPIAdapter &Instance();
int GetRankId() const;
bool ReduceScatter(const float *input, float *output, const std::vector<int> &ranks_group, size_t data_num,
const std::string &op_type = kOpTypeSum);
bool ReduceScatterOverwriteInput(float *input, const std::vector<int> &ranks_group, size_t input_data_num,
size_t output_size, const std::string &op_type = kOpTypeSum,
float *output = nullptr);
bool AllGather(const float *input, float *output, const std::vector<int> &ranks_group, size_t data_num);
FUNC_EXPORT static std::shared_ptr<MPIAdapter> Instance();
FUNC_EXPORT int GetRankId() const { return rank_id_; }
FUNC_EXPORT int GetRankSize() const { return rank_size_; }
#ifdef ENABLE_MPI
FUNC_EXPORT ~MPIAdapter();
FUNC_EXPORT bool ReduceScatter(const float *input, float *output, const std::vector<int> &ranks_group,
size_t data_num, const std::string &op_type = kOpTypeSum);
FUNC_EXPORT bool ReduceScatterOverwriteInput(float *input, const std::vector<int> &ranks_group, size_t in_data_num,
size_t output_size, const std::string &op_type = kOpTypeSum,
float *output = nullptr);
FUNC_EXPORT bool AllGather(const float *input, float *output, const std::vector<int> &ranks_group, size_t data_num);
#else
FUNC_EXPORT ~MPIAdapter() = default;
#endif // ENABLE_MPI
private:
#ifdef ENABLE_MPI
MPIAdapter();
void Init();
MPI_Group AddGroup(const std::vector<int> &ranks);
int rank_id_;
int rank_size_;
MPI_Group comm_group_world_;
// key:ranks group, value: mpi group
std::map<std::vector<int>, MPI_Group> ranks_group_;
std::mutex group_mutex_;
#else
MPIAdapter() = default;
#endif // ENABLE_MPI
int rank_id_{-1};
int rank_size_{0};
static std::shared_ptr<MPIAdapter> instance_;
};
} // namespace cpu
} // namespace device
} // namespace mindspore
#endif // ENABLE_MPI
#endif // MINDSPORE_CCSRC_DEVICE_CPU_MPI_MPI_ADAPTER_H_

33
mindspore/ccsrc/device/cpu/mpi/mpi_interface.cc
Unescape View File

@ -0,0 +1,33 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <pybind11/operators.h>
#include "device/cpu/mpi/mpi_adapter.h"
namespace mindspore {
namespace device {
namespace cpu {
int get_rank_id() { return MPIAdapter::Instance()->GetRankId(); }
int get_rank_size() { return MPIAdapter::Instance()->GetRankSize(); }
PYBIND11_MODULE(_ms_mpi, mpi_interface) {
mpi_interface.doc() = "mindspore mpi python wrapper";
mpi_interface.def("get_rank_id", &get_rank_id, "get rank id");
mpi_interface.def("get_rank_size", &get_rank_size, "get rank size");
}
} // namespace cpu
} // namespace device
} // namespace mindspore

7
mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc
Unescape View File

@ -17,7 +17,6 @@
#include "device/gpu/mpi/mpi_initializer.h"
#include <mpi.h>
#include <pybind11/operators.h>
#include <iostream>
namespace mindspore {
@ -54,12 +53,6 @@ MPIInitializer &MPIInitializer::GetInstance() {
int MPIInitializer::get_rank_id() { return MPIInitializer::GetInstance().rank_id_; }
int MPIInitializer::get_rank_size() { return MPIInitializer::GetInstance().rank_size_; }
PYBIND11_MODULE(_ms_mpi, mpi_initializer) {
mpi_initializer.doc() = "mindspore mpi python wrapper";
mpi_initializer.def("get_rank_id", &MPIInitializer::get_rank_id, "get rank id");
mpi_initializer.def("get_rank_size", &MPIInitializer::get_rank_size, "get rank size");
}
} // namespace gpu
} // namespace device
} // namespace mindspore

2
mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc
Unescape View File

@ -47,7 +47,7 @@ bool AllGatherCPUKernel::Launch(const std::vector<kernel::AddressPtr> &inputs,
auto output_addr = reinterpret_cast<float *>(outputs[0]->addr);
auto input_data_num = inputs[0]->size / sizeof(float);
return device::cpu::MPIAdapter::Instance().AllGather(input_addr, output_addr, ranks_group_, input_data_num);
return device::cpu::MPIAdapter::Instance()->AllGather(input_addr, output_addr, ranks_group_, input_data_num);
}
} // namespace kernel
} // namespace mindspore

4
mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc
Unescape View File

@ -51,8 +51,8 @@ bool EmbeddingLookUpCommGradCPUKernel::Launch(const std::vector<kernel::AddressP
size_t input_split_lens = input_size / split_num_ / sizeof(float_t);
size_t output_split_lens = output_size / split_num_ / sizeof(float_t);
for (int i = 0; i < split_num_; i++) {
device::cpu::MPIAdapter::Instance().AllGather(input_addr + i * input_split_lens,
output_addr + i * output_split_lens, rank_group, input_split_lens);
device::cpu::MPIAdapter::Instance()->AllGather(input_addr + i * input_split_lens,
output_addr + i * output_split_lens, rank_group, input_split_lens);
}
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();

6
mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc
Unescape View File

@ -105,9 +105,9 @@ bool EmbeddingLookUpCPUKernel::Launch(const std::vector<kernel::AddressPtr> &inp
size_t reduce_scatter_out_lens = one_split_lens / 8;
const std::vector<int> &group = {0, 1, 2, 3, 4, 5, 6, 7};
for (int i = 0; i < split_num_; i++) {
device::cpu::MPIAdapter::Instance().ReduceScatter(reinterpret_cast<float *>(gather_v2_out_) + i * one_split_lens,
output_addr + i * reduce_scatter_out_lens, group,
one_split_lens / 8, "sum");
device::cpu::MPIAdapter::Instance()->ReduceScatter(reinterpret_cast<float *>(gather_v2_out_) + i * one_split_lens,
output_addr + i * reduce_scatter_out_lens, group,
one_split_lens / 8, "sum");
}
}
#endif

4
mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc
Unescape View File

@ -47,8 +47,8 @@ bool ReduceScatterCPUKernel::Launch(const std::vector<kernel::AddressPtr> &input
auto output_addr = reinterpret_cast<float *>(outputs[0]->addr);
auto output_data_num = outputs[0]->size / sizeof(float);
return device::cpu::MPIAdapter::Instance().ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num,
op_type_);
return device::cpu::MPIAdapter::Instance()->ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num,
op_type_);
}
} // namespace kernel
} // namespace mindspore

38
mindspore/context.py
Unescape View File

@ -25,7 +25,6 @@ from mindspore._c_expression import MSContext
from mindspore._checkparam import args_type_check
from mindspore.parallel._auto_parallel_context import _set_auto_parallel_context, _get_auto_parallel_context, \
_reset_auto_parallel_context
from mindspore.parallel.mpi._mpi_config import _set_mpi_config, _get_mpi_config
__all__ = ['GRAPH_MODE', 'PYNATIVE_MODE', 'set_context', 'get_context', 'set_auto_parallel_context',
'get_auto_parallel_context', 'reset_auto_parallel_context']
@ -608,40 +607,3 @@ def get_context(attr_key):
raise ValueError(
"Get context keyword %s is not recognized!" % attr_key)
return getattr(_context(), attr_key)
@args_type_check(enable_mpi=bool)
def set_mpi_config(**kwargs):
"""
Sets mpi config for running environment.
mpi config should be configured before running your program. If there is no configuration,
mpi moudle will be disabled by default.
Note:
Attribute name is required for setting attributes.
Args:
enable_mpi (bool): Whether to enable mpi. Default: False.
Raises:
ValueError: If input key is not an attribute in mpi config.
Examples:
>>> mpiconfig.set_mpi_config(enable_mpi=True)
"""
_set_mpi_config(**kwargs)
def get_mpi_config(attr_key):
"""
Gets mpi config attribute value according to the input key.
Args:
attr_key (str): The key of the attribute.
Returns:
Object, The value of given attribute key.
Raises:
ValueError: If input key is not an attribute in context.
"""
return _get_mpi_config(attr_key)

2
mindspore/parallel/mpi/_mpi_config.py
Unescape View File

@ -104,7 +104,7 @@ def _get_mpi_config(attr_key):
Object, The value of given attribute key.
Raises:
ValueError: If input key is not an attribute in context.
ValueError: If input key is not an attribute in config.
"""
if not hasattr(_mpi_config(), attr_key):
raise ValueError("Get context keyword %s is not recognized!" % attr_key)

Write Preview
Loading…
Cancel
Save