!9532 build for mac

From: @xiaoyisd
Reviewed-by: 
Signed-off-by:
pull/9532/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
commit c2ad16352f

@ -13,7 +13,8 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
endif ()
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
set(CMAKE_OSX_SYSROOT "")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wuser-defined-warnings -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
else()
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
endif()

@ -5,14 +5,14 @@ set(LIB_ICU_I18N icui18n)
if (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/icu/repository/archive/release-67-1.tar.gz")
set(MD5 "72415ffd1af3acf19f9aa3fa82c7b5bc")
else()
else ()
set(REQ_URL "https://github.com/unicode-org/icu/archive/release-67-1.tar.gz")
set(MD5 "fd525fb47d8827b0b7da78b51dd2d93f")
endif ()
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
message("icu4c thirdparty do not support windows currently.")
else()
else ()
set(JSON_FILE "{ \n\
\"strategy\": \"additive\",\n\
\"featureFilters\": {\n\
@ -21,17 +21,28 @@ else()
}\
")
file(WRITE ${CMAKE_BINARY_DIR}/icu4c_filter.json ${JSON_FILE})
mindspore_add_pkg(icu4c
VER 67.1
LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N}
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU Linux --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
mindspore_add_pkg(icu4c
VER 67.1
LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N}
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
else ()
mindspore_add_pkg(icu4c
VER 67.1
LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N}
URL ${REQ_URL}
MD5 ${MD5}
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01
CONFIGURE_COMMAND ./icu4c/source/runConfigureICU Linux --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json
)
endif ()
include_directories(${icu4c_INC})
add_library(mindspore::icuuc ALIAS icu4c::${LIB_ICU_COMMON})
add_library(mindspore::icudata ALIAS icu4c::${LIB_ICU_DATA})
add_library(mindspore::icui18n ALIAS icu4c::${LIB_ICU_I18N})
add_definitions(-D ENABLE_ICU4C)
endif()
endif ()

@ -1,5 +1,7 @@
set(libevent_CFLAGS "-fstack-protector-all -D_FORTIFY_SOURCE=2 -O2")
set(libevent_LDFLAGS "-Wl,-z,now")
if (NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(libevent_LDFLAGS "-Wl,-z,now")
endif ()
if (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/libevent/repository/archive/release-2.1.12-stable.tar.gz")

@ -24,8 +24,8 @@ else ()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
else()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif()
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
mindspore_add_pkg(sqlite
VER 3.32.2
LIBS sqlite3

@ -4,15 +4,15 @@ set(tinyxml2_CFLAGS "-fstack-protector -D_FORTIFY_SOURCE=2 -O2")
if (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/tinyxml2/repository/archive/8.0.0.tar.gz")
set(MD5 "6a70cea637d0b17179e8bfd77860f811")
else()
else ()
set(REQ_URL "https://github.com/leethomason/tinyxml2/archive/8.0.0.tar.gz")
set(MD5 "5dc535c8b34ee621fe2128f072d275b5")
endif ()
if (NOT WIN32)
set(tinyxml2_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif()
if (NOT WIN32 AND NOT APPLE)
set(tinyxml2_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif ()
mindspore_add_pkg(tinyxml2
VER 8.0.0

@ -2,7 +2,10 @@
include_directories(${CMAKE_SOURCE_DIR}/mindspore/core)
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories(${CMAKE_BINARY_DIR})
link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine)
if (NOT(CMAKE_SYSTEM_NAME MATCHES "Darwin"))
link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine)
endif ()
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
@ -252,9 +255,11 @@ if (ENABLE_D)
target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf -Wl,--end-group)
elseif (CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf mindspore::sentencepiece -Wl,--end-group)
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore -Wl proto_input mindspore::protobuf mindspore::sentencepiece -Wl)
else ()
target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf -Wl,--end-group)
endif()
endif ()
# set c_expression building
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
@ -296,6 +301,10 @@ if (CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(mindspore mindspore::pybind11_module)
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive)
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore mindspore::pybind11_module)
target_link_libraries(mindspore mindspore_gvar)
target_link_libraries(_c_expression PRIVATE -Wl,-force_load mindspore -Wl,-noall_load)
else ()
if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU))
target_link_libraries(mindspore mindspore::pslite proto_input mindspore::protobuf mindspore::event mindspore::event_pthreads ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a)
@ -326,7 +335,7 @@ if (ENABLE_GPU)
${CUDA_PATH}/lib64/libcusolver.so)
if (ENABLE_MPI)
set_target_properties(_ms_mpi PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
endif()
endif ()
endif ()
if (ENABLE_CPU)
@ -349,8 +358,14 @@ add_library(inference SHARED
)
set_target_properties(inference PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})
target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore proto_input -Wl,--no-whole-archive mindspore_gvar)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,-force_load mindspore proto_input -Wl,-noall_load mindspore_gvar)
else()
target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore proto_input -Wl,--no-whole-archive mindspore_gvar)
endif()
if (ENABLE_D)
find_library(adump_server libadump_server.a ${ASCEND_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH})

@ -5,6 +5,10 @@ file(GLOB_RECURSE KERNEL_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"oplib/*.cc"
)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
endif ()
if (ENABLE_D)
file(GLOB_RECURSE D_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"kernel_query.cc"

@ -22,7 +22,7 @@ namespace kernel {
void SigmoidCrossEntropyWithLogitsCPUKernel::InitKernel(const CNodePtr &kernel_node) {
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

@ -22,7 +22,7 @@ namespace kernel {
void SigmoidCrossEntropyWithLogitsGradCPUKernel::InitKernel(const CNodePtr &kernel_node) {
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

@ -23,7 +23,7 @@ void SmoothL1LossCPUKernel::InitKernel(const CNodePtr &kernel_node) {
beta_ = AnfAlgo::GetNodeAttr<float>(kernel_node, "beta");
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

@ -23,7 +23,7 @@ void SmoothL1LossGradCPUKernel::InitKernel(const CNodePtr &kernel_node) {
beta_ = AnfAlgo::GetNodeAttr<float>(kernel_node, "beta");
CheckParam(kernel_node);
dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0);
std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0);
for (const uint64_t &d : x_shape) {
tensor_size_ *= d;
}

@ -21,5 +21,9 @@ if (ENABLE_GPU)
list(APPEND _PREACTIVATE_SRC_LIST ${_GPU_SRC_LIST})
endif ()
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -Wno-user-defined-warnings -Wno-inconsistent-missing-override -Wno-overloaded-virtual -Wno-unused-const-variable -Wno-pessimizing-move")
endif ()
set_property(SOURCE ${_PREACTIVATE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PRE_ACT)
add_library(_mindspore_backend_optimizer_obj OBJECT ${_PREACTIVATE_SRC_LIST})

@ -8,17 +8,17 @@ file(GLOB_RECURSE _SESSION_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"anf_runtime_algorithm.cc"
)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual")
endif ()
if (ENABLE_GPU)
file(GLOB_RECURSE _GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"gpu_session.cc"
)
file(GLOB_RECURSE _GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "gpu_session.cc")
list(APPEND _SESSION_SRC_LIST ${_GPU_SRC_LIST})
endif ()
if (ENABLE_CPU)
file(GLOB_RECURSE _CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"cpu_session.cc"
)
file(GLOB_RECURSE _CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cpu_session.cc")
list(APPEND _SESSION_SRC_LIST ${_CPU_SRC_LIST})
endif ()

@ -257,7 +257,7 @@ GraphId Executor::CompileGraph(const SessionPtr &session, const GraphSegmentPtr
GraphId Executor::CompileGraph(const SessionPtr &session, NotNull<FuncGraphPtr> func_graph) {
auto task = std::make_shared<CompileGraphTask>();
task->session_ = session;
task->func_graph_ = func_graph;
task->func_graph_ = func_graph.get();
SyncRunTask(task);
return task->graph_id_;
}

@ -472,7 +472,11 @@ void KernelGraph::SetKernelInfoForNode(const AnfNodePtr &node) const {
if (kOpAssignKernelNameList.find(AnfAlgo::GetCNodeName(node)) != kOpAssignKernelNameList.end()) {
ResetAssignInputFeaatureMapFlag(node->cast<CNodePtr>());
}
#ifdef __APPLE__
std::vector<int> feature_map_input_indexs;
#else
std::vector<size_t> feature_map_input_indexs;
#endif
kernel_info->set_feature_map_flag(false);
for (size_t index = 0; index < AnfAlgo::GetInputTensorNum(node); ++index) {
if (AnfAlgo::IsFeatureMapInput(node, index)) {

@ -53,7 +53,11 @@ struct OpRunInfo {
bool is_dynamic_shape = false;
bool is_auto_mixed_precision = false;
std::string next_op_name = "";
#ifdef __APPLE__
int next_input_index = 0;
#else
size_t next_input_index = 0;
#endif
};
using OpRunInfoPtr = std::shared_ptr<OpRunInfo>;
class Executor;

@ -36,8 +36,13 @@ set(MSLIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/types.cc
add_library(mindspore_shared_lib SHARED ${MSLIB_SRC})
set_target_properties(mindspore_shared_lib PROPERTIES OUTPUT_NAME mindspore)
target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore -Wl,--no-whole-archive proto_input mindspore_gvar mindspore::protobuf)
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,-force_load mindspore -Wl,-noall_load proto_input mindspore_gvar mindspore::protobuf)
else ()
target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}
-Wl,--whole-archive mindspore -Wl,--no-whole-archive proto_input mindspore_gvar mindspore::protobuf)
endif ()
if (ENABLE_CPU)
target_link_libraries(mindspore_shared_lib PRIVATE mindspore::dnnl mindspore::mkldnn)

@ -1,3 +1,8 @@
file(GLOB_RECURSE _OPTIMIZER_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
endif ()
set_property(SOURCE ${_OPTIMIZER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_OPTIMIZER)
add_library(_mindspore_frontend_optimizer_obj OBJECT ${_OPTIMIZER_SRC_FILES})

@ -4,5 +4,9 @@ if (ENABLE_DUMP_PROTO)
list(REMOVE_ITEM _PARALLEL_SRC_FILES "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc")
endif ()
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
endif ()
set_property(SOURCE ${_PARALLEL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PARALLEL)
add_library(_mindspore_frontend_parallel_obj OBJECT ${_PARALLEL_SRC_FILES})

@ -111,7 +111,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
case kNumberTypeInt:
return sizeof(int64_t);
case kNumberTypeUInt:
return sizeof(unsigned int64_t);
return sizeof(unsigned);
case kNumberTypeFloat:
return sizeof(float);
default:

@ -3,7 +3,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch")
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations -Wno-delete-non-abstract-non-virtual-dtor -Wno-constant-conversion")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized")
endif()
@ -49,16 +49,20 @@ include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels
####################### Flags ########################################
# compile flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default")
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-abstract-non-virtual-dtor")
else ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib")
endif()
################## Include sub-modules ###############################
add_subdirectory(util)
add_subdirectory(core)
add_subdirectory(kernels)
add_subdirectory(engine)
add_subdirectory(api)
add_subdirectory(text)
add_subdirectory(callback)
######################################################################
add_dependencies(utils core)
@ -75,16 +79,20 @@ add_dependencies(engine-perf core)
add_dependencies(engine-gnn core)
add_dependencies(engine core)
add_dependencies(callback core)
add_dependencies(text core)
add_dependencies(text-kernels core)
add_dependencies(cpp-API core)
add_dependencies(engine-ir-datasetops core)
add_dependencies(engine-ir-datasetops-source core)
add_dependencies(engine-ir-cache core)
if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin"))
add_subdirectory(text)
add_dependencies(text core)
add_dependencies(text-kernels core)
endif ()
if (ENABLE_PYTHON)
add_dependencies(APItoPython core)
endif()
endif ()
if (ENABLE_TDTQUE)
add_dependencies(engine-tdt core)
endif ()
@ -100,37 +108,42 @@ if (ENABLE_CACHE)
endif ()
################### Create _c_dataengine Library ######################
set(submodules
$<TARGET_OBJECTS:core>
$<TARGET_OBJECTS:callback>
$<TARGET_OBJECTS:utils>
$<TARGET_OBJECTS:kernels>
$<TARGET_OBJECTS:kernels-image>
$<TARGET_OBJECTS:lite-cv>
$<TARGET_OBJECTS:kernels-data>
$<TARGET_OBJECTS:cpp-API>
$<TARGET_OBJECTS:engine-ir-datasetops>
$<TARGET_OBJECTS:core>
$<TARGET_OBJECTS:callback>
$<TARGET_OBJECTS:utils>
$<TARGET_OBJECTS:kernels>
$<TARGET_OBJECTS:kernels-image>
$<TARGET_OBJECTS:lite-cv>
$<TARGET_OBJECTS:kernels-data>
$<TARGET_OBJECTS:cpp-API>
$<TARGET_OBJECTS:engine-ir-datasetops>
$<TARGET_OBJECTS:engine-ir-datasetops-source>
$<TARGET_OBJECTS:engine-ir-cache>
$<TARGET_OBJECTS:kernels-soft-dvpp-image>
$<TARGET_OBJECTS:soft-dvpp-utils>
$<TARGET_OBJECTS:engine-datasetops-source>
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
$<TARGET_OBJECTS:engine-datasetops-mapop>
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:engine-perf>
$<TARGET_OBJECTS:engine-datasetops>
$<TARGET_OBJECTS:engine-opt>
$<TARGET_OBJECTS:engine-cache-client>
$<TARGET_OBJECTS:engine>
$<TARGET_OBJECTS:text>
$<TARGET_OBJECTS:text-kernels>
)
$<TARGET_OBJECTS:soft-dvpp-utils>
$<TARGET_OBJECTS:engine-datasetops-source>
$<TARGET_OBJECTS:engine-datasetops-source-sampler>
$<TARGET_OBJECTS:engine-datasetops-mapop>
$<TARGET_OBJECTS:engine-gnn>
$<TARGET_OBJECTS:engine-perf>
$<TARGET_OBJECTS:engine-datasetops>
$<TARGET_OBJECTS:engine-opt>
$<TARGET_OBJECTS:engine-cache-client>
$<TARGET_OBJECTS:engine>
)
if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin"))
set(submodules
${submodules}
$<TARGET_OBJECTS:text>
$<TARGET_OBJECTS:text-kernels>
)
endif ()
if (ENABLE_PYTHON)
set(submodules
${submodules}
$<TARGET_OBJECTS:APItoPython>)
endif()
endif ()
if (ENABLE_TDTQUE)
add_library(_c_dataengine SHARED ${submodules} $<TARGET_OBJECTS:engine-tdt>)
@ -143,7 +156,7 @@ set_target_properties(_c_dataengine PROPERTIES
PREFIX "${PYTHON_MODULE_PREFIX}"
SUFFIX "${PYTHON_MODULE_EXTENSION}"
)
endif()
endif ()
######################################################################
@ -152,18 +165,18 @@ target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
if (ENABLE_PYTHON)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY})
else()
else ()
target_link_libraries(_c_dataengine PRIVATE ${SECUREC_LIBRARY})
endif()
else()
endif ()
else ()
set(ICU_LIB mindspore::icuuc mindspore::icudata mindspore::icui18n)
if (ENABLE_PYTHON)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl ${SECUREC_LIBRARY})
else()
else ()
target_link_libraries(_c_dataengine PRIVATE -ldl ${SECUREC_LIBRARY})
endif()
endif ()
target_link_libraries(_c_dataengine PUBLIC mindspore::sentencepiece)
endif()
endif ()
target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::turbojpeg mindspore::opencv_core mindspore::opencv_imgcodecs
mindspore::opencv_imgproc mindspore::tinyxml2 mindspore::sentencepiece_train ${ICU_LIB})
if (ENABLE_GPUQUE)
@ -181,26 +194,26 @@ add_dependencies(_c_dataengine _c_mindrecord)
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/minddata/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite)
else()
else ()
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU))
target_link_libraries(_c_dataengine PRIVATE mindspore::pslite ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a)
if (${ENABLE_IBVERBS} STREQUAL "ON")
target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm)
endif()
endif()
endif()
endif ()
endif ()
endif ()
if (USE_GLOG)
target_link_libraries(_c_dataengine PRIVATE mindspore::glog)
else()
else ()
if (CMAKE_SYSTEM_NAME MATCHES "Linux")
target_link_options(_c_dataengine PRIVATE -Wl,-init,mindspore_log_init)
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_dataengine PROPERTIES MACOSX_RPATH ON)
endif ()
endif()
endif ()
if (MS_BUILD_GRPC)
target_link_libraries(_c_dataengine PRIVATE mindspore::grpc++)
endif()
endif ()

@ -1,37 +1,71 @@
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
if (ENABLE_PYTHON)
add_library(APItoPython OBJECT
python/pybind_register.cc
python/pybind_conversion.cc
python/bindings/dataset/include/datasets_bindings.cc
python/bindings/dataset/include/iterator_bindings.cc
python/bindings/dataset/include/execute_binding.cc
python/bindings/dataset/include/schema_bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/kernels/data/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/datasetops/source/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/kernels/image/bindings.cc
python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/bindings.cc
python/bindings/mindrecord/include/bindings.cc
)
if (APPLE)
add_library(APItoPython OBJECT
python/pybind_register.cc
python/pybind_conversion.cc
python/bindings/dataset/include/datasets_bindings.cc
python/bindings/dataset/include/iterator_bindings.cc
python/bindings/dataset/include/execute_binding.cc
python/bindings/dataset/include/schema_bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/kernels/data/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/datasetops/source/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/kernels/image/bindings.cc
python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
python/bindings/mindrecord/include/bindings.cc
)
else()
add_library(APItoPython OBJECT
python/pybind_register.cc
python/pybind_conversion.cc
python/bindings/dataset/include/datasets_bindings.cc
python/bindings/dataset/include/iterator_bindings.cc
python/bindings/dataset/include/execute_binding.cc
python/bindings/dataset/include/schema_bindings.cc
python/bindings/dataset/engine/cache/bindings.cc
python/bindings/dataset/core/bindings.cc
python/bindings/dataset/callback/bindings.cc
python/bindings/dataset/kernels/data/bindings.cc
python/bindings/dataset/kernels/bindings.cc
python/bindings/dataset/engine/datasetops/bindings.cc
python/bindings/dataset/engine/datasetops/source/bindings.cc
python/bindings/dataset/engine/gnn/bindings.cc
python/bindings/dataset/kernels/image/bindings.cc
python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
python/bindings/dataset/text/bindings.cc
python/bindings/dataset/text/kernels/bindings.cc
python/bindings/mindrecord/include/bindings.cc
)
endif()
target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS})
endif ()
add_library(cpp-API OBJECT
config.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
if (APPLE)
add_library(cpp-API OBJECT
config.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
vision.cc
)
else()
add_library(cpp-API OBJECT
config.cc
datasets.cc
execute.cc
iterator.cc
transforms.cc
samplers.cc
text.cc
vision.cc
)
endif()

@ -695,13 +695,17 @@ Status Tensor::GetDataAsNumpy(py::array *data) {
}
Status Tensor::GetDataAsNumpyStrings(py::array *data) {
auto itr = begin<std::string_view>();
uint64_t max = 0;
uint64_t max_value = 0;
for (; itr != end<std::string_view>(); itr++) {
max = std::max((*itr).length(), max);
#ifdef __APPLE__
max_value = fmax((*itr).length(), max_value);
#else
max_value = std::max((*itr).length(), max_value);
#endif
}
// if all strings are empty, numpy stores a byte for each string |S1
max = (max == 0 ? 1 : max);
uint64_t total_size = shape_.NumOfElements() * max;
max_value = (max_value == 0 ? 1 : max_value);
uint64_t total_size = shape_.NumOfElements() * max_value;
char *tmp_data = reinterpret_cast<char *>(data_allocator_->allocate(total_size));
if (tmp_data == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create temp array.");
int ret_code = memset_s(tmp_data, total_size, 0, total_size);
@ -711,13 +715,14 @@ Status Tensor::GetDataAsNumpyStrings(py::array *data) {
uint64_t i = 0;
for (; itr != end<std::string_view>(); itr++, i++) {
if (!(*itr).empty()) {
ret_code = memcpy_s(tmp_data + i * max, total_size, (*itr).data(), (*itr).length());
ret_code = memcpy_s(tmp_data + i * max_value, total_size, (*itr).data(), (*itr).length());
CHECK_FAIL_RETURN_UNEXPECTED(ret_code == 0, "Failed to copy string data.");
}
}
auto strides = shape_.Strides();
std::transform(strides.begin(), strides.end(), strides.begin(), [&max](const auto &s) { return s * max; });
*data = py::array(py::dtype("S" + std::to_string(max)), shape_.AsVector(), strides, tmp_data);
std::transform(strides.begin(), strides.end(), strides.begin(),
[&max_value](const auto &s) { return s * max_value; });
*data = py::array(py::dtype("S" + std::to_string(max_value)), shape_.AsVector(), strides, tmp_data);
data_allocator_->deallocate(reinterpret_cast<uchar *>(tmp_data));
return Status::OK();
}

@ -154,6 +154,9 @@ Status CacheServerHW::GetNumaNodeInfo() {
}
Status CacheServerHW::SetAffinity(const Task &tk, numa_id_t numa_node) {
#ifdef __APPLE__
return Status::OK();
#else
auto r = numa_cpuset_.find(numa_node);
if (r != numa_cpuset_.end()) {
auto err = pthread_setaffinity_np(tk.GetNativeHandle(), sizeof(r->second), &r->second);
@ -165,6 +168,7 @@ Status CacheServerHW::SetAffinity(const Task &tk, numa_id_t numa_node) {
RETURN_STATUS_UNEXPECTED("Numa node " + std::to_string(numa_node) + " not found");
}
return Status::OK();
#endif
}
std::vector<cpu_id_t> CacheServerHW::GetCpuList(numa_id_t numa_id) {
@ -182,6 +186,9 @@ std::vector<cpu_id_t> CacheServerHW::GetCpuList(numa_id_t numa_id) {
}
numa_id_t CacheServerHW::GetMyNode() const {
#ifdef __APPLE__
numa_id_t node_id = -1;
#else
numa_id_t node_id = 0;
auto cpu = sched_getcpu();
#ifdef NUMA_ENABLED
@ -197,7 +204,8 @@ numa_id_t CacheServerHW::GetMyNode() const {
}
}
MS_LOG(DEBUG) << "cpu id " << cpu << " found : " << std::boolalpha << found;
#endif
#endif // end NUMA_ENABLED
#endif // end __APPLE__
return node_id;
}

@ -18,7 +18,7 @@
#ifdef NUMA_ENABLED
#include <numa.h>
#endif
#endif // NUMA_ENABLED
#include <sched.h>
#include <stdlib.h>
#include <map>
@ -32,6 +32,23 @@
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/util/task.h"
#ifdef __APPLE__
#define SYSCTL_CORE_COUNT "machdep.cpu.core_count"
#include <sys/sysctl.h>
#include <mach/thread_policy.h>
typedef struct cpu_set {
uint32_t count;
} cpu_set_t;
static inline void CPU_ZERO(cpu_set_t *cs) { cs->count = 0; }
static inline void CPU_SET(int num, cpu_set_t *cs) { cs->count |= (1 << num); }
static inline int CPU_ISSET(int num, cpu_set_t *cs) { return (cs->count & (1 << num)); }
#endif // __APPLE__
namespace mindspore {
namespace dataset {
class CacheServerHW {

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save