Synchronize latest Ascend software suite 18 Dec 2020

pull/664/head
yanghaoran 5 years ago
parent 20a0326976
commit e1050ab406

@ -16,8 +16,11 @@ endif()
if(DEFINED ENV{D_PKG_SERVER})
set(GE_PB_PKG $ENV{D_PKG_SERVER})
message("Download packages from PKG server")
endif()
message("Download packages from DPKG server")
elseif(DEFINED ENV{MSLIBS_SERVER})
set(GE_PB_PKG "http://$ENV{MSLIBS_SERVER}:8081")
message("Download packages from MSPKG server")
endif ()
set(ASCEND_DRIVER_DIR ${ASCEND_DIR}/driver/lib64)
set(ASCEND_DRIVER_COMMON_DIR ${ASCEND_DIR}/driver/lib64/common)
@ -37,7 +40,7 @@ set(ATLAS_MS_RUNTIME_PATH ${ATLAS_RUNTIME_DIR} ${ATLAS_ACL_DIR} ${ATLAS_ATC_DIR}
option(ENABLE_OPEN_SRC "Enable graphengine compile in opensource." FALSE)
if (ENABLE_OPEN_SRC)
set(HI_PYTHON python3.7)
set(HI_PYTHON python3)
include(cmake/external_libs/protobuf_shared.cmake)
include(cmake/external_libs/protobuf_static.cmake)
@ -71,7 +74,7 @@ if (ENABLE_OPEN_SRC)
set(STATIC_ACL_LIB ${GE_LIB_PATH})
find_module(slog libslog.so ${GE_LIB_PATH})
find_module(static_mmpa libmmpa.a ${GE_LIB_PATH})
find_module(msprof libmsprof.so ${GE_LIB_PATH})
find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH})
find_module(hccl libhccl.so ${GE_LIB_PATH})
find_module(adump_server libadump_server.a ${GE_LIB_PATH})
find_module(runtime libruntime.so ${GE_LIB_PATH})
@ -80,20 +83,19 @@ if (ENABLE_OPEN_SRC)
find_module(error_manager liberror_manager.so ${GE_LIB_PATH})
find_module(ascend_hal_stub libascend_hal.so ${GE_LIB_PATH})
find_module(error_manager_static liberror_manager.a ${GE_LIB_PATH})
find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH})
find_module(msprofiler_fwk libmsprofiler_fwk.a ${GE_LIB_PATH})
#find_module(ascendcl_static libascendcl.a ${GE_LIB_PATH})
else()
find_module(slog libslog.so ${ASCEND_ATC_DIR} ${ASCEND_DRIVER_COMMON_DIR})
find_module(static_mmpa libmmpa.a ${ASCEND_ATC_DIR} ${ASCEND_RUNTIME_DIR})
find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR} ${ASCEND_RUNTIME_DIR})
if(PLATFORM STREQUAL "train")
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR})
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR})
find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR})
find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR})
find_module(resource libresource.so ${ASCEND_RUNTIME_DIR})
find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR})
find_module(msprofiler libmsprofiler.a ${ASCEND_RUNTIME_DIR})
find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_RUNTIME_DIR})
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver)
if(PRODUCT STREQUAL "flr3")
message(FATAL_ERROR "This platform is not supported in train mode, build terminated")
@ -106,20 +108,17 @@ if (ENABLE_OPEN_SRC)
find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR})
find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR})
find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR})
#find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR})
#find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR})
if(PRODUCT STREQUAL "flr3")
find_module(msprof libmsprof.so ${ASCEND_DRIVER_SHARE_DIR})
elseif(PRODUCT STREQUAL "flr1")
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver)
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR})
elseif(PRODUCT STREQUAL "flr2")
# flr2 ascend_hal_stub limsprof ?
else()
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR})
find_module(msprof libmsprof.so ${ASCEND_DRIVER_DIR})
endif()
elseif(PLATFORM STREQUAL "all")
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR})
find_module(msprofiler libmsprofiler.a ${ASCEND_DRIVER_COMMON_DIR})
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR})
find_module(adump_server libadump_server.a ${ASCEND_ACL_DIR})
find_module(runtime libruntime.so ${ASCEND_ACL_DIR})
@ -127,14 +126,14 @@ if (ENABLE_OPEN_SRC)
find_module(resource libresource.so ${ASCEND_ATC_DIR})
find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR})
find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR})
find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR})
find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_ACL_DIR})
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver)
#find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR})
else()
message(STATUS "PLATFORM param is invalid, should be train or inference, you choose nothing!")
message(STATUS "PLATFORM param is invalid, should be train or inference, you choose nothing!")
endif()
if (ENABLE_GE_COV OR ENABLE_GE_UT)
if (ENABLE_GE_COV OR ENABLE_GE_UT)
add_subdirectory(tests)
endif()

@ -23,7 +23,7 @@ export BUILD_PATH="${BASEPATH}/build/"
usage()
{
echo "Usage:"
echo "sh build.sh [-j[n]] [-h] [-v] [-s] [-t] [-u] [-c] [-S on|off]"
echo "sh build.sh [-j[n]] [-h] [-v] [-s] [-t] [-u] [-c] [-S on|off] [-M]"
echo ""
echo "Options:"
echo " -h Print usage"
@ -35,6 +35,7 @@ usage()
echo " -p Build inference or train"
echo " -v Display build command"
echo " -S Enable enable download cmake compile dependency from gitee , default off"
echo " -M build MindSpore mode"
echo "to be continued ..."
}
@ -62,8 +63,9 @@ checkopts()
PLATFORM=""
PRODUCT="normal"
ENABLE_GITEE="off"
MINDSPORE_MODE="off"
# Process the options
while getopts 'ustchj:p:g:vS:' opt
while getopts 'ustchj:p:g:vS:M' opt
do
OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
case "${opt}" in
@ -104,6 +106,9 @@ checkopts()
ENABLE_GITEE="$OPTARG"
echo "enable download from gitee"
;;
M)
MINDSPORE_MODE="on"
;;
*)
echo "Undefined option: ${opt}"
usage
@ -113,8 +118,8 @@ checkopts()
}
checkopts "$@"
git submodule update --init metadef
git submodule update --init parser
#git submodule update --init metadef
#git submodule update --init parser
mk_dir() {
local create_dir="$1" # the target to make
@ -150,7 +155,13 @@ build_graphengine()
if [[ "X$ENABLE_GITEE" = "Xon" ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GITEE=ON"
fi
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_OPEN_SRC=True -DCMAKE_INSTALL_PREFIX=${OUTPUT_PATH} -DPLATFORM=${PLATFORM} -DPRODUCT=${PRODUCT}"
if [[ "X$MINDSPORE_MODE" = "Xoff" ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_OPEN_SRC=True -DCMAKE_INSTALL_PREFIX=${OUTPUT_PATH} -DPLATFORM=${PLATFORM} -DPRODUCT=${PRODUCT}"
else
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_D=ON -DCMAKE_INSTALL_PREFIX=${OUTPUT_PATH}"
fi
echo "${CMAKE_ARGS}"
cmake ${CMAKE_ARGS} ..
if [ $? -ne 0 ]
@ -169,6 +180,9 @@ build_graphengine()
elif [ "X$ENABLE_GE_UT" = "Xon" ]
then
TARGET="ut_libgraph ut_libge_multiparts_utest ut_libge_others_utest ut_libge_kernel_utest ut_libge_distinct_load_utest"
elif [ "X$MINDSPORE_MODE" = "Xon" ]
then
TARGET="ge_common graph"
elif [ "x${PLATFORM}" = "xall" ]
then
# build all the target
@ -314,7 +328,13 @@ generate_package()
fi
}
if [[ "X$ENABLE_GE_UT" = "Xoff" ]]; then
if [[ "X$ENABLE_GE_UT" = "Xoff" && "X$MINDSPORE_MODE" = "Xoff" ]]; then
generate_package
echo "---------------- GraphEngine package archive generated ----------------"
elif [ "X$MINDSPORE_MODE" = "Xon" ]
then
cd "${OUTPUT_PATH}"
find ./ -name graphengine_lib.tar -exec rm {} \;
tar -cf graphengine_lib.tar lib
fi
echo "---------------- GraphEngine package archive generated ----------------"

@ -23,6 +23,7 @@ ExternalProject_Add(gflags_build
URL ${REQ_URL}
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
#SOURCE_DIR ${GE_CODE_DIR}/../../third_party/gflags/src/gflags-2.2.2
TLS_VERIFY OFF
CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_CXX_FLAGS=${gflags_CXXFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/gflags <SOURCE_DIR>
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install

@ -10,7 +10,10 @@ if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()
if (ENABLE_GITEE)
if (GE_PB_PKG)
set(REQ_URL "${GE_PB_PKG}/libs/gtest/release-1.8.0.tar.gz")
set(MD5 "")
elseif (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/googletest/repository/archive/release-1.8.0.tar.gz")
set(MD5 "")
else()
@ -22,8 +25,9 @@ set (gtest_CXXFLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -D_FORTIFY_SOURCE=2 -O2 -fstack-
set (gtest_CFLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -D_FORTIFY_SOURCE=2 -O2 -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
ExternalProject_Add(gtest_build
URL ${REQ_URL}
TLS_VERIFY OFF
CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_CXX_FLAGS=${gtest_CXXFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/gtest <SOURCE_DIR>
-DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_MACOSX_RPATH=TRUE -Dgtest_disable_pthreads=ON
-DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_MACOSX_RPATH=TRUE -Dgtest_disable_pthreads=ON
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE

@ -5,10 +5,14 @@ endif()
include(ExternalProject)
set(JSON_SRC_DIR ${CMAKE_BINARY_DIR}/opensrc/json/include)
if (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/JSON-for-Modern-CPP/repository/archive/v3.6.1.zip")
set(MD5 "5bda78ce308e6cfcf614dcf1d5ff27a7")
set(JSON_INCLUDE_DIR "${JSON_SRC_DIR}/include")
if (GE_PB_PKG)
set(REQ_URL "${GE_PB_PKG}/libs/ge_nlohmann_json/include.zip")
set(MD5 "0dc903888211db3a0f170304cd9f3a89")
set(JSON_INCLUDE_DIR ${JSON_SRC_DIR})
#elseif (ENABLE_GITEE)
# set(REQ_URL "https://gitee.com/mirrors/JSON-for-Modern-CPP/repository/archive/v3.6.1.zip")
# set(MD5 "5bda78ce308e6cfcf614dcf1d5ff27a7")
#set(JSON_INCLUDE_DIR "${JSON_SRC_DIR}/include")
else()
set(REQ_URL "https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip")
set(MD5 "0dc903888211db3a0f170304cd9f3a89")
@ -18,6 +22,7 @@ ExternalProject_Add(json_build
URL ${REQ_URL}
#URL /home/txd/workspace/cloud_code/pkg/include.zip
SOURCE_DIR ${JSON_SRC_DIR}
TLS_VERIFY OFF
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""

@ -6,7 +6,10 @@ set(ONNX_PROTO_DIR ${CMAKE_BINARY_DIR}/onnx)
set(ONNX_PROTO_FILE ${ONNX_PROTO_DIR}/onnx.proto)
file(MAKE_DIRECTORY ${ONNX_PROTO_DIR})
if (ENABLE_GITEE)
if (GE_PB_PKG)
set(REQ_URL "${GE_PB_PKG}/libs/onnx/onnx-1.6.0.tar.gz")
set(MD5 "512f2779d6215d4a36f366b6b9acdf1e")
elseif (ENABLE_GITEE)
set(REQ_URL "https://gitee.com/mirrors/ONNX/repository/archive/v1.6.0.tar.gz")
set(MD5 "1bdbcecdd68ea8392630467646776e02")
else()
@ -19,6 +22,7 @@ ExternalProject_Add(onnx
#URL /home/txd/workspace/cloud_code/pkg/onnx-1.6.0.tar.gz
#URL_HASH SHA256=3b88c3fe521151651a0403c4d131cb2e0311bd28b753ef692020a432a81ce345
#SOURCE_DIR ${ONNX_SRC_DIR}
TLS_VERIFY OFF
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
#INSTALL_COMMAND ""

@ -26,6 +26,7 @@ set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fst
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
ExternalProject_Add(protobuf_build
URL ${REQ_URL}
TLS_VERIFY OFF
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-Dprotobuf_WITH_ZLIB=OFF
-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}

@ -27,6 +27,7 @@ ExternalProject_Add(protobuf_static_build
URL ${REQ_URL}
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
#SOURCE_DIR ${METADEF_DIR}/../../third_party/protobuf/src/protobuf-3.8.0
TLS_VERIFY OFF
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

@ -30,6 +30,7 @@ ExternalProject_Add(protoc_build
URL ${REQ_URL}
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
#SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0
TLS_VERIFY OFF
CONFIGURE_COMMAND ${CMAKE_COMMAND} -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protoc <SOURCE_DIR>/cmake
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install

@ -10,11 +10,20 @@ if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()
if (GE_PB_PKG)
set(REQ_URL "${GE_PB_PKG}/libs/securec/v1.1.10.tar.gz")
set(MD5 "")
else()
set(REQ_URL "https://gitee.com/openeuler/libboundscheck/repository/archive/v1.1.10.tar.gz")
set(MD5 "")
endif ()
ExternalProject_Add(c_sec_build
URL https://gitee.com/openeuler/libboundscheck/repository/archive/v1.1.10.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
URL ${REQ_URL}
#URL https://gitee.com/openeuler/libboundscheck/repository/archive/v1.1.10.tar.gz
#SOURCE_DIR ${GE_CODE_DIR}/../libc_sec
PATCH_COMMAND patch -p1 < ${GE_CODE_DIR}/metadef/third_party/patch/securec/0001-add-securec-cmake-script.patch
TLS_VERIFY OFF
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

@ -60,6 +60,8 @@ set(TRAIN_SRC_LIST
"common/dump/dump_manager.cc"
"common/dump/dump_properties.cc"
"common/dump/dump_op.cc"
"common/profiling/ge_profiling.cc"
"common/profiling/ge_runner_profiling.cc"
"engine_manager/dnnengine_manager.cc"
"ge_local_engine/engine/host_cpu_engine.cc"
"generator/ge_generator.cc"
@ -142,6 +144,7 @@ set(TRAIN_SRC_LIST
"graph/passes/atomic_addr_clean_pass.cc"
"graph/passes/mark_same_addr_pass.cc"
"graph/passes/mark_graph_unknown_status_pass.cc"
"graph/passes/dynamic_single_op_reset_shape_pass.cc"
"graph/passes/mark_agnostic_pass.cc"
"graph/partition/dynamic_shape_partition.cc"
"graph/partition/stage_partition.cc"
@ -201,6 +204,7 @@ set(TRAIN_SRC_LIST
"host_kernels/sub_kernel.cc"
"host_kernels/transdata_kernel.cc"
"host_kernels/unpack_kernel.cc"
"host_kernels/reformat_kernel.cc"
"graph/passes/folding_pass.cc"
"graph/passes/get_original_format_pass.cc"
"graph/passes/guarantee_const_pass.cc"
@ -331,7 +335,6 @@ set(TRAIN_SRC_LIST
"hybrid/hybrid_davinci_model.cc"
"executor/ge_executor.cc"
"client/ge_api.cc"
"client/ge_prof.cc"
"analyzer/analyzer.cc"
"ir_build/ge_ir_build.cc"
"ir_build/atc_ir_common.cc"
@ -432,6 +435,7 @@ set(INFER_SRC_LIST
"graph/passes/atomic_addr_clean_pass.cc"
"graph/passes/mark_same_addr_pass.cc"
"graph/passes/mark_graph_unknown_status_pass.cc"
"graph/passes/dynamic_single_op_reset_shape_pass.cc"
"graph/passes/mark_agnostic_pass.cc"
"graph/common/omg_util.cc"
"graph/common/bcast.cc"
@ -487,6 +491,7 @@ set(INFER_SRC_LIST
"host_kernels/slice_d_kernel.cc"
"host_kernels/dynamic_stitch_kernel.cc"
"host_kernels/identity_kernel.cc"
"host_kernels/reformat_kernel.cc"
"graph/passes/stop_gradient_pass.cc"
"graph/passes/prevent_gradient_pass.cc"
"graph/passes/identity_pass.cc"
@ -602,7 +607,7 @@ set(INFER_SRC_LIST
if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES)
############ libge_runner.so ############
add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS})
add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS} $<TARGET_OBJECTS:msprofiler_fwk>)
target_compile_definitions(ge_runner PRIVATE
PROTOBUF_INLINE_NOT_IN_HEADERS=0
@ -647,7 +652,6 @@ target_link_libraries(ge_runner
$<BUILD_INTERFACE:intf_pub>
ge_memory
adump_server
msprofiler
static_mmpa
-Wl,--no-as-needed
graph
@ -656,7 +660,6 @@ target_link_libraries(ge_runner
register
c_sec
slog
msprof
runtime
resource
error_manager
@ -781,7 +784,6 @@ target_link_libraries(opensrc_ascendcl PRIVATE
c_sec
runtime
slog
msprof
ascend_hal_stub
-Wl,--as-needed
-lrt
@ -797,12 +799,10 @@ set_target_properties(opensrc_ascendcl PROPERTIES
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc
COMMAND echo "Generating stub files."
&& ${HI_PYTHON} ${CMAKE_CURRENT_LIST_DIR}/stub/gen_stubapi.py ${GE_CODE_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR}
&& mv ge_ir_build.cc stub_ge_ir_build.cc
&& mv ge_api.cc stub_ge_api.cc
&& mv ge_prof.cc stub_ge_prof.cc
&& echo "Generating stub files end."
#WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
#DEPENDS stub/gen_stubapi.py ${TOP_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR}
@ -811,7 +811,6 @@ add_custom_command(
add_custom_target(ge_stub
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc
)
##################################################################
@ -853,7 +852,6 @@ target_include_directories(atc_stub_ge_compiler PRIVATE
############ stub/libge_runner.so ############
add_library(fwk_stub_ge_runner SHARED
stub_ge_api.cc
stub_ge_prof.cc
stub_ge_ir_build.cc
)

@ -134,7 +134,7 @@ Status GEInitialize(const std::map<string, string> &options) {
Status GEInitialize(const std::map<AscendString, AscendString> &options) {
std::map<std::string, std::string> str_options;
for (auto & option : options) {
for (auto &option : options) {
if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) {
GELOGE(FAILED, "GEInitialize options is nullptr.");
return FAILED;

File diff suppressed because it is too large Load Diff

@ -4,7 +4,6 @@ LOCAL_PATH := $(call my-dir)
COMMON_LOCAL_SRC_FILES := \
proto/ge_api.proto \
ge_api.cc \
ge_prof.cc \
COMMON_LOCAL_C_INCLUDES := \
@ -69,9 +68,9 @@ LOCAL_SHARED_LIBRARIES := \
libgraph \
libregister \
libge_compiler \
libge_common \
libmsprof
libge_common
LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \
LOCAL_LDFLAGS := -lrt -ldl
@ -104,8 +103,10 @@ LOCAL_SHARED_LIBRARIES := \
libregister \
libruntime \
libge_compiler \
libge_common \
libmsprof
libge_common
LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \
LOCAL_LDFLAGS := -lrt -ldl

@ -24,6 +24,7 @@ set(SRC_LIST
"helper/om_file_helper.cc"
"helper/model_helper.cc"
"../model/ge_model.cc"
"../model/ge_root_model.cc"
"auth/file_saver.cc"
"fp16_t.cc"
"math/fp16_math.cc"
@ -129,6 +130,7 @@ target_compile_definitions(ge_common_static PRIVATE
google=ascend_private
$<IF:$<STREQUAL:${TARGET_SYSTEM_NAME},Windows>,OS_TYPE=WIN,OS_TYPE=0>
$<$<STREQUAL:${TARGET_SYSTEM_NAME},Windows>:SECUREC_USING_STD_SECURE_LIB=0 NOMINMAX>
LOG_CPP
)
target_compile_options(ge_common_static PRIVATE
@ -183,6 +185,7 @@ target_compile_options(ge_common PRIVATE
-fvisibility=hidden
-O2
-Werror
-Wno-deprecated-declarations
)
target_include_directories(ge_common PRIVATE

@ -54,8 +54,8 @@ Status FileSaver::OpenFile(int32_t &fd, const std::string &file_path) {
Status FileSaver::WriteData(const void *data, uint32_t size, int32_t fd) {
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(size == 0 || data == nullptr, return PARAM_INVALID);
mmSsize_t write_count;
uint32_t size_2g = ((uint32_t) 0x1 << 31);
uint32_t size_1g = ((uint32_t) 0x1 << 30);
uint32_t size_2g = 2147483648; // 0x1 << 31
uint32_t size_1g = 1073741824; // 0x1 << 30
// Write data
if (size > size_2g) {
auto seek = reinterpret_cast<uint8_t *>(const_cast<void *>(data));
@ -258,6 +258,65 @@ FileSaver::SaveToFile(const string &file_path, ModelFileHeader &file_header, Mod
return SUCCESS;
}
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status
FileSaver::SaveToFile(const string &file_path, ModelFileHeader &file_header,
vector<ModelPartitionTable *> &model_partition_tables,
const vector<vector<ModelPartition>> &all_partition_datas) {
file_header.is_encrypt = ModelEncryptType::UNENCRYPTED;
const Status ret = SaveWithFileHeader(file_path, file_header, model_partition_tables, all_partition_datas);
GE_CHK_BOOL_RET_STATUS(ret == SUCCESS, FAILED, "save file failed, file_path:%s, file header len:%u.",
file_path.c_str(), file_header.length);
return SUCCESS;
}
Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFileHeader &file_header,
vector<ModelPartitionTable *> &model_partition_tables,
const vector<vector<ModelPartition>> &all_partition_datas) {
GE_CHK_BOOL_EXEC(model_partition_tables.size() == all_partition_datas.size(),
return PARAM_INVALID,
"model table size %zu does not match partition size %zu",
model_partition_tables.size(), all_partition_datas.size())
for (size_t index = 0; index < model_partition_tables.size(); ++index) {
auto &cur_partiton_data = all_partition_datas[index];
auto &cur_model_partition_table = *model_partition_tables[index];
GE_CHK_BOOL_RET_STATUS(!cur_partiton_data.empty() && cur_model_partition_table.num != 0
&& cur_model_partition_table.num == cur_partiton_data.size(), FAILED,
"Invalid param:partition data size is (%u), model_partition_table.num is (%zu).",
cur_model_partition_table.num, cur_partiton_data.size());
}
// Open file
int32_t fd = 0;
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(OpenFile(fd, file_path) != SUCCESS, return FAILED);
Status ret = SUCCESS;
do {
// Write file header
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
WriteData(static_cast<const void *>(&file_header), sizeof(ModelFileHeader), fd) != SUCCESS, ret = FAILED;
break);
for (size_t index = 0; index < model_partition_tables.size(); ++index) {
// Write model partition table
auto &cur_tabel = *model_partition_tables[index];
uint32_t table_size = static_cast<uint32_t>(SIZE_OF_MODEL_PARTITION_TABLE(cur_tabel));
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
WriteData(static_cast<const void *>(&cur_tabel), table_size, fd) != SUCCESS, ret = FAILED; break);
// Write partition data
auto &cur_partition_datas = all_partition_datas[index];
for (const auto &partition_data : cur_partition_datas) {
GELOGI("GC:size[%zu]", partition_data.size);
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
WriteData(static_cast<const void *>(partition_data.data), partition_data.size, fd) != SUCCESS, ret = FAILED;
break);
}
}
} while (0);
// Close file
GE_CHK_BOOL_RET_STATUS(mmClose(fd) == EN_OK, FAILED, "Close file failed.");
return ret;
}
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status FileSaver::SaveToFile(const string &file_path, const void *data,
int len) {
if (data == nullptr || len <= 0) {

@ -74,6 +74,10 @@ class FileSaver {
ModelPartitionTable &model_partition_table,
const std::vector<ModelPartition> &partition_datas);
static Status SaveToFile(const string &file_path, ModelFileHeader &file_header,
vector<ModelPartitionTable *> &model_partition_tables,
const vector<vector<ModelPartition>> &all_partition_datas);
static Status SaveToBuffWithFileHeader(const ModelFileHeader &file_header,
ModelPartitionTable &model_partition_table,
const std::vector<ModelPartition> &partitionDatas,
@ -108,6 +112,9 @@ class FileSaver {
static Status SaveWithFileHeader(const std::string &file_path, const ModelFileHeader &file_header,
ModelPartitionTable &model_partition_table,
const std::vector<ModelPartition> &partition_datas);
static Status SaveWithFileHeader(const std::string &file_path, const ModelFileHeader &file_header,
vector<ModelPartitionTable *> &model_partition_tables,
const vector<vector<ModelPartition>> &all_partition_datas);
};
} // namespace ge
#endif // GE_COMMON_AUTH_FILE_SAVER_H_

@ -25,32 +25,38 @@
namespace ge {
namespace {
const char* kBase64Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
const char *kBase64Chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
const char kEqualSymbol = '=';
const size_t kBase64CharsNum = 64;
const size_t kThreeByteOneGroup = 3;
const size_t kFourByteOneGroup = 4;
}
const size_t kThreeByteOneGroupIndex0 = 0;
const size_t kThreeByteOneGroupIndex1 = 1;
const size_t kThreeByteOneGroupIndex2 = 2;
const size_t kFourByteOneGroupIndex0 = 0;
const size_t kFourByteOneGroupIndex1 = 1;
const size_t kFourByteOneGroupIndex2 = 2;
const size_t kFourByteOneGroupIndex3 = 3;
} // namespace
namespace base64 {
static inline bool IsBase64Char(const char &c) {
return (isalnum(c) || (c == '+') || (c == '/'));
}
static inline bool IsBase64Char(const char &c) { return (isalnum(c) || (c == '+') || (c == '/')); }
static std::string EncodeToBase64(const std::string &raw_data) {
size_t encode_length = raw_data.size() / kThreeByteOneGroup * kFourByteOneGroup;
encode_length += raw_data.size() % kThreeByteOneGroup == 0 ? 0 : kFourByteOneGroup;
size_t raw_data_index = 0 ;
size_t raw_data_index = 0;
size_t encode_data_index = 0;
std::string encode_data;
encode_data.resize(encode_length);
for (; raw_data_index + kThreeByteOneGroup <= raw_data.size(); raw_data_index += kThreeByteOneGroup) {
auto char_1 = static_cast<uint8_t>(raw_data[raw_data_index]);
auto char_2 = static_cast<uint8_t>(raw_data[raw_data_index + 1]);
auto char_3 = static_cast<uint8_t>(raw_data[raw_data_index + 2]);
auto char_2 = static_cast<uint8_t>(raw_data[raw_data_index + kThreeByteOneGroupIndex1]);
auto char_3 = static_cast<uint8_t>(raw_data[raw_data_index + kThreeByteOneGroupIndex2]);
encode_data[encode_data_index++] = kBase64Chars[char_1 >> 2u];
encode_data[encode_data_index++] = kBase64Chars[((char_1 << 4u) & 0x30) | (char_2 >> 4u)];
encode_data[encode_data_index++] = kBase64Chars[((char_2 << 2u) & 0x3c) | (char_3 >> 6u)];
@ -80,8 +86,7 @@ static std::string EncodeToBase64(const std::string &raw_data) {
#pragma GCC diagnostic ignored "-Wunused-function"
static Status DecodeFromBase64(const std::string &base64_data, std::string &decode_data) {
if (base64_data.size() % kFourByteOneGroup != 0) {
GELOGE(PARAM_INVALID, "base64 data size must can be divided by 4, but given data size is %zu",
base64_data.size());
GELOGE(PARAM_INVALID, "base64 data size must can be divided by 4, but given data size is %zu", base64_data.size());
return PARAM_INVALID;
}
decode_data.clear();
@ -92,10 +97,10 @@ static Status DecodeFromBase64(const std::string &base64_data, std::string &deco
return static_cast<uint8_t>(std::distance(kBase64Chars, char_pos)) & 0xff;
};
for (std::size_t input_data_index = 0; input_data_index < base64_data_len; input_data_index += 4) {
for (std::size_t input_data_index = 0; input_data_index < base64_data_len; input_data_index += kFourByteOneGroup) {
for (size_t i = 0; i < kFourByteOneGroup; ++i) {
if (base64_data[input_data_index + i] == kEqualSymbol &&
input_data_index >= base64_data_len - 4 && i > 1) {
input_data_index >= base64_data_len - kFourByteOneGroup && i > 1) {
byte_4[i] = kBase64CharsNum;
} else if (IsBase64Char(base64_data[input_data_index + i])) {
byte_4[i] = FindCharInBase64Chars(base64_data[input_data_index + i]);
@ -104,19 +109,23 @@ static Status DecodeFromBase64(const std::string &base64_data, std::string &deco
return PARAM_INVALID;
}
}
decode_data += static_cast<char>((byte_4[0] << 2u) + ((byte_4[1] & 0x30) >> 4u));
if (byte_4[2] >= kBase64CharsNum){
decode_data +=
static_cast<char>((byte_4[kFourByteOneGroupIndex0] << 2u) + ((byte_4[kFourByteOneGroupIndex1] & 0x30) >> 4u));
if (byte_4[kFourByteOneGroupIndex2] >= kBase64CharsNum) {
break;
} else if (byte_4[3] >= kBase64CharsNum) {
decode_data += static_cast<char>(((byte_4[1] & 0x0f) << 4u) + ((byte_4[2] & 0x3c) >> 2u));
} else if (byte_4[kFourByteOneGroupIndex3] >= kBase64CharsNum) {
decode_data += static_cast<char>(((byte_4[kFourByteOneGroupIndex1] & 0x0f) << 4u) +
((byte_4[kFourByteOneGroupIndex2] & 0x3c) >> 2u));
break;
}
decode_data += static_cast<char>(((byte_4[1] & 0x0f) << 4u) + ((byte_4[2] & 0x3c) >> 2u));
decode_data += static_cast<char>(((byte_4[2] & 0x03) << 6u) + byte_4[3]);
decode_data += static_cast<char>(((byte_4[kFourByteOneGroupIndex1] & 0x0f) << 4u) +
((byte_4[kFourByteOneGroupIndex2] & 0x3c) >> 2u));
decode_data +=
static_cast<char>(((byte_4[kFourByteOneGroupIndex2] & 0x03) << 6u) + byte_4[kFourByteOneGroupIndex3]);
}
return SUCCESS;
}
#pragma GCC diagnostic pop
}
} // namespace base64
} // namespace ge
#endif // GE_COMMON_BASE64_H_

@ -139,7 +139,8 @@ int MemoryDumper::OpenFile(const char *filename) {
GE_IF_BOOL_EXEC(
-1 != path_split_pos, string prefix_path = std::string(filename).substr(0, path_split_pos);
string last_path = std::string(filename).substr(path_split_pos, strlen(filename) - 1);
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(prefix_path.length() >= MMPA_MAX_PATH, return kInvalidFd, "Prefix path is too long!");
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(prefix_path.length() >= MMPA_MAX_PATH,
return kInvalidFd, "Prefix path is too long!");
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(mmRealPath(prefix_path.c_str(), tmp_path, MMPA_MAX_PATH) != EN_OK, return kInvalidFd,
"Dir %s does not exit.", prefix_path.c_str());
real_path = std::string(tmp_path) + last_path;)

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save