update cmake

pull/76/head
taoxiangdong 4 years ago
parent b9244c3081
commit 9d6f983611

@ -1,133 +1,55 @@
# Copyright 2019-2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
cmake_minimum_required(VERSION 3.14) cmake_minimum_required(VERSION 3.14)
project (GraphEngine[CXX]) project (GraphEngine[CXX])
set(CMAKE_CXX_STANDARD 17)
add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) set(GE_CODE_DIR ${CMAKE_CURRENT_LIST_DIR})
set(GE_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR})
set(GE_PROTO_DIR ${GE_SOURCE_DIR}/)
if (NOT BUILD_PATH) if (NOT BUILD_PATH)
set(BUILD_PATH "${CMAKE_SOURCE_DIR}/build") set(BUILD_PATH "${CMAKE_SOURCE_DIR}/build")
endif() endif()
# architecture: aarch64 or x86_64
message(STATUS "System architecture: ${CMAKE_HOST_SYSTEM_PROCESSOR}")
# system: euleros or ubuntu
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
execute_process(
COMMAND bash "-c" "cat /etc/os-release | grep ^ID= | awk -F '=' '{print $2}'"
OUTPUT_VARIABLE SYSTEM_TYPE
)
MESSAGE(STATUS "System type: ${SYSTEM_TYPE}.")
endif()
# download json headers, rather than whole repository option(ENABLE_OPEN_SRC "Enable graphengine compile in opensource." FALSE)
include(${GE_SOURCE_DIR}/cmake/ge_utils.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/json.cmake) if (ENABLE_OPEN_SRC)
include(${GE_SOURCE_DIR}/cmake/external_libs/eigen.cmake) set(HI_PYTHON python3.7)
include(${GE_SOURCE_DIR}/cmake/external_libs/gtest.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake) include(cmake/external_libs/protobuf_shared.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/onnx.cmake) include(cmake/external_libs/protoc.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/securec.cmake) include(cmake/external_libs/gflags.cmake)
set(CMAKE_SKIP_RPATH TRUE) include(cmake/external_libs/securec.cmake)
include(cmake/external_libs/json.cmake)
include(cmake/FindModule.cmake)
include(cmake/intf_pub_linux.cmake)
# for CPU/GPU mode, find c_sec and slog from local prebuild
if(NOT ENABLE_D AND NOT GE_ONLY)
set(GE_PREBUILD_PATH ${GE_SOURCE_DIR}/third_party/prebuild/${CMAKE_HOST_SYSTEM_PROCESSOR})
find_library(slog libslog.so ${GE_PREBUILD_PATH})
# if D_LINK_PATH is set in environment variables, search libraries in given path
elseif(DEFINED ENV{D_LINK_PATH})
# D_LINK_PATH is set
set(GE_LIB_PATH $ENV{D_LINK_PATH})
set(GE_SYS_ARCH "")
if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64")
# x86 ubuntu
set(GE_SYS_ARCH "x86_64")
elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
# arm euleros
set(GE_SYS_ARCH "aarch64")
else()
message(FATAL_ERROR "Running on a unsupported architecture: ${SYSTEM_TYPE}, build terminated")
endif()
set(GE_LIB_PATH ${GE_LIB_PATH}/${GE_SYS_ARCH})
find_library(slog libslog.so ${GE_LIB_PATH})
find_library(mmpa libmmpa.so ${GE_LIB_PATH})
find_library(runtime libruntime.so ${GE_LIB_PATH})
find_library(msprof libmsprof.so ${GE_LIB_PATH})
find_library(register libregister.so ${GE_LIB_PATH})
find_library(hccl libhccl.so ${GE_LIB_PATH})
find_library(resource libresource.so ${GE_LIB_PATH})
find_library(error_manager liberror_manager.so ${GE_LIB_PATH})
else()
# Ascend mode
if(DEFINED ENV{ASCEND_CUSTOM_PATH}) if(DEFINED ENV{ASCEND_CUSTOM_PATH})
set(ASCEND_DIR $ENV{ASCEND_CUSTOM_PATH}) set(ASCEND_DIR $ENV{ASCEND_CUSTOM_PATH})
else() else()
set(ASCEND_DIR /usr/local/Ascend) set(ASCEND_DIR /usr/local/Ascend)
endif() endif()
set(ASCEND_DRIVER_DIR ${ASCEND_DIR}/driver/lib64/common) set(ASCEND_DRIVER_DIR ${ASCEND_DIR}/driver/lib64/common)
set(ASCEND_RUNTIME_DIR ${ASCEND_DIR}/fwkacllib/lib64) set(ASCEND_RUNTIME_DIR ${ASCEND_DIR}/fwkacllib/lib64)
find_library(slog libslog.so ${ASCEND_DRIVER_DIR}) find_module(slog libslog.so ${ASCEND_DRIVER_DIR})
find_library(mmpa libmmpa.so ${ASCEND_DRIVER_DIR}) find_module(mmpa libmmpa.so ${ASCEND_DRIVER_DIR})
find_library(msprof libmsprof.so ${ASCEND_DRIVER_DIR}) find_module(msprof libmsprof.so ${ASCEND_DRIVER_DIR})
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR})
find_library(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR})
find_library(runtime libruntime.so ${ASCEND_RUNTIME_DIR}) find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR})
find_library(register libregister.so ${ASCEND_RUNTIME_DIR}) find_module(runtime_compile libruntime_compile.so ${ASCEND_RUNTIME_DIR})
find_library(resource libresource.so ${ASCEND_RUNTIME_DIR}) find_module(resource libresource.so ${ASCEND_RUNTIME_DIR})
find_library(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR}) find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR})
endif() find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR})
# add compile flags set(METADEF_DIR ${CMAKE_CURRENT_LIST_DIR}/metadef)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") set(PARSER_DIR ${CMAKE_CURRENT_LIST_DIR}/parser)
message("Build in Debug mode") set(GE_DEPEND_DIR ${CMAKE_CURRENT_LIST_DIR}/..)
set(CMAKE_C_FLAGS "-O0 -g -Wall -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe -fPIC ${CMAKE_C_FLAGS}")
set(CMAKE_CXX_FLAGS "-O0 -g -Wall -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe -fPIC ${CMAKE_CXX_FLAGS}") add_subdirectory(metadef)
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") #add_subdirectory(metadef/graph)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -rdynamic") #add_subdirectory(metadef/register)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -rdynamic")
endif()
else() else()
set(CMAKE_C_FLAGS "-O2 -Wall -fPIC -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe ${CMAKE_C_FLAGS}") set(METADEF_DIR ${CMAKE_CURRENT_LIST_DIR}/../metadef)
set(CMAKE_CXX_FLAGS "-O2 -Wall -fPIC -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe ${CMAKE_CXX_FLAGS}") set(PARSER_DIR ${CMAKE_CURRENT_LIST_DIR}/../parser)
endif () set(GE_DEPEND_DIR ${CMAKE_CURRENT_LIST_DIR}/..)
# force __FILE__ to show relative path of file, from source directory, as cmake project makes __FILE__ absolute directory
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILE__='\"$(subst $(realpath ${CMAKE_SOURCE_DIR})/,,$(abspath $<))\"' -Wno-builtin-macro-redefined")
# compile libraries from following directories
# libgraph is compiled in any situation
add_subdirectory(${GE_SOURCE_DIR}/metadef/graph)
if(ENABLE_D)
# if MindSpore compiles in D mode, compile the following libraries
add_subdirectory(${GE_SOURCE_DIR}/ge/common)
add_subdirectory(${GE_SOURCE_DIR}/ge/ge_runtime)
elseif(GE_ONLY)
# standalone GraphEngine compiles all following libraries
add_subdirectory(${GE_SOURCE_DIR}/ge/common)
add_subdirectory(${GE_SOURCE_DIR}/ge/ge_runtime)
add_subdirectory(${GE_SOURCE_DIR}/ge/ge_local_engine)
add_subdirectory(${GE_SOURCE_DIR}/ge/graph/build/memory)
add_subdirectory(${GE_SOURCE_DIR}/ge/)
add_subdirectory(${GE_SOURCE_DIR}/ge/plugin/engine)
endif() endif()
# if (ENABLE_GE_COV OR ENABLE_GE_UT OR ENABLE_GE_ST) add_subdirectory(ge)
# add_subdirectory(tests)
# endif()

@ -0,0 +1,23 @@
#[[
module - the name of export imported target
name - find the library name
path - find the library path
#]]
function(find_module module name path)
if (TARGET ${module})
return()
endif()
find_library(${module}_LIBRARY_DIR NAMES ${name} NAMES_PER_DIR PATHS ${path}
PATH_SUFFIXES lib
)
message(STATUS "find ${name} location ${${module}_LIBRARY_DIR}")
if ("${${module}_LIBRARY_DIR}" STREQUAL "${module}_LIBRARY_DIR-NOTFOUND")
message(FATAL_ERROR "${name} not found in ${path}")
endif()
add_library(${module} SHARED IMPORTED)
set_target_properties(${module} PROPERTIES
IMPORTED_LOCATION ${${module}_LIBRARY_DIR}
)
endfunction()

@ -1,13 +0,0 @@
set(Eigen3_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(Eigen3_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(Eigen3_NS "ge_")
graphengine_add_pkg(Eigen3
VER 3.3.7
URL https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz
MD5 9e30f67e8531477de4117506fe44669b
CMAKE_OPTION -DBUILD_TESTING=OFF)
find_package(Eigen3 3.3.7 REQUIRED ${GE_FIND_NO_DEFAULT_PATH})
set_property(TARGET Eigen3::Eigen PROPERTY IMPORTED_GLOBAL TRUE)
add_library(graphengine::eigen ALIAS Eigen3::Eigen)
include_directories(${EIGEN3_INCLUDE_DIRS})

@ -0,0 +1,38 @@
if (HAVE_GFLAGS)
return()
endif()
include(ExternalProject)
#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output)
if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()
ExternalProject_Add(gflags_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../third_party/gflags/src/gflags-2.2.2
CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/gflags <SOURCE_DIR>
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
set(GFLAGS_PKG_DIR ${CMAKE_INSTALL_PREFIX}/gflags)
add_library(gflags_static STATIC IMPORTED)
set_target_properties(gflags_static PROPERTIES
IMPORTED_LOCATION ${GFLAGS_PKG_DIR}/lib/libgflags.a
)
add_library(gflags INTERFACE)
target_include_directories(gflags INTERFACE ${GFLAGS_PKG_DIR}/include)
target_link_libraries(gflags INTERFACE gflags_static)
add_dependencies(gflags gflags_build)
set(HAVE_GFLAGS TRUE CACHE BOOL "gflags build add")

@ -1,16 +0,0 @@
set(ge_gtest_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
set(ge_gtest_CFLAGS "-D_FORTIFY_SOURCE=2 -O2 -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
graphengine_add_pkg(ge_gtest
VER 1.8.0
LIBS gtest gtest_main
URL https://github.com/google/googletest/archive/release-1.8.0.tar.gz
MD5 16877098823401d1bf2ed7891d7dce36
CMAKE_OPTION -DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON
-DCMAKE_MACOSX_RPATH=TRUE -Dgtest_disable_pthreads=ON)
add_library(graphengine::gtest ALIAS ge_gtest::gtest)
add_library(graphengine::gtest_main ALIAS ge_gtest::gtest_main)
include_directories(${ge_gtest_INC})
file(COPY ${ge_gtest_INC}/../lib/libgtest.so DESTINATION ${CMAKE_SOURCE_DIR}/build/graphengine)
file(COPY ${ge_gtest_INC}/../lib/libgtest_main.so DESTINATION ${CMAKE_SOURCE_DIR}/build/graphengine)

@ -1,9 +1,23 @@
set(nlohmann_json_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") if (HAVE_JSON)
set(nlohmann_json_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") return()
graphengine_add_pkg(ge_nlohmann_json endif()
VER 3.6.1
HEAD_ONLY ./ include(ExternalProject)
URL https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip
MD5 0dc903888211db3a0f170304cd9f3a89) set(JSON_SRC_DIR ${GE_CODE_DIR}/../third_party/json/include)
include_directories(${ge_nlohmann_json_INC}) ExternalProject_Add(json_build
add_library(graphengine::json ALIAS ge_nlohmann_json) #URL https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip
#URL /home/txd/workspace/cloud_code/pkg/include.zip
SOURCE_DIR ${JSON_SRC_DIR}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
EXCLUDE_FROM_ALL TRUE
)
add_library(json INTERFACE)
target_include_directories(json INTERFACE ${JSON_SRC_DIR})
add_dependencies(json json_build)
set(HAVE_JSON TRUE CACHE BOOL "json build add")

@ -1,5 +1,29 @@
graphengine_add_pkg(onnx include(ExternalProject)
VER 1.6.0
HEAD_ONLY ./ #set(ONNX_SRC_DIR /home/txd/workspace/cloud_code/graphengine/build/graphengine/open_source/onnx)
URL https://github.com/onnx/onnx/releases/download/v1.6.0/onnx-1.6.0.tar.gz #set(ONNX_PROTO ${ONNX_SRC_DIR}/onnx/onnx.proto)
MD5 512f2779d6215d4a36f366b6b9acdf1e) set(ONNX_PROTO_DIR ${CMAKE_BINARY_DIR}/onnx)
set(ONNX_PROTO_FILE ${ONNX_PROTO_DIR}/onnx.proto)
file(MAKE_DIRECTORY ${ONNX_PROTO_DIR})
ExternalProject_Add(onnx
#URL https://github.com/onnx/onnx/releases/download/v1.6.0/onnx-1.6.0.tar.gz
URL /home/txd/workspace/cloud_code/pkg/onnx-1.6.0.tar.gz
#URL_HASH SHA256=3b88c3fe521151651a0403c4d131cb2e0311bd28b753ef692020a432a81ce345
#SOURCE_DIR ${ONNX_SRC_DIR}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
#INSTALL_COMMAND ""
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/onnx/onnx.proto ${ONNX_PROTO_FILE}
#BUILD_ALWAYS TRUE
EXCLUDE_FROM_ALL TRUE
)
macro(onnx_protobuf_generate comp c_var h_var)
add_custom_command(OUTPUT ${ONNX_PROTO_FILE}
DEPENDS onnx
)
ge_protobuf_generate(${comp} ${c_var} ${h_var} ${ONNX_PROTO_FILE})
endmacro()

@ -1,54 +0,0 @@
if (NOT TARGET protobuf::protobuf)
set(protobuf_USE_STATIC_LIBS ON)
set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
set(_ge_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
graphengine_add_pkg(protobuf
VER 3.8.0
LIBS protobuf
EXE protoc
URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
MD5 3d9e32700639618a4d2d342c99d4507a
CMAKE_PATH ../cmake/
CMAKE_OPTION -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF)
set(CMAKE_CXX_FLAGS ${_ge_tmp_CMAKE_CXX_FLAGS})
endif()
add_library(graphengine::protobuf ALIAS protobuf::protobuf)
set(PROTOBUF_LIBRARY protobuf::protobuf)
include_directories(${protobuf_INC})
include_directories(${protobuf_DIRPATH}/src)
function(ge_protobuf_generate comp c_var h_var)
if(NOT ARGN)
message(SEND_ERROR "Error: ge_protobuf_generate() called without any proto files")
return()
endif()
set(${c_var})
set(${h_var})
foreach(file ${ARGN})
get_filename_component(abs_file ${file} ABSOLUTE)
get_filename_component(file_name ${file} NAME_WE)
get_filename_component(file_dir ${abs_file} PATH)
list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc")
list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h")
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto/${comp}/proto"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto/${comp}/proto ${abs_file}
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
endforeach()
set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
set(${c_var} ${${c_var}} PARENT_SCOPE)
set(${h_var} ${${h_var}} PARENT_SCOPE)
endfunction()

@ -0,0 +1,59 @@
if (HAVE_PROTOBUF)
return()
endif()
include(ExternalProject)
include(GNUInstallDirs)
if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()
set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 $<$<STREQUAL:${PRODUCT_SIDE},host>:-D_GLIBCXX_USE_CXX11_ABI=0> -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
ExternalProject_Add(protobuf_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
#SOURCE_DIR ${GE_CODE_DIR}/third_party/protobuf/src/protobuf-3.8.0
DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E copy_directory ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0 <SOURCE_DIR>
#CONFIGURE_COMMAND ${CMAKE_COMMAND}
#-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}
#-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
#-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
#-DCMAKE_LINKER=${CMAKE_LINKER}
#-DCMAKE_AR=${CMAKE_AR}
#-DCMAKE_RANLIB=${CMAKE_RANLIB}
#-Dprotobuf_WITH_ZLIB=OFF
#-Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protobuf <SOURCE_DIR>/cmake
CONFIGURE_COMMAND cd <SOURCE_DIR>
&& ./autogen.sh && cd <BINARY_DIR> && <SOURCE_DIR>/configure --prefix=${CMAKE_INSTALL_PREFIX}/protobuf --with-zlib=no CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CXXFLAGS=${protobuf_CXXFLAGS} LDFLAGS=${protobuf_LDFLAGS}
&& bash -c "sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=\"\"|g' libtool && sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool"
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
include(GNUInstallDirs)
set(PROTOBUF_SHARED_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protobuf)
add_library(protobuf SHARED IMPORTED)
file(MAKE_DIRECTORY ${PROTOBUF_SHARED_PKG_DIR}/include)
set_target_properties(protobuf PROPERTIES
IMPORTED_LOCATION ${PROTOBUF_SHARED_PKG_DIR}/lib/libprotobuf.so
)
target_include_directories(protobuf INTERFACE ${PROTOBUF_SHARED_PKG_DIR}/include)
set(INSTALL_BASE_DIR "")
set(INSTALL_LIBRARY_DIR lib)
install(FILES ${PROTOBUF_SHARED_PKG_DIR}/lib/libprotobuf.so ${PROTOBUF_SHARED_PKG_DIR}/lib/libprotobuf.so.19.0.0 OPTIONAL
DESTINATION ${INSTALL_LIBRARY_DIR})
add_dependencies(protobuf protobuf_build)
set(HAVE_PROTOBUF TRUE CACHE BOOL "protobuf build add")

@ -0,0 +1,43 @@
include(ExternalProject)
include(GNUInstallDirs)
#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output)
if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()
set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 $<$<STREQUAL:${PRODUCT_SIDE},host>:-D_GLIBCXX_USE_CXX11_ABI=0> -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
set(PROTOBUF_STATIC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protobuf_static)
ExternalProject_Add(protobuf_static_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}
-DCMAKE_LINKER=${CMAKE_LINKER}
-DCMAKE_AR=${CMAKE_AR}
-DCMAKE_RANLIB=${CMAKE_RANLIB}
-Dprotobuf_WITH_ZLIB=OFF
-Dprotobuf_BUILD_TESTS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${PROTOBUF_STATIC_PKG_DIR} <SOURCE_DIR>/cmake
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
include(GNUInstallDirs)
add_library(protobuf_static_lib STATIC IMPORTED)
set_target_properties(protobuf_static_lib PROPERTIES
IMPORTED_LOCATION ${PROTOBUF_STATIC_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/libprotobuf.a
)
add_library(protobuf_static INTERFACE)
target_include_directories(protobuf_static INTERFACE ${PROTOBUF_STATIC_PKG_DIR}/include)
target_link_libraries(protobuf_static INTERFACE protobuf_static_lib)
add_dependencies(protobuf_static protobuf_static_build)

@ -0,0 +1,102 @@
if (HAVE_PROTOC)
return()
endif()
include(ExternalProject)
include(GNUInstallDirs)
#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output)
if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()
set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
ExternalProject_Add(protoc_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0
CONFIGURE_COMMAND ${CMAKE_COMMAND} -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protoc <SOURCE_DIR>/cmake
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
set(PROTOC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protoc)
set(protoc_EXECUTABLE ${PROTOC_PKG_DIR}/${CMAKE_INSTALL_BINDIR}/protoc)
function(protobuf_generate comp c_var h_var)
if(NOT ARGN)
message(SEND_ERROR "Error: protobuf_generate() called without any proto files")
return()
endif()
set(${c_var})
set(${h_var})
foreach(file ${ARGN})
get_filename_component(abs_file ${file} ABSOLUTE)
get_filename_component(file_name ${file} NAME_WE)
get_filename_component(file_dir ${abs_file} PATH)
get_filename_component(parent_subdir ${file_dir} NAME)
if("${parent_subdir}" STREQUAL "proto")
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto)
else()
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir})
endif()
list(APPEND ${c_var} "${proto_output_path}/${file_name}.pb.cc")
list(APPEND ${h_var} "${proto_output_path}/${file_name}.pb.h")
add_custom_command(
OUTPUT "${proto_output_path}/${file_name}.pb.cc" "${proto_output_path}/${file_name}.pb.h"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}"
COMMAND ${protoc_EXECUTABLE} -I${file_dir} --cpp_out=${proto_output_path} ${abs_file}
DEPENDS protoc_build ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
endforeach()
set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
set(${c_var} ${${c_var}} PARENT_SCOPE)
set(${h_var} ${${h_var}} PARENT_SCOPE)
endfunction()
function(protobuf_generate_py comp py_var)
if(NOT ARGN)
message(SEND_ERROR "Error: protobuf_generate_py() called without any proto files")
return()
endif()
set(${py_var})
foreach(file ${ARGN})
get_filename_component(abs_file ${file} ABSOLUTE)
get_filename_component(file_name ${file} NAME_WE)
get_filename_component(file_dir ${abs_file} PATH)
get_filename_component(parent_subdir ${file_dir} NAME)
if("${parent_subdir}" STREQUAL "proto")
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto)
else()
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir})
endif()
list(APPEND ${py_var} "${proto_output_path}/${file_name}_pb2.py")
add_custom_command(
OUTPUT "${proto_output_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}"
COMMAND ${protoc_EXECUTABLE} -I${file_dir} --python_out=${proto_output_path} ${abs_file}
DEPENDS protoc_build ${abs_file}
COMMENT "Running PYTHON protocol buffer compiler on ${file}" VERBATIM )
endforeach()
set_source_files_properties(${${py_var}} PROPERTIES GENERATED TRUE)
set(${py_var} ${${py_var}} PARENT_SCOPE)
endfunction()
set(HAVE_PROTOC TRUE CACHE BOOL "protoc build add")

@ -1,11 +1,60 @@
graphengine_add_pkg(securec if (HAVE_C_SEC)
VER 1.1.10 return()
URL https://gitee.com/openeuler/bounds_checking_function/repository/archive/v1.1.10.tar.gz endif()
MD5 0782dd2351fde6920d31a599b23d8c91
LIBS c_sec include(ExternalProject)
PATCHES ${GE_SOURCE_DIR}/third_party/patch/securec/securec.patch001
CMAKE_OPTION "-DCMAKE_BUILD_TYPE=Release" if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
) (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
include_directories(${securec_INC}) set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
file(COPY ${securec_INC}/../lib/libc_sec.so DESTINATION ${CMAKE_SOURCE_DIR}/build/graphengine) message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
add_library(graphengine::securec ALIAS securec::c_sec) endif()
ExternalProject_Add(c_sec_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../libc_sec
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_LINKER=${CMAKE_LINKER}
-DCMAKE_AR=${CMAKE_AR}
-DCMAKE_RANLIB=${CMAKE_RANLIB}
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/c_sec <SOURCE_DIR>
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
set(C_SEC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/c_sec)
add_library(c_sec SHARED IMPORTED)
file(MAKE_DIRECTORY ${C_SEC_PKG_DIR}/include)
set_target_properties(c_sec PROPERTIES
IMPORTED_LOCATION ${C_SEC_PKG_DIR}/lib/libc_sec.so
)
target_include_directories(c_sec INTERFACE ${C_SEC_PKG_DIR}/include)
add_dependencies(c_sec c_sec_build)
set(INSTALL_BASE_DIR "")
set(INSTALL_LIBRARY_DIR lib)
install(FILES ${C_SEC_PKG_DIR}/lib/libc_sec.so OPTIONAL
DESTINATION ${INSTALL_LIBRARY_DIR})
add_library(c_sec_static_lib STATIC IMPORTED)
set_target_properties(c_sec_static_lib PROPERTIES
IMPORTED_LOCATION ${C_SEC_PKG_DIR}/lib/libc_sec.a
)
add_library(c_sec_static INTERFACE)
target_include_directories(c_sec_static INTERFACE ${C_SEC_PKG_DIR}/include)
target_link_libraries(c_sec_static INTERFACE c_sec_static_lib)
add_dependencies(c_sec_static c_sec_build)
set(HAVE_C_SEC TRUE CACHE BOOL "c_sec build add")

File diff suppressed because it is too large Load Diff

@ -0,0 +1,52 @@
add_library(intf_pub INTERFACE)
target_compile_options(intf_pub INTERFACE
-Wall
-fPIC
-fstack-protector-strong
)
target_compile_definitions(intf_pub INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
WIN64=1
LINUX=0
)
target_link_options(intf_pub INTERFACE
-Wl,-z,relro
-Wl,-z,now
-Wl,-z,noexecstack
$<$<CONFIG:Release>:-Wl,--build-id=none>
)
target_link_directories(intf_pub INTERFACE
)
add_library(intf_ccec INTERFACE)
target_compile_options(intf_ccec INTERFACE
-mcpu=cortex-a73
--target=aarch64-linux-android29
--sysroot=${HCC_PATH}/../sysroot
-L${HCC_PATH}/../lib/gcc/aarch64-linux-android/4.9.x
-Wall
-fPIC
-fstack-protector-strong
)
target_compile_definitions(intf_ccec INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
)
target_link_options(intf_ccec INTERFACE
-mcpu=cortex-a73
--target=aarch64-linux-android29
--sysroot=${HCC_PATH}/../sysroot
-L${HCC_PATH}/../lib/gcc/aarch64-linux-android/4.9.x
-Wl,-cce-host-android
-Wl,-z,relro
-Wl,-z,now
-Wl,-z,noexecstack
$<$<CONFIG:Release>:-Wl,--build-id=none>
)

@ -0,0 +1,32 @@
if (HAVE_PUB)
return()
endif()
add_library(intf_pub INTERFACE)
target_compile_options(intf_pub INTERFACE
-Wall
-fPIC
$<IF:$<STREQUAL:${CMAKE_SYSTEM_NAME},centos>,-fstack-protector-all,-fstack-protector-strong>
$<$<COMPILE_LANGUAGE:CXX>:-std=c++11>
)
target_compile_definitions(intf_pub INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
WIN64=1
LINUX=0
)
target_link_options(intf_pub INTERFACE
-Wl,-z,relro
-Wl,-z,now
-Wl,-z,noexecstack
$<$<CONFIG:Release>:-Wl,--build-id=none>
)
target_link_directories(intf_pub INTERFACE
)
target_link_libraries(intf_pub INTERFACE
-lpthread
)
set(HAVE_PUB TRUE CACHE BOOL "pub add")

@ -0,0 +1,24 @@
add_library(intf_pub INTERFACE)
target_compile_options(intf_pub INTERFACE
-Wall
-fPIC
$<IF:$<STREQUAL:${OS_TYPE},centos>,-fstack-protector-all,-fstack-protector-strong>
$<$<COMPILE_LANGUAGE:CXX>:-std=c++11>
)
target_compile_definitions(intf_pub INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
OS_TYPE=WIN64
WIN64=1
LINUX=0
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
)
target_link_options(intf_pub INTERFACE
$<$<CONFIG:Release>:-Wl,--build-id=none>
)
target_link_directories(intf_pub INTERFACE
)
target_link_libraries(intf_pub INTERFACE
)

File diff suppressed because it is too large Load Diff

@ -1,5 +1,5 @@
/** /**
* Copyright 2019-2020 Huawei Technologies Co., Ltd * Copyright 2020 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -40,32 +40,34 @@ const std::string kFilePath = "./";
const std::string kAnalyzeFile = "ge_check_op.json"; const std::string kAnalyzeFile = "ge_check_op.json";
const std::string kUnknownShape = "unknownshape"; const std::string kUnknownShape = "unknownshape";
const std::string kUnsupport = "unsupport"; const std::string kUnsupport = "unsupport";
const std::string kSessionId = "session_id"; const std::string kSessionId = "session_id";
const std::string kGraphId = "graph_id"; const std::string kGraphId = "graph_id";
const std::string kOpInfo = "op_info"; const std::string kOpInfo = "op_info";
const std::string kErrorType = "error_type"; const std::string kErrorType = "error_type";
const std::string kOpName = "name"; const std::string kOpName = "name";
const std::string kOpType = "type"; const std::string kOpType = "type";
const std::string kReason = "reason"; const std::string kReason = "reason";
const std::string kInput = "input"; const std::string kInput = "input";
const std::string kOutput = "output"; const std::string kOutput = "output";
const std::string kShape = "shape"; const std::string kShape = "shape";
const std::string kDataType = "data_type"; const std::string kDataType = "data_type";
const std::string kLayout = "layout"; const std::string kLayout = "layout";
const std::string kResult = "result"; const std::string kResult = "result";
const std::string kOp = "op"; const std::string kOp = "op";
std::map<analyzer::AnalyzeType, std::string> errors_map{{PARSER, "paser_error"}, std::map<analyzer::AnalyzeType, std::string> errors_map {
{INFER_SHAPE, "infer_shape_error"}, {PARSER, "paser_error"},
{CHECKSUPPORT, "check_support_error"}, {INFER_SHAPE, "infer_shape_error"},
{GRAPH_OPTIMIZE, "graph_optimize_error"}, {CHECKSUPPORT, "check_support_error"},
{GRAPH_PARTION, "graph_partion_error"}, {GRAPH_OPTIMIZE, "graph_optimize_error"},
{GRAPH_BUILDER, "graph_builder_error"}}; {GRAPH_PARTION, "graph_partion_error"},
} // namespace {GRAPH_BUILDER, "graph_builder_error"}
};
Analyzer *Analyzer::GetInstance() { }
Analyzer* Analyzer::GetInstance() {
static Analyzer instance; static Analyzer instance;
return &instance; return &instance;
} }
@ -75,7 +77,7 @@ Status Analyzer::BuildJsonObject(uint64_t session_id, uint64_t graph_id) {
std::lock_guard<std::recursive_mutex> lg(mutex_); std::lock_guard<std::recursive_mutex> lg(mutex_);
auto iter = graph_infos_.find(session_id); auto iter = graph_infos_.find(session_id);
if (iter == graph_infos_.end()) { if (iter == graph_infos_.end()) {
auto p = new (std::nothrow) GraphInfo(); auto p = new(std::nothrow) GraphInfo();
GE_CHECK_NOTNULL(p); GE_CHECK_NOTNULL(p);
std::shared_ptr<GraphInfo> graph_info(p); std::shared_ptr<GraphInfo> graph_info(p);
std::map<uint64_t, std::shared_ptr<GraphInfo>> graph_map; std::map<uint64_t, std::shared_ptr<GraphInfo>> graph_map;
@ -86,7 +88,7 @@ Status Analyzer::BuildJsonObject(uint64_t session_id, uint64_t graph_id) {
} else { } else {
auto iter1 = (iter->second).find(graph_id); auto iter1 = (iter->second).find(graph_id);
if (iter1 == (iter->second).end()) { if (iter1 == (iter->second).end()) {
auto p = new (std::nothrow) GraphInfo(); auto p = new(std::nothrow) GraphInfo();
GE_CHECK_NOTNULL(p); GE_CHECK_NOTNULL(p);
std::shared_ptr<GraphInfo> graph_info(p); std::shared_ptr<GraphInfo> graph_info(p);
graph_info->session_id = session_id; graph_info->session_id = session_id;
@ -100,7 +102,14 @@ Status Analyzer::BuildJsonObject(uint64_t session_id, uint64_t graph_id) {
} }
ge::Status Analyzer::Initialize() { ge::Status Analyzer::Initialize() {
ClearHistoryFile(); // Initialize file
string real_path = RealPath(kFilePath.c_str());
if (real_path.empty()) {
GELOGE(FAILED, "File path is invalid.");
return FAILED;
}
json_file_name_ = real_path + "/" + kAnalyzeFile;
return SUCCESS; return SUCCESS;
} }
@ -174,15 +183,8 @@ ge::Status Analyzer::CreateAnalyzerFile() {
return SUCCESS; return SUCCESS;
} }
GELOGD("start to create analyzer file!"); GELOGD("start to create analyzer file!");
// Check whether the manifest exists, if not, create it.
string real_path = RealPath(kFilePath.c_str());
if (real_path.empty()) {
GELOGE(FAILED, "File path is invalid.");
return FAILED;
}
std::lock_guard<std::mutex> lg(file_mutex_); std::lock_guard<std::mutex> lg(file_mutex_);
json_file_name_ = real_path + "/" + kAnalyzeFile;
GELOGD("Created analyzer file:[%s]", json_file_name_.c_str());
int fd = open(json_file_name_.c_str(), O_WRONLY | O_CREAT | O_TRUNC, kFileAuthority); int fd = open(json_file_name_.c_str(), O_WRONLY | O_CREAT | O_TRUNC, kFileAuthority);
if (fd < 0) { if (fd < 0) {
GELOGE(INTERNAL_ERROR, "Fail to open the file: %s.", json_file_name_.c_str()); GELOGE(INTERNAL_ERROR, "Fail to open the file: %s.", json_file_name_.c_str());
@ -198,25 +200,27 @@ ge::Status Analyzer::CreateAnalyzerFile() {
return SUCCESS; return SUCCESS;
} }
ge::Status Analyzer::SaveAnalyzerDataToFile() { ge::Status Analyzer::SaveAnalyzerDataToFile(uint64_t session_id, uint64_t graph_id) {
GELOGD("start to save analyze file!"); GELOGD("start to save analyze file!");
auto graph_info = GetJsonObject(session_id, graph_id);
GE_CHECK_NOTNULL(graph_info);
if (graph_info->op_info.size() == 0) {
GELOGD("session_id:%lu graph_id:%lu does not owner op info, break it!", session_id, graph_id);
return SUCCESS;
}
std::lock_guard<std::mutex> lg(file_mutex_); std::lock_guard<std::mutex> lg(file_mutex_);
json_file_.open(json_file_name_, std::ios::out); json_file_.open(json_file_name_, std::ios::app);
if (!json_file_.is_open()) { if (!json_file_.is_open()) {
GELOGE(FAILED, "analyzer file does not exist[%s]", json_file_name_.c_str()); GELOGE(FAILED, "analyzer file does not exist[%s]", json_file_name_.c_str());
return PARAM_INVALID; return PARAM_INVALID;
} }
std::lock_guard<std::recursive_mutex> lk(mutex_); json jsn;
for (auto &ele : graph_infos_) { GraphInfoToJson(jsn, *graph_info);
for (auto &ele2 : ele.second) { json_file_ << jsn.dump(kJsonDumpLevel) << std::endl;
json jsn;
GraphInfoToJson(jsn, *(ele2.second));
json_file_ << jsn.dump(kJsonDumpLevel) << std::endl;
}
}
json_file_.close(); json_file_.close();
return SUCCESS; return SUCCESS;
} }
@ -237,13 +241,7 @@ ge::Status Analyzer::DoAnalyze(DataInfo &data_info) {
return FAILED; return FAILED;
} }
// create json file // create json file
status = CreateAnalyzerFile(); return CreateAnalyzerFile();
if (status != SUCCESS) {
GELOGE(status, "create analyzer file failed!");
return status;
}
// save data to file
return SaveAnalyzerDataToFile();
} }
ge::Status Analyzer::SaveOpInfo(ge::OpDescPtr desc, DataInfo &data_info, ge::Status Analyzer::SaveOpInfo(ge::OpDescPtr desc, DataInfo &data_info,
@ -256,18 +254,18 @@ ge::Status Analyzer::SaveOpInfo(ge::OpDescPtr desc, DataInfo &data_info,
op_info.error_type = iter->second; op_info.error_type = iter->second;
op_info.op_name = desc->GetName(); op_info.op_name = desc->GetName();
op_info.op_type = desc->GetType(); op_info.op_type = desc->GetType();
op_info.reason = data_info.reason; op_info.reason = data_info.reason;
for (const auto &ptr : desc->GetAllInputsDescPtr()) { for (const auto &ptr : desc->GetAllInputsDescPtr()) {
TensorInfo tensor_info; TensorInfo tensor_info;
tensor_info.shape = ptr->GetShape().GetDims(); tensor_info.shape = ptr->GetShape().GetDims();
tensor_info.d_type = ge::TypeUtils::DataTypeToSerialString(ptr->GetDataType()); tensor_info.d_type = ge::TypeUtils::DataTypeToSerialString(ptr->GetDataType());
tensor_info.layout = ge::TypeUtils::FormatToSerialString(ptr->GetFormat()); tensor_info.layout = ge::TypeUtils::FormatToSerialString(ptr->GetFormat());
op_info.input_info.emplace_back(tensor_info); op_info.input_info.emplace_back(tensor_info);
} }
for (const auto &ptr : desc->GetAllOutputsDescPtr()) { for (const auto &ptr : desc->GetAllOutputsDescPtr()) {
TensorInfo tensor_info; TensorInfo tensor_info;
tensor_info.shape = ptr->GetShape().GetDims(); tensor_info.shape = ptr->GetShape().GetDims();
tensor_info.d_type = ge::TypeUtils::DataTypeToSerialString(ptr->GetDataType()); tensor_info.d_type = ge::TypeUtils::DataTypeToSerialString(ptr->GetDataType());
tensor_info.layout = ge::TypeUtils::FormatToSerialString(ptr->GetFormat()); tensor_info.layout = ge::TypeUtils::FormatToSerialString(ptr->GetFormat());
op_info.output_info.emplace_back(tensor_info); op_info.output_info.emplace_back(tensor_info);
@ -277,13 +275,13 @@ ge::Status Analyzer::SaveOpInfo(ge::OpDescPtr desc, DataInfo &data_info,
return SUCCESS; return SUCCESS;
} }
void Analyzer::TensorInfoToJson(json &j, const TensorInfo &tensor_info) { void Analyzer::TensorInfoToJson(json& j, const TensorInfo &tensor_info) {
j[kShape] = tensor_info.shape; j[kShape] = tensor_info.shape;
j[kDataType] = tensor_info.d_type; j[kDataType] = tensor_info.d_type;
j[kLayout] = tensor_info.layout; j[kLayout] = tensor_info.layout;
} }
void Analyzer::OpInfoToJson(json &j, const OpInfo &op_info) { void Analyzer::OpInfoToJson(json& j, const OpInfo &op_info) {
j[kErrorType] = op_info.error_type; j[kErrorType] = op_info.error_type;
j[kOpName] = op_info.op_name; j[kOpName] = op_info.op_name;
j[kOpType] = op_info.op_type; j[kOpType] = op_info.op_type;
@ -300,7 +298,7 @@ void Analyzer::OpInfoToJson(json &j, const OpInfo &op_info) {
} }
} }
void Analyzer::GraphInfoToJson(json &j, const GraphInfo &graph_info) { void Analyzer::GraphInfoToJson(json& j, const GraphInfo &graph_info) {
GELOGD("start to buff graph info!"); GELOGD("start to buff graph info!");
j[kSessionId] = graph_info.session_id; j[kSessionId] = graph_info.session_id;
j[kGraphId] = graph_info.graph_id; j[kGraphId] = graph_info.graph_id;
@ -312,4 +310,4 @@ void Analyzer::GraphInfoToJson(json &j, const GraphInfo &graph_info) {
} }
j[kOp] = json_op_infos; j[kOp] = json_op_infos;
} }
} // namespace ge } // namespace ge

@ -1,5 +1,5 @@
/** /**
* Copyright 2019-2020 Huawei Technologies Co., Ltd * Copyright 2020 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -33,12 +33,12 @@
namespace ge { namespace ge {
namespace analyzer { namespace analyzer {
enum AnalyzeType { enum AnalyzeType {
PARSER = 0, PARSER = 0,
INFER_SHAPE = 1, INFER_SHAPE = 1,
CHECKSUPPORT = 2, CHECKSUPPORT = 2,
GRAPH_OPTIMIZE = 3, GRAPH_OPTIMIZE = 3,
GRAPH_PARTION = 4, GRAPH_PARTION = 4,
GRAPH_BUILDER = 5, GRAPH_BUILDER = 5,
}; };
struct TensorInfo { struct TensorInfo {
@ -66,7 +66,8 @@ struct DataInfo {
DataInfo() = default; DataInfo() = default;
~DataInfo() = default; ~DataInfo() = default;
DataInfo(uint64_t sess, uint64_t graph, AnalyzeType type, ge::NodePtr node, std::string error_info) { DataInfo(uint64_t sess, uint64_t graph, AnalyzeType type,
ge::NodePtr node, std::string error_info) {
session_id = sess; session_id = sess;
graph_id = graph; graph_id = graph;
analyze_type = type; analyze_type = type;
@ -79,10 +80,10 @@ struct DataInfo {
ge::NodePtr node_ptr{nullptr}; ge::NodePtr node_ptr{nullptr};
std::string reason; std::string reason;
}; };
} // namespace analyzer }
class Analyzer { class Analyzer {
public: public:
/** /**
* @ingroup ge * @ingroup ge
* @brief: get analyzer instance. * @brief: get analyzer instance.
@ -156,33 +157,39 @@ class Analyzer {
*/ */
ge::Status DoAnalyze(analyzer::DataInfo &data_info); ge::Status DoAnalyze(analyzer::DataInfo &data_info);
/**
* @ingroup ge
* @brief: Buff analyzed data and output to json file
* @param [in]: session id , graph id
* @return: 0: SUCCESS other: FAILED
*/
ge::Status SaveAnalyzerDataToFile(uint64_t session_id, uint64_t graph_id);
Analyzer(const Analyzer &) = delete; Analyzer(const Analyzer &) = delete;
Analyzer &operator=(const Analyzer &) = delete; Analyzer& operator=(const Analyzer&) = delete;
Analyzer(Analyzer &&) = delete; Analyzer(Analyzer &&) = delete;
Analyzer &operator=(Analyzer &&) = delete; Analyzer& operator=(Analyzer &&) = delete;
private:
private: void TensorInfoToJson(nlohmann::json& j, const analyzer::TensorInfo &tensor_info);
void TensorInfoToJson(nlohmann::json &j, const analyzer::TensorInfo &tensor_info); void OpInfoToJson(nlohmann::json& j, const analyzer::OpInfo &op_info);
void OpInfoToJson(nlohmann::json &j, const analyzer::OpInfo &op_info); void GraphInfoToJson(nlohmann::json& j, const analyzer::GraphInfo &graph_info);
void GraphInfoToJson(nlohmann::json &j, const analyzer::GraphInfo &graph_info);
ge::Status SaveAnalyzerDataToFile();
ge::Status SaveOpInfo(ge::OpDescPtr desc, analyzer::DataInfo &data_info, ge::Status SaveOpInfo(ge::OpDescPtr desc, analyzer::DataInfo &data_info,
std::shared_ptr<analyzer::GraphInfo> graph_info); std::shared_ptr<analyzer::GraphInfo> graph_info);
void ClearHistoryFile(); void ClearHistoryFile();
ge::Status CreateAnalyzerFile(); ge::Status CreateAnalyzerFile();
explicit Analyzer(){}; explicit Analyzer() {};
~Analyzer() = default; ~Analyzer() = default;
private: private:
std::map<uint64_t, std::map<uint64_t, std::shared_ptr<analyzer::GraphInfo>>> graph_infos_; std::map<uint64_t, std::map<uint64_t, std::shared_ptr<analyzer::GraphInfo>>> graph_infos_;
std::recursive_mutex mutex_; // protect graph_infos_ std::recursive_mutex mutex_; // protect graph_infos_
std::mutex file_mutex_; // protect json_file_ std::mutex file_mutex_; // protect json_file_
std::ofstream json_file_; std::ofstream json_file_;
std::string json_file_name_; std::string json_file_name_;
std::atomic_bool is_json_file_create_{false}; std::atomic_bool is_json_file_create_{false};
}; };
} // namespace ge } // namespace ge
#endif // DOMI_ANALYZER_ANANLYZER_H_ #endif // DOMI_ANALYZER_ANANLYZER_H_

@ -1,72 +0,0 @@
# Copyright 2019-2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
# libge_client.so
# add all proto files, generate corresponding .h and .cc files
set(CMAKE_CXX_FLAGS "-Wno-unused-variable ${CMAKE_CXX_FLAGS}")
file(GLOB PROTO_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"../proto/ge_api.proto"
)
file(GLOB PROTO_HEADER_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"../proto/ge_ir.proto"
"../proto/task.proto"
"../proto/om.proto"
"../proto/insert_op.proto"
)
file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"ge_api.cc"
"ge_prof.cc"
)
ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
ge_protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST})
# include directories
include_directories(${CMAKE_CURRENT_LIST_DIR})
include_directories(${GE_SOURCE_DIR}/ge)
include_directories(${GE_SOURCE_DIR}/inc)
include_directories(${GE_SOURCE_DIR}/inc/external)
include_directories(${GE_SOURCE_DIR}/inc/common)
include_directories(${GE_SOURCE_DIR}/inc/framework)
include_directories(${GE_SOURCE_DIR}/metadef/inc)
include_directories(${GE_SOURCE_DIR}/metadef/inc/external/graph)
include_directories(${GE_SOURCE_DIR}/metadef/inc/graph)
include_directories(${GE_SOURCE_DIR}/third_party/fwkacllib/inc)
include_directories(${GE_SOURCE_DIR}/third_party/fwkacllib/inc/cce)
include_directories(${CMAKE_BINARY_DIR})
include_directories(${CMAKE_BINARY_DIR}/proto/ge)
############ libge_client.so ################
add_library(ge_client SHARED ${SRC_LIST} ${PROTO_SRCS} ${PROTO_HEADER_HDRS})
target_compile_definitions(ge_client PRIVATE
Werror
PROTOBUF_INLINE_NOT_IN_HEADERS=0
REUSE_MEMORY=1
PLATFORM_CLOUD)
target_link_libraries(ge_client
graph
ge_compiler
ge_common
${PROTOBUF_LIBRARY}
${register}
${c_sec}
${slog}
${mmpa}
${runtime}
${msprof}
rt
dl)

@ -380,7 +380,7 @@ Status Session::RunGraphAsync(uint32_t graph_id, const std::vector<InputTensorIn
} }
GELOGT(TRACE_RUNNING, "Run Graph Asynchronously"); GELOGT(TRACE_RUNNING, "Run Graph Asynchronously");
GELOGW( GELOGW(
"The callback function will not be checked. Please ensure that the implementation of the function is trusted."); "The callback function will not be checked. Please ensure that the implementation of the function is trusted.");
Status ret = ge::GELib::GetInstance()->SessionManagerObj().RunGraphAsync(sessionId_, graph_id, inputs, callback); Status ret = ge::GELib::GetInstance()->SessionManagerObj().RunGraphAsync(sessionId_, graph_id, inputs, callback);
if (ret != SUCCESS) { if (ret != SUCCESS) {

@ -1,5 +1,5 @@
/** /**
* Copyright 2019-2020 Huawei Technologies Co., Ltd * Copyright 2020 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -29,39 +29,22 @@ using std::vector;
namespace { namespace {
const uint32_t kMaxDeviceNum = 64; const uint32_t kMaxDeviceNum = 64;
const std::string PROFILING_INIT = "prof_init"; const uint32_t kDeviceListIndex = 3;
const std::string PROFILING_FINALIZE = "prof_finalize"; const std::string kProfilingInit = "prof_init";
const std::string PROFILING_START = "prof_start"; const std::string kProfilingFinalize = "prof_finalize";
const std::string PROFILING_STOP = "prof_stop"; const std::string kProfilingStart = "prof_start";
const std::string DEVICES_NUMS = "devNums"; const std::string kProfilingStop = "prof_stop";
const std::string DEVICE_ID_LIST = "devIdList"; const std::string kDeviceNums = "devNums";
const std::string AICORE_METRICS = "aicoreMetrics"; const std::string kDeviceIdList = "devIdList";
const std::string kAicoreMetrics = "aicoreMetrics";
const std::map<ge::ProfilingAicoreMetrics, std::string> kProfAicoreMetricsToString = { const std::map<ge::ProfilingAicoreMetrics, std::string> kProfAicoreMetricsToString = {
{ge::kAicoreArithmaticThroughput, "AICORE_ARITHMATIC_THROUGHPUT"}, {ge::kAicoreArithmaticThroughput, "AICORE_ARITHMATIC_THROUGHPUT"},
{ge::kAicorePipeline, "AICORE_PIPELINE"}, {ge::kAicorePipeline, "AICORE_PIPELINE"},
{ge::kAicoreSynchronization, "AICORE_SYNCHRONIZATION"}, {ge::kAicoreSynchronization, "AICORE_SYNCHRONIZATION"},
{ge::kAicoreMemory, "AICORE_MEMORY"}, {ge::kAicoreMemory, "AICORE_MEMORY"},
{ge::kAicoreInternalMemory, "AICORE_INTERNAL_MEMORY"}, {ge::kAicoreInternalMemory, "AICORE_INTERNAL_MEMORY"},
{ge::kAicoreStall, "AICORE_STALL"}, {ge::kAicoreStall, "AICORE_STALL"}};
{ge::kAicoreMetricsAll, "AICORE_METRICS_ALL"}};
const std::map<uint64_t, uint64_t> kDataTypeConfigMapping = {{ge::kProfAcl, PROF_ACL_API},
{ge::kProfTaskTime, PROF_TASK_TIME},
{ge::kProfAiCoreMetrics, PROF_AICORE_METRICS},
{ge::kProfAicpuTrace, PROF_AICPU_TRACE},
{ge::kProfModelExecute, PROF_MODEL_EXECUTE},
{ge::kProfRuntimeApi, PROF_RUNTIME_API},
{ge::kProfRuntimeTrace, PROF_RUNTIME_TRACE},
{ge::kProfScheduleTimeline, PROF_SCHEDULE_TIMELINE},
{ge::kProfScheduleTrace, PROF_SCHEDULE_TRACE},
{ge::kProfAiVectorCoreMetrics, PROF_AIVECTORCORE_METRICS},
{ge::kProfSubtaskTime, PROF_SUBTASK_TIME},
{ge::kProfTrainingTrace, PROF_TRAINING_TRACE},
{ge::kProfHcclTrace, PROF_HCCL_TRACE},
{ge::kProfDataProcess, PROF_DATA_PROCESS},
{ge::kProfTaskTrace, PROF_TASK_TRACE},
{ge::kProfModelLoad, PROF_MODEL_LOAD}};
} // namespace } // namespace
static bool g_graph_prof_init_ = false; static bool g_graph_prof_init_ = false;
@ -107,11 +90,11 @@ Status aclgrphProfInit(const char *profiler_path, uint32_t length) {
GraphLoader graph_loader; GraphLoader graph_loader;
Command command; Command command;
command.cmd_params.clear(); command.cmd_params.clear();
command.cmd_type = PROFILING_INIT; command.cmd_type = kProfilingInit;
command.module_index = kProfModelLoad | kProfTrainingTrace; command.module_index = PROF_MODEL_LOAD;
ret = graph_loader.CommandHandle(command); ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) { if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command %s failed, config = %s", PROFILING_INIT.c_str(), profiler_path); GELOGE(ret, "Handle profiling command %s failed, config = %s", kProfilingInit.c_str(), profiler_path);
return ret; return ret;
} }
if (!g_graph_prof_init_) { if (!g_graph_prof_init_) {
@ -143,10 +126,10 @@ Status aclgrphProfFinalize() {
GraphLoader graph_loader; GraphLoader graph_loader;
Command command; Command command;
command.cmd_params.clear(); command.cmd_params.clear();
command.cmd_type = PROFILING_FINALIZE; command.cmd_type = kProfilingFinalize;
Status ret = graph_loader.CommandHandle(command); Status ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) { if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command %s failed.", PROFILING_FINALIZE.c_str()); GELOGE(ret, "Handle profiling command %s failed.", kProfilingFinalize.c_str());
return ret; return ret;
} }
@ -164,9 +147,9 @@ Status aclgrphProfFinalize() {
bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector<string> &prof_config_params) { bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector<string> &prof_config_params) {
prof_config_params.clear(); prof_config_params.clear();
prof_config_params.emplace_back(DEVICES_NUMS); prof_config_params.emplace_back(kDeviceNums);
prof_config_params.emplace_back(std::to_string(profiler_config->config.devNums)); prof_config_params.emplace_back(std::to_string(profiler_config->config.devNums));
prof_config_params.emplace_back(DEVICE_ID_LIST); prof_config_params.emplace_back(kDeviceIdList);
std::string devID = ""; std::string devID = "";
if (profiler_config->config.devNums == 0) { if (profiler_config->config.devNums == 0) {
GELOGW("The device num is invalid."); GELOGW("The device num is invalid.");
@ -180,9 +163,9 @@ bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector<str
} }
prof_config_params.push_back(devID); prof_config_params.push_back(devID);
prof_config_params.push_back(AICORE_METRICS); prof_config_params.push_back(kAicoreMetrics);
auto iter = auto iter =
kProfAicoreMetricsToString.find(static_cast<ProfilingAicoreMetrics>(profiler_config->config.aicoreMetrics)); kProfAicoreMetricsToString.find(static_cast<ProfilingAicoreMetrics>(profiler_config->config.aicoreMetrics));
if (iter == kProfAicoreMetricsToString.end()) { if (iter == kProfAicoreMetricsToString.end()) {
GELOGW("The prof aicore metrics is invalid."); GELOGW("The prof aicore metrics is invalid.");
return false; return false;
@ -250,13 +233,7 @@ aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t dev
} }
config->config.aicoreMetrics = static_cast<ProfAicoreMetrics>(aicore_metrics); config->config.aicoreMetrics = static_cast<ProfAicoreMetrics>(aicore_metrics);
uint64_t data_type = 0; config->config.dataTypeConfig = data_type_config;
for (auto &iter : kDataTypeConfigMapping) {
if ((iter.first & data_type_config) == iter.first) {
data_type |= iter.second;
}
}
config->config.dataTypeConfig = data_type;
GELOGI("Successfully create prof config."); GELOGI("Successfully create prof config.");
return config; return config;
} }
@ -309,9 +286,11 @@ Status aclgrphProfStart(aclgrphProfConfig *profiler_config) {
GraphLoader graph_loader; GraphLoader graph_loader;
Command command; Command command;
command.cmd_params.clear(); command.cmd_params.clear();
command.cmd_type = PROFILING_START; command.cmd_type = kProfilingStart;
command.cmd_params = prof_params; command.cmd_params = prof_params;
command.module_index = profiler_config->config.dataTypeConfig; command.module_index = profiler_config->config.dataTypeConfig;
GELOGI("Profiling will start, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(),
prof_params[kDeviceListIndex].c_str(), command.module_index);
ret = graph_loader.CommandHandle(command); ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) { if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command failed"); GELOGE(ret, "Handle profiling command failed");
@ -360,9 +339,11 @@ Status aclgrphProfStop(aclgrphProfConfig *profiler_config) {
GraphLoader graph_loader; GraphLoader graph_loader;
Command command; Command command;
command.cmd_params.clear(); command.cmd_params.clear();
command.cmd_type = PROFILING_STOP; command.cmd_type = kProfilingStop;
command.cmd_params = prof_params; command.cmd_params = prof_params;
command.module_index = profiler_config->config.dataTypeConfig; command.module_index = profiler_config->config.dataTypeConfig;
GELOGI("Profiling will stop, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(),
prof_params[kDeviceListIndex].c_str(), command.module_index);
ret = graph_loader.CommandHandle(command); ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) { if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command failed"); GELOGE(ret, "Handle profiling command failed");

@ -0,0 +1 @@
../../proto/ge_api.proto

@ -0,0 +1 @@
../../proto/ge_ir.proto

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save