From dc532ff6a9625d002096b2d61b46e419c97b1c78 Mon Sep 17 00:00:00 2001 From: xulei2020 <“xulei83@huawei.com”> Date: Sat, 5 Dec 2020 16:42:50 +0800 Subject: [PATCH] fix the minddata lite build, keep the third package compiled in the same way --- build.sh | 93 ------------------- cmake/external_libs/jpeg_turbo.cmake | 32 ++++++- cmake/package_lite.cmake | 10 +- cmake/utils.cmake | 3 - mindspore/lite/CMakeLists.txt | 30 ++---- .../java/app/src/main/native/CMakeLists.txt | 5 + mindspore/lite/minddata/CMakeLists.txt | 19 +++- third_party/eigen | 1 - third_party/libjpeg-turbo | 1 - third_party/opencv | 1 - 10 files changed, 62 insertions(+), 133 deletions(-) delete mode 160000 third_party/eigen delete mode 160000 third_party/libjpeg-turbo delete mode 160000 third_party/opencv diff --git a/build.sh b/build.sh index cc48a39cf1..0e71603c50 100755 --- a/build.sh +++ b/build.sh @@ -591,95 +591,6 @@ build_opencl() { fi } -build_opencv() { - # check what platform we are building opencv on - cd ${BASEPATH} - if [[ "${LITE_PLATFORM}" == "x86_64" ]]; then - OPENCV_BIN="${BASEPATH}"/third_party/opencv/build/lib/libopencv_core.so.4.2.0 - elif [[ "${LITE_PLATFORM}" == "arm32" ]]; then - OPENCV_BIN="${BASEPATH}"/third_party/opencv/build/lib/armeabi-v7a/libopencv_core.so - else - OPENCV_BIN="${BASEPATH}"/third_party/opencv/build/lib/arm64-v8a/libopencv_core.so - - fi - if [[ ! -f "${OPENCV_BIN}" ]]; then - if [[ "${MSLIBS_SERVER}" ]]; then - cd "${BASEPATH}"/third_party/ - rm -rf 4.2.0.tar.gz ./opencv - wget http://${MSLIBS_SERVER}:8081/libs/opencv/4.2.0.tar.gz - tar -zxvf ./4.2.0.tar.gz - mv ./opencv-4.2.0 ./opencv - rm -rf 4.2.0.tar.gz - else - git submodule update --init --recursive third_party/opencv - fi - cd ${BASEPATH}/third_party/opencv - rm -rf build && mkdir -p build && cd build && cmake ${CMAKE_MINDDATA_ARGS} -DBUILD_SHARED_LIBS=ON -DBUILD_ANDROID_PROJECTS=OFF \ - -DBUILD_LIST=core,imgcodecs,imgproc -DBUILD_ZLIB=ON .. && make -j$THREAD_NUM - fi -} - -build_jpeg_turbo() { - if [ -d "${BASEPATH}"/third_party/libjpeg-turbo/lib ];then - rm -rf "${BASEPATH}"/third_party/libjpeg-turbo/lib - fi - cd ${BASEPATH} - if [[ "${LITE_PLATFORM}" == "x86_64" ]]; then - JPEG_TURBO="${BASEPATH}"/third_party/libjpeg-turbo/lib/libjpeg.so.62.3.0 - else - JPEG_TURBO="${BASEPATH}"/third_party/libjpeg-turbo/lib/libjpeg.so - fi - - if [[ ! -f "${JPEG_TURBO}" ]]; then - if [[ "${MSLIBS_SERVER}" ]]; then - cd "${BASEPATH}"/third_party/ - rm -rf 2.0.4.tar.gz ./libjpeg-turbo - wget http://${MSLIBS_SERVER}:8081/libs/jpeg_turbo/2.0.4.tar.gz - tar -zxvf ./2.0.4.tar.gz - mv ./libjpeg-turbo-2.0.4 ./libjpeg-turbo - rm -rf ./2.0.4.tar.gz - else - git submodule update --init --recursive third_party/libjpeg-turbo - fi - - cd ${BASEPATH}/third_party/libjpeg-turbo - rm -rf build && mkdir -p build && cd build && cmake ${CMAKE_MINDDATA_ARGS} -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="${BASEPATH}/third_party/libjpeg-turbo" .. && make -j$THREAD_NUM && make install - fi -} - -build_eigen() { - cd ${BASEPATH} - if [[ "${MSLIBS_SERVER}" ]]; then - cd "${BASEPATH}"/third_party/ - rm -rf ./eigen-3.*.tar.gz ./eigen - wget http://${MSLIBS_SERVER}:8081/libs/eigen3/eigen-3.3.7.tar.gz - tar -zxvf ./eigen-3.3.7.tar.gz - mv ./eigen-3.3.7 ./eigen - rm -rf ./eigen-3.*.tar.gz - else - git submodule update --init --recursive third_party/eigen - - fi -} - -build_minddata_lite_deps() -{ - echo "start build minddata lite project" - if [[ "${LITE_PLATFORM}" == "arm64" ]]; then - CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19 \ - -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=arm64-v8a -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang \ - -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - elif [[ "${LITE_PLATFORM}" == "arm32" ]]; then - CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19 \ - -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=armeabi-v7a -DANDROID_TOOLCHAIN_NAME=clang \ - -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - else - CMAKE_MINDDATA_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - fi - build_eigen - build_jpeg_turbo -} get_version() { VERSION_MAJOR=$(grep "const int ms_version_major =" ${BASEPATH}/mindspore/lite/include/version.h | tr -dc "[0-9]") @@ -703,10 +614,6 @@ build_lite() build_gtest fi - if [[ "${COMPILE_MINDDATA_LITE}" == "lite" || "${COMPILE_MINDDATA_LITE}" == "full" || "${COMPILE_MINDDATA_LITE}" == "wrapper" ]]; then - build_minddata_lite_deps - fi - cd "${BASEPATH}/mindspore/lite" if [[ "${INC_BUILD}" == "off" ]]; then rm -rf build diff --git a/cmake/external_libs/jpeg_turbo.cmake b/cmake/external_libs/jpeg_turbo.cmake index b04f26e123..3040f6cd63 100644 --- a/cmake/external_libs/jpeg_turbo.cmake +++ b/cmake/external_libs/jpeg_turbo.cmake @@ -1,4 +1,3 @@ -set(jpeg_turbo_USE_STATIC_LIBS ON) if (ENABLE_GITEE) set(REQ_URL "https://gitee.com/mirrors/libjpeg-turbo/repository/archive/2.0.4.tar.gz") @@ -15,13 +14,40 @@ else() endif() set(jpeg_turbo_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") + + +set(jpeg_turbo_USE_STATIC_LIBS ON) +set(JPEG_TURBO_PATCHE ${CMAKE_SOURCE_DIR}/third_party/patch/jpeg_turbo/jpeg_turbo.patch001) +set(CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=TRUE -DWITH_SIMD=ON) +if (BUILD_LITE) + set(jpeg_turbo_USE_STATIC_LIBS OFF) + set(JPEG_TURBO_PATCHE ${TOP_DIR}/third_party/patch/jpeg_turbo/jpeg_turbo.patch001) + if (PLATFORM_ARM64) + set(CMAKE_OPTION -DCMAKE_TOOLCHAIN_FILE=$ENV{ANDROID_NDK}/build/cmake/android.toolchain.cmake + -DANDROID_NATIVE_API_LEVEL=19 + -DANDROID_NDK=$ENV{ANDROID_NDK} + -DANDROID_ABI=arm64-v8a + -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang + -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}) + endif() + if (PLATFORM_ARM32) + set(CMAKE_OPTION -DCMAKE_TOOLCHAIN_FILE=$ENV{ANDROID_NDK}/build/cmake/android.toolchain.cmake + -DANDROID_NATIVE_API_LEVEL=19 + -DANDROID_NDK=$ENV{ANDROID_NDK} + -DANDROID_ABI=armeabi-v7a + -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang + -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}) + endif() + +endif () + mindspore_add_pkg(jpeg_turbo VER 2.0.4 LIBS jpeg turbojpeg URL ${REQ_URL} MD5 ${MD5} - CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=TRUE -DWITH_SIMD=ON - PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/jpeg_turbo/jpeg_turbo.patch001 + CMAKE_OPTION ${CMAKE_OPTION} + PATCHES ${JPEG_TURBO_PATCHE} ) include_directories(${jpeg_turbo_INC}) add_library(mindspore::jpeg_turbo ALIAS jpeg_turbo::jpeg) diff --git a/cmake/package_lite.cmake b/cmake/package_lite.cmake index 3a7ebb4e0e..ed515bfa4a 100644 --- a/cmake/package_lite.cmake +++ b/cmake/package_lite.cmake @@ -22,18 +22,16 @@ set(FLATBF_DIR_RUN_X86 ${MAIN_DIR}-${RUN_X86_COMPONENT_NAME}/third_party/flatbuf if (BUILD_MINDDATA STREQUAL "full" OR BUILD_MINDDATA STREQUAL "wrapper") install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/ DESTINATION ${MIND_DATA_INC_DIR} COMPONENT ${COMPONENT_NAME} FILES_MATCHING PATTERN "*.h" PATTERN "vision.h" EXCLUDE) + file(GLOB JPEGTURBO_LIB_LIST ${jpeg_turbo_LIBPATH}/*.so) if (PLATFORM_ARM64) install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR} COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) elseif (PLATFORM_ARM32) install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR} COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) + install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME}) else () install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR_RUN_X86} COMPONENT ${RUN_X86_COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62.3.0 DESTINATION ${TURBO_DIR_RUN_X86}/lib RENAME libjpeg.so.62 COMPONENT ${RUN_X86_COMPONENT_NAME}) - install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so.0.2.0 DESTINATION ${TURBO_DIR_RUN_X86}/lib RENAME libturbojpeg.so.0 COMPONENT ${RUN_X86_COMPONENT_NAME}) + install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR_RUN_X86}/lib COMPONENT ${RUN_X86_COMPONENT_NAME}) endif () endif () diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 465c7b8e8a..cae5a45924 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -134,9 +134,6 @@ function(__find_pkg_then_add_target pkg_name pkg_exe lib_path) endif () set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND) find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH) - if (NOT ${_LIB_NAME}_LIB AND BUILD_LITE AND PLATFORM_ARM) - set(${_LIB_NAME}_LIB "${${pkg_name}_BASE_DIR}/${lib_path}/lib${_LIB_SEARCH_NAME}.so") - endif(NOT ${_LIB_NAME}_LIB AND BUILD_LITE AND PLATFORM_ARM) if(NOT ${_LIB_NAME}_LIB) return() endif() diff --git a/mindspore/lite/CMakeLists.txt b/mindspore/lite/CMakeLists.txt index cca5534ad5..c0595ae818 100644 --- a/mindspore/lite/CMakeLists.txt +++ b/mindspore/lite/CMakeLists.txt @@ -33,8 +33,14 @@ set(BUILD_LITE "on") set(PLATFORM_ARM "off") if (PLATFORM_ARM64 OR PLATFORM_ARM32) set(PLATFORM_ARM "on") + #set for cross-compiling toolchain + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) + set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) endif() + + if (SUPPORT_GPU) set(PROCESS_UNIT gpu) else () @@ -93,6 +99,10 @@ include(${TOP_DIR}/cmake/dependency_utils.cmake) include(${TOP_DIR}/cmake/dependency_securec.cmake) include(${TOP_DIR}/cmake/external_libs/flatbuffers.cmake) +if (ENABLE_CONVERTER OR BUILD_MINDDATA STREQUAL "full") +include(${TOP_DIR}/cmake/external_libs/json.cmake) +endif() + file(GLOB FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/schema/*.fbs) ms_build_flatbuffers_lite(FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/schema/ fbs_src ${CMAKE_BINARY_DIR}/schema "") ms_build_flatbuffers_lite(FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/schema/ fbs_inner_src ${CMAKE_BINARY_DIR}/schema/inner "inner") @@ -147,7 +157,6 @@ if (ENABLE_CONVERTER) MESSAGE(FATAL_ERROR "Cannot build converter in arm platform") endif() include_directories(${PYTHON_INCLUDE_DIRS}) - include(${TOP_DIR}/cmake/external_libs/json.cmake) include(${TOP_DIR}/cmake/external_libs/eigen.cmake) include(${TOP_DIR}/cmake/external_libs/protobuf.cmake) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/tools/converter) @@ -183,25 +192,6 @@ if (NOT PLATFORM_ARM32 AND NOT PLATFORM_ARM64) endif () if (BUILD_MINDDATA STREQUAL "lite" OR BUILD_MINDDATA STREQUAL "full" OR BUILD_MINDDATA STREQUAL "wrapper") - # add sentencepiece dependency - # include(${TOP_DIR}/cmake/external_libs/sentencepiece.cmake) - # json - if (NOT ENABLE_CONVERTER) - include(${TOP_DIR}/cmake/external_libs/json.cmake) - endif () - # eigen - include_directories(${TOP_DIR}/third_party/eigen/) - # jpeg-turbo - add_library(jpeg-turbo SHARED IMPORTED) - set_target_properties(jpeg-turbo PROPERTIES - IMPORTED_LOCATION ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so - ) - add_library(jpeg SHARED IMPORTED) - set_target_properties(jpeg PROPERTIES - IMPORTED_LOCATION ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so - ) - include_directories(${TOP_DIR}/third_party/libjpeg-turbo/include) - add_compile_definitions(ENABLE_ANDROID) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/minddata) endif () diff --git a/mindspore/lite/java/java/app/src/main/native/CMakeLists.txt b/mindspore/lite/java/java/app/src/main/native/CMakeLists.txt index 5df2cb150e..1afbbe8274 100644 --- a/mindspore/lite/java/java/app/src/main/native/CMakeLists.txt +++ b/mindspore/lite/java/java/app/src/main/native/CMakeLists.txt @@ -11,6 +11,11 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMS_VERSION_MAJOR=${MS_VERSION_MAJOR} -DMS_ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMS_VERSION_MAJOR=${MS_VERSION_MAJOR} -DMS_VERSION_MINOR=${MS_VERSION_MINOR} -DMS_VERSION_REVISION=${MS_VERSION_REVISION}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") +#set for cross-compiling toolchain +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) + if (ENABLE_VERBOSE) set(CMAKE_VERBOSE_MAKEFILE on) endif () diff --git a/mindspore/lite/minddata/CMakeLists.txt b/mindspore/lite/minddata/CMakeLists.txt index 66a2af4992..1908466178 100644 --- a/mindspore/lite/minddata/CMakeLists.txt +++ b/mindspore/lite/minddata/CMakeLists.txt @@ -1,9 +1,18 @@ +find_package(Patch) +if (NOT Patch_FOUND) + message(FATAL_ERROR "Patch not found, please set environment variable MS_PATCH_PATH to path where Patch is located, " + "usually found in GIT_PATH/usr/bin on Windows") +endif () + +include(${TOP_DIR}/cmake/external_libs/jpeg_turbo.cmake) + set(MINDDATA_DIR ${CCSRC_DIR}/minddata/dataset) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D_LIBCPP_DISABLE_EXTERN_TEMPLATE=1 -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override") + set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -I/usr/local/include -std=c++17 -Wall -fPIC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTION_CXX_FLAGS}") @@ -281,8 +290,8 @@ if (BUILD_MINDDATA STREQUAL "full") find_package(Threads REQUIRED) target_link_libraries(minddata-lite securec - jpeg-turbo - jpeg + mindspore::jpeg_turbo + mindspore::turbojpeg mindspore::json Threads::Threads ) @@ -349,8 +358,8 @@ elseif (BUILD_MINDDATA STREQUAL "wrapper") find_package(Threads REQUIRED) target_link_libraries(minddata-lite securec - jpeg-turbo - jpeg + mindspore::jpeg_turbo + mindspore::turbojpeg mindspore::json Threads::Threads ) @@ -428,7 +437,7 @@ elseif (BUILD_MINDDATA STREQUAL "lite") add_library(minddata-lite SHARED ${MINDDATA_CORE_SRC_FILES} ${MINDDATA_KERNELS_SRC_FILES} - ${MINDDATA_KERNELS_IMAGE_LITE_CV_FILES} + ${MINDDATA_KERNELS_IMAGE_LITE_CV_FILES} ${MINDDATA_KERNELS_IMAGE_SRC_FILES} ${MINDDATA_KERNELS_DATA_SRC_FILES} ${MINDDATA_DIR}/util/status.cc diff --git a/third_party/eigen b/third_party/eigen deleted file mode 160000 index daf9bbeca2..0000000000 --- a/third_party/eigen +++ /dev/null @@ -1 +0,0 @@ -Subproject commit daf9bbeca26e98da2eed0058835cbb04e0a30ad8 diff --git a/third_party/libjpeg-turbo b/third_party/libjpeg-turbo deleted file mode 160000 index b443c541b9..0000000000 --- a/third_party/libjpeg-turbo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b443c541b9a6fdcac214f9f003de0aa13e480ac1 diff --git a/third_party/opencv b/third_party/opencv deleted file mode 160000 index bda89a6469..0000000000 --- a/third_party/opencv +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bda89a6469aa79ecd8713967916bd754bff1d931