Pre Merge pull request !14773 from zhoufeng/use-shared-grpc

cmake/external_libs/grpc.cmake

@@ -1,13 +1,10 @@
-set(grpc_USE_STATIC_LIBS ON)
+set(grpc_USE_STATIC_LIBS OFF)
 if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC \
-        -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
 elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
-    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter \
-        -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
 else()
-    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter \
-        -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
     if(NOT ENABLE_GLIBCXX)
         set(grpc_CXXFLAGS "${grpc_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
     endif()
@@ -15,7 +12,6 @@ endif()
 
 set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
 
-
 if(EXISTS ${protobuf_ROOT}/lib64)
   set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf")
 else()
@@ -48,7 +44,9 @@ mindspore_add_pkg(grpc
         EXE grpc_cpp_plugin
         URL ${REQ_URL}
         MD5 ${MD5}
+        PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/grpc/grpc.patch001
         CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release
+        -DBUILD_SHARED_LIBS=ON
         -DgRPC_INSTALL:BOOL=ON
         -DgRPC_BUILD_TESTS:BOOL=OFF
         -DgRPC_PROTOBUF_PROVIDER:STRING=package
@@ -71,14 +69,6 @@ add_library(mindspore::grpc++ ALIAS grpc::grpc++)
 # link other grpc libs
 target_link_libraries(grpc::grpc++ INTERFACE grpc::grpc grpc::gpr grpc::upb grpc::address_sorting)
 
-# link built dependencies
-target_link_libraries(grpc::grpc++ INTERFACE mindspore::z)
-target_link_libraries(grpc::grpc++ INTERFACE mindspore::cares)
-target_link_libraries(grpc::grpc++ INTERFACE mindspore::absl_strings mindspore::absl_throw_delegate
-                      mindspore::absl_raw_logging_internal mindspore::absl_int128 mindspore::absl_bad_optional_access)
-target_link_libraries(grpc::grpc++ INTERFACE mindspore::ssl mindspore::crypto)
-
-
 function(ms_grpc_generate c_var h_var)
     if(NOT ARGN)
         message(SEND_ERROR "Error: ms_grpc_generate() called without any proto files")

cmake/external_libs/openssl.cmake

@@ -0,0 +1,13 @@
+if(ENABLE_GITEE)
+  set(REQ_URL "https://gitee.com/mirrors/openssl/repository/archive/OpenSSL_1_1_1k.tar.gz")
+  set(MD5 "d4acbcc4a5e6c31d86ede95b5d22f7a0")
+else()
+  set(REQ_URL "https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1k.tar.gz")
+  set(MD5 "bdd51a68ad74618dd2519da8e0bcc759")
+endif()
+mindspore_add_pkg(openssl
+  VER 1.1.0
+  LIBS ssl crypto
+  URL ${REQ_URL}
+  MD5 ${MD5}
+  CONFIGURE_COMMAND ./config no-zlib no-shared)

cmake/external_libs/openssl_stub.cmake

@@ -1,18 +0,0 @@
-if(ENABLE_GITEE)
-  set(REQ_URL "https://gitee.com/mirrors/openssl/repository/archive/OpenSSL_1_1_0l.tar.gz")
-  set(MD5 "9d18479e0cac8ff62f7e3df3cceb69dc")
-else()
-  set(REQ_URL "https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_0l.tar.gz")
-  set(MD5 "46d9a2a92fd39198501503b40954e6f0")
-endif()
-mindspore_add_pkg(openssl
-  VER 1.1.0
-  LIBS ssl crypto
-  URL ${REQ_URL}
-  MD5 ${MD5}
-  PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/openssl-stub/openssl-stub.patch001
-  CONFIGURE_COMMAND ./config no-zlib)
-
-include_directories(${openssl_INC})
-add_library(mindspore::ssl ALIAS openssl::ssl)
-add_library(mindspore::crypto ALIAS openssl::crypto)

cmake/mind_expression.cmake

@@ -17,7 +17,7 @@ include(${CMAKE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake)
 
 if(MS_BUILD_GRPC)
     # build dependencies of gRPC
-    include(${CMAKE_SOURCE_DIR}/cmake/external_libs/openssl_stub.cmake)
+    include(${CMAKE_SOURCE_DIR}/cmake/external_libs/openssl.cmake)
     include(${CMAKE_SOURCE_DIR}/cmake/external_libs/absl.cmake)
     include(${CMAKE_SOURCE_DIR}/cmake/external_libs/c-ares.cmake)
     include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zlib.cmake)

cmake/package.cmake

@@ -264,6 +264,19 @@ if(NOT ENABLE_GE)
     endif()
 endif()
 
+if(MS_BUILD_GRPC)
+    install(FILES ${grpc_LIBPATH}/libgrpc++.so.1.27.3
+      DESTINATION ${INSTALL_LIB_DIR} RENAME libgrpc++.so.1 COMPONENT mindspore)
+    install(FILES ${grpc_LIBPATH}/libgrpc.so.9.0.0
+      DESTINATION ${INSTALL_LIB_DIR} RENAME libgrpc.so.9 COMPONENT mindspore)
+    install(FILES ${grpc_LIBPATH}/libgpr.so.9.0.0
+      DESTINATION ${INSTALL_LIB_DIR} RENAME libgpr.so.9 COMPONENT mindspore)
+    install(FILES ${grpc_LIBPATH}/libupb.so.9.0.0
+      DESTINATION ${INSTALL_LIB_DIR} RENAME libupb.so.9 COMPONENT mindspore)
+    install(FILES ${grpc_LIBPATH}/libaddress_sorting.so.9.0.0
+      DESTINATION ${INSTALL_LIB_DIR} RENAME libaddress_sorting.so.9 COMPONENT mindspore)
+endif()
+
 if(CMAKE_SYSTEM_NAME MATCHES "Windows")
     get_filename_component(CXX_DIR ${CMAKE_CXX_COMPILER} PATH)
     file(GLOB CXX_LIB_LIST ${CXX_DIR}/*.dll)

mindspore/ccsrc/CMakeLists.txt

@@ -241,7 +241,7 @@ target_link_libraries(mindspore mindspore_core)
 
 if(ENABLE_DEBUGGER)
     # debugger: link grpc
-    target_link_libraries(proto_input mindspore::grpc++)
+    target_link_libraries(proto_input PUBLIC -Wl,--no-as-needed mindspore::grpc++)
 endif()
 
 target_link_libraries(mindspore securec mindspore::flatbuffers)

mindspore/ccsrc/minddata/dataset/CMakeLists.txt

@@ -308,7 +308,7 @@ else()
 endif()
 
 if(MS_BUILD_GRPC)
-    target_link_libraries(_c_dataengine PRIVATE mindspore::grpc++)
+    target_link_libraries(_c_dataengine PRIVATE -Wl,--no-as-needed mindspore::grpc++)
 endif()
 
 set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH})

mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt

@@ -75,12 +75,13 @@ if(ENABLE_CACHE)
         _c_mindrecord
         mindspore
         mindspore::protobuf
-        mindspore::grpc++
         mindspore_gvar
         ${CUDNN_LIBRARY_PATH}
         ${PYTHON_LIBRARIES}
         ${SECUREC_LIBRARY}
-        pthread)
+        pthread
+        -Wl,--no-as-needed
+        mindspore::grpc++)
   else()
     target_link_libraries(cache_server
         engine-cache-server
@@ -88,11 +89,12 @@ if(ENABLE_CACHE)
         _c_mindrecord
         mindspore
         mindspore::protobuf
-        mindspore::grpc++
         mindspore_gvar
         ${PYTHON_LIBRARIES}
         ${SECUREC_LIBRARY}
-        pthread)
+        pthread
+        -Wl,--no-as-needed
+        mindspore::grpc++)
   endif()
 
   if(USE_GLOG)

third_party/patch/grpc/grpc.patch001

@@ -0,0 +1,12 @@
+diff -Npur grpc/CMakeLists.txt grpc-patch/CMakeLists.txt
+--- grpc/CMakeLists.txt	2020-02-27 03:12:33.000000000 +0800
++++ grpc-patch/CMakeLists.txt	2021-04-07 21:27:12.317207600 +0800
+@@ -12992,7 +12992,7 @@ if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_GRP
+ add_executable(grpc_cpp_plugin
+   src/compiler/cpp_plugin.cc
+ )
+-
++set_target_properties(grpc_cpp_plugin PROPERTIES INSTALL_RPATH $ORIGIN/../lib)
+ target_include_directories(grpc_cpp_plugin
+   PRIVATE
+     ${CMAKE_CURRENT_SOURCE_DIR}

third_party/patch/openssl-stub/openssl-stub.patch001

@@ -1,39 +0,0 @@
-diff -Npur openssl-OpenSSL_1_1_0l/Makefile.shared openssl-OpenSSL_1_1_0l-patch/Makefile.shared
---- openssl-OpenSSL_1_1_0l/Makefile.shared	2019-09-10 21:16:54.000000000 +0800
-+++ openssl-OpenSSL_1_1_0l-patch/Makefile.shared	2021-04-01 09:26:06.018179700 +0800
-@@ -154,7 +154,7 @@ LINK_SO_SHLIB_UNPACKED=	\
- DETECT_GNU_LD=($(CC) -Wl,-V /dev/null 2>&1 | grep '^GNU ld' )>/dev/null
- 
- DO_GNU_SO_COMMON=\
--	SHAREDFLAGS='$(CFLAGS) $(SHARED_LDFLAGS) -shared -Wl,-Bsymbolic -Wl,-soname=$(SHLIBNAME_FULL)'
-+	SHAREDFLAGS='$(CFLAGS) $(SHARED_LDFLAGS) -shared -Wl,-Bsymbolic -Wl,-soname=$(SHLIBNAME)'
- DO_GNU_DSO=\
- 	$(DO_GNU_SO_COMMON)
- DO_GNU_SO=\
-@@ -181,7 +181,7 @@ link_app.gnu:
- link_shlib.linux-shared:
- 	@$(PERL) $(SRCDIR)/util/mkdef.pl $(LIBNAME) linux >$(LIBNAME).map; \
- 	$(DO_GNU_SO); \
--	ALLSYMSFLAGS='-Wl,--whole-archive,--version-script=$(LIBNAME).map'; \
-+	ALLSYMSFLAGS='-Wl,--whole-archive'; \
- 	$(LINK_SO_SHLIB)
- 
- link_dso.bsd:
-@@ -380,7 +380,7 @@ link_dso.irix:
- 	else \
- 		ALLSYMSFLAGS=''; \
- 		NOALLSYMSFLAGS=''; \
--		SHAREDFLAGS='$(CFLAGS) $(SHARED_LDFLAGS) -shared -Wl,-soname,$(SHLIBNAME_FULL),-B,symbolic'; \
-+		SHAREDFLAGS='$(CFLAGS) $(SHARED_LDFLAGS) -shared -Wl,-soname,$(SHLIBNAME),-B,symbolic'; \
- 	fi; \
- 	$(LINK_SO_DSO)
- link_shlib.irix:
-@@ -391,7 +391,7 @@ link_shlib.irix:
- 		($(CC) -v 2>&1 | grep gcc) > /dev/null && MINUSWL='-Wl,'; \
- 		ALLSYMSFLAGS="$${MINUSWL}-all"; \
- 		NOALLSYMSFLAGS="$${MINUSWL}-none"; \
--		SHAREDFLAGS='$(CFLAGS) $(SHARED_LDFLAGS) -shared -Wl,-soname,$(SHLIBNAME_FULL),-B,symbolic'; \
-+		SHAREDFLAGS='$(CFLAGS) $(SHARED_LDFLAGS) -shared -Wl,-soname,$(SHLIBNAME),-B,symbolic'; \
- 	fi; \
- 	$(LINK_SO_SHLIB)
- link_app.irix: