From b1a311c44d9554a1710d26c78f487f9786dd1934 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 28 Jun 2017 21:48:40 +0800 Subject: [PATCH 01/11] Add pb_cc_library in generic.cmake Fix #2567 --- cmake/external/protobuf.cmake | 8 ++++++-- cmake/generic.cmake | 33 +++++++++++++++++++++++++++++++++ paddle/api/CMakeLists.txt | 2 +- paddle/capi/CMakeLists.txt | 2 +- paddle/function/CMakeLists.txt | 2 +- paddle/gserver/CMakeLists.txt | 2 +- paddle/math/CMakeLists.txt | 2 +- paddle/optimizer/CMakeLists.txt | 2 +- paddle/parameter/CMakeLists.txt | 2 +- paddle/pserver/CMakeLists.txt | 4 ++-- paddle/testing/CMakeLists.txt | 4 ++-- paddle/trainer/CMakeLists.txt | 2 +- paddle/utils/CMakeLists.txt | 2 +- proto/CMakeLists.txt | 16 +--------------- 14 files changed, 53 insertions(+), 30 deletions(-) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index d43badc1da..891fb29118 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -39,12 +39,16 @@ macro(PROMPT_PROTOBUF_LIB) ADD_LIBRARY(protobuf_lite ${protobuf_LIBTYPE} IMPORTED GLOBAL) SET_PROPERTY(TARGET protobuf_lite PROPERTY IMPORTED_LOCATION ${PROTOBUF_LITE_LIBRARY}) - ADD_LIBRARY(protoc ${protobuf_LIBTYPE} IMPORTED GLOBAL) - SET_PROPERTY(TARGET protoc PROPERTY IMPORTED_LOCATION ${PROTOC_LIBRARY}) + ADD_LIBRARY(libprotoc ${protobuf_LIBTYPE} IMPORTED GLOBAL) + SET_PROPERTY(TARGET libprotoc PROPERTY IMPORTED_LOCATION ${PROTOC_LIBRARY}) + + ADD_EXECUTABLE(protoc IMPORTED GLOBAL) + SET_PROPERTY(TARGET protoc PROPERTY IMPORTED_LOCATION ${PROTOBUF_PROTOC_EXECUTABLE}) FOREACH(dep ${protobuf_DEPS}) ADD_DEPENDENCIES(protobuf ${dep}) ADD_DEPENDENCIES(protobuf_lite ${dep}) + ADD_DEPENDENCIES(libprotoc ${dep}) ADD_DEPENDENCIES(protoc ${dep}) ENDFOREACH() diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 11c1f677ae..0370ab31f3 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -331,3 +331,36 @@ function(go_test TARGET_NAME) add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS}) add_test(${TARGET_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}) endfunction(go_test) + +function(pb_cc_library TARGET_NAME) + set(oneValueArgs "") + set(multiValueArgs SRCS) + cmake_parse_arguments(pb_cc_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(proto_srcs) + set(proto_hdrs) + foreach(FIL ${pb_cc_library_SRCS}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH) + get_filename_component(FIL_DIR ${FIL} DIRECTORY) + if(FIL_DIR) + set(FIL_WE "${FIL_DIR}/${FIL_WE}") + endif() + endif() + + list(APPEND proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") + list(APPEND proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS "--cpp_out=${DLL_EXPORT_DECL}${CMAKE_CURRENT_BINARY_DIR}" "-I" ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL} + DEPENDS ${ABS_FIL} protoc + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + set_source_files_properties(${proto_srcs} ${proto_hdrs} PROPERTIES GENERATED TRUE) + include_directories(${CMAKE_CURRENT_BINARY_DIR}) + cc_library(${TARGET_NAME} SRCS ${proto_srcs}) +endfunction() \ No newline at end of file diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index f2315e31cc..39d8aa075b 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -16,7 +16,7 @@ set(API_HEADER Internal.h) add_library(paddle_api STATIC ${API_SOURCES}) -add_dependencies(paddle_api gen_proto_cpp paddle_trainer_lib) +add_dependencies(paddle_api paddle_proto paddle_trainer_lib) INCLUDE(${SWIG_USE_FILE}) INCLUDE_DIRECTORIES(${PROJ_ROOT}/paddle) diff --git a/paddle/capi/CMakeLists.txt b/paddle/capi/CMakeLists.txt index 206f512563..11022d1754 100644 --- a/paddle/capi/CMakeLists.txt +++ b/paddle/capi/CMakeLists.txt @@ -26,7 +26,7 @@ target_include_directories(paddle_capi PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) add_style_check_target(paddle_capi ${CAPI_SOURCES} ${CAPI_HEADER} ${CAPI_PRIVATE_HEADER}) -add_dependencies(paddle_capi gen_proto_cpp) +add_dependencies(paddle_capi paddle_proto) # combine all paddle static libraries together, into libpaddle_capi_whole.a diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index 5e170714cf..1c39ced3c9 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -12,7 +12,7 @@ endif() add_library(paddle_function STATIC ${cpp_files} ${cu_objs}) add_dependencies(paddle_function ${external_project_dependencies}) -add_dependencies(paddle_function gen_proto_cpp) +add_dependencies(paddle_function paddle_proto) if(WITH_TESTING) if(WITH_GPU) diff --git a/paddle/gserver/CMakeLists.txt b/paddle/gserver/CMakeLists.txt index 93a6a99848..3bd583773a 100644 --- a/paddle/gserver/CMakeLists.txt +++ b/paddle/gserver/CMakeLists.txt @@ -58,7 +58,7 @@ endif() add_style_check_target(paddle_gserver ${GSERVER_SOURCES}) add_style_check_target(paddle_gserver ${GSERVER_HEADER}) -add_dependencies(paddle_gserver gen_proto_cpp) +add_dependencies(paddle_gserver paddle_proto) if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/paddle/math/CMakeLists.txt b/paddle/math/CMakeLists.txt index f5657c4690..326cdb156c 100644 --- a/paddle/math/CMakeLists.txt +++ b/paddle/math/CMakeLists.txt @@ -33,7 +33,7 @@ endif() add_style_check_target(paddle_math ${MATH_SOURCES}) add_style_check_target(paddle_math ${MATH_HEADERS}) -add_dependencies(paddle_math gen_proto_cpp) # depends +add_dependencies(paddle_math paddle_proto) # depends if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/paddle/optimizer/CMakeLists.txt b/paddle/optimizer/CMakeLists.txt index 4536f62ec7..bf878baaf0 100644 --- a/paddle/optimizer/CMakeLists.txt +++ b/paddle/optimizer/CMakeLists.txt @@ -10,7 +10,7 @@ set(OPITMIZER_SRCS ) add_library(paddle_optimizer STATIC ${OPITMIZER_SRCS}) -add_dependencies(paddle_optimizer gen_proto_cpp) +add_dependencies(paddle_optimizer paddle_proto) if(WITH_TESTING) add_simple_unittest(serialization_test) diff --git a/paddle/parameter/CMakeLists.txt b/paddle/parameter/CMakeLists.txt index a35e46997f..a9e344afdc 100644 --- a/paddle/parameter/CMakeLists.txt +++ b/paddle/parameter/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(paddle_parameter STATIC ${PARAMETERS_SOURCES}) add_style_check_target(paddle_parameter ${PARAMETERS_SOURCES}) add_style_check_target(paddle_parameter ${PARAMETERS_HEADERS}) -add_dependencies(paddle_parameter gen_proto_cpp) +add_dependencies(paddle_parameter paddle_proto) if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/paddle/pserver/CMakeLists.txt b/paddle/pserver/CMakeLists.txt index b7f85ea1a6..92dd286f04 100644 --- a/paddle/pserver/CMakeLists.txt +++ b/paddle/pserver/CMakeLists.txt @@ -17,7 +17,7 @@ add_library(paddle_network STATIC add_style_check_target(paddle_network ${NETWORK_SOURCES}) add_style_check_target(paddle_network ${NETWORK_HEADERS}) -add_dependencies(paddle_network gen_proto_cpp) +add_dependencies(paddle_network paddle_proto) ################### paddle_pserver ###################### set(PSERVER_SOURCES @@ -40,7 +40,7 @@ add_library(paddle_pserver STATIC add_style_check_target(paddle_pserver ${PSERVER_SOURCES}) add_style_check_target(paddle_pserver ${PSERVER_HEADERS}) -add_dependencies(paddle_pserver gen_proto_cpp) +add_dependencies(paddle_pserver paddle_proto) set(PSERVER_MAIN_SOURCES ParameterServer2Main.cpp) diff --git a/paddle/testing/CMakeLists.txt b/paddle/testing/CMakeLists.txt index c47add04b0..4aa6eae681 100644 --- a/paddle/testing/CMakeLists.txt +++ b/paddle/testing/CMakeLists.txt @@ -2,7 +2,7 @@ if(WITH_TESTING) add_library(paddle_test_main STATIC TestMain.cpp) - add_dependencies(paddle_test_main gen_proto_cpp) + add_dependencies(paddle_test_main paddle_proto) add_library(paddle_test_util STATIC TestUtil.cpp) - add_dependencies(paddle_test_util gen_proto_cpp) + add_dependencies(paddle_test_util paddle_proto) endif() diff --git a/paddle/trainer/CMakeLists.txt b/paddle/trainer/CMakeLists.txt index f34d53ae99..b8f03fa7e7 100644 --- a/paddle/trainer/CMakeLists.txt +++ b/paddle/trainer/CMakeLists.txt @@ -41,7 +41,7 @@ add_style_check_target(paddle_trainer_lib add_style_check_target(paddle_trainer_lib ${TRAINER_HEADERS}) add_dependencies(paddle_trainer_lib - gen_proto_cpp) + paddle_proto) macro(add_paddle_exe TARGET_NAME) add_executable(${TARGET_NAME} ${ARGN}) diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index af59951752..f5c399256a 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -17,7 +17,7 @@ add_library(paddle_utils STATIC add_style_check_target(paddle_utils ${UTIL_HEADERS}) add_style_check_target(paddle_utils ${UTIL_SOURCES} ${UTIL_ARCH_SOURCES}) -add_dependencies(paddle_utils gen_proto_cpp) +add_dependencies(paddle_utils paddle_proto) if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index c942620990..948d7db6b2 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -13,18 +13,6 @@ set(PROTO_GEN_PY) foreach(filename ${proto_filenames}) get_filename_component(base_filename ${filename} NAME_WE) - set(CUR_PROTO_GEN - ${CMAKE_CURRENT_BINARY_DIR}/${base_filename}.pb.h - ${CMAKE_CURRENT_BINARY_DIR}/${base_filename}.pb.cc) - set(PROTO_GEN - ${PROTO_GEN} - ${CUR_PROTO_GEN}) - add_custom_command(OUTPUT ${CUR_PROTO_GEN} - COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE} - --cpp_out ${CMAKE_CURRENT_BINARY_DIR} - --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} - DEPENDS ${filename} ${external_project_dependencies}) - set(CUR_PROTO_GEN_PY ${PROJ_ROOT}/paddle/python/paddle/proto/${base_filename}_pb2.py) set(PROTO_GEN_PY @@ -36,8 +24,6 @@ foreach(filename ${proto_filenames}) DEPENDS ${filename} ${external_project_dependencies}) endforeach() -add_custom_target(gen_proto_cpp ALL DEPENDS ${PROTO_GEN}) add_custom_target(gen_proto_py ALL DEPENDS ${PROTO_GEN_PY}) -add_library(paddle_proto STATIC ${PROTO_GEN}) -target_include_directories(paddle_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) +pb_cc_library(paddle_proto SRCS ${proto_filenames}) From 30b75a51035dee978225f5e5eff2c0d4b9c09aec Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 28 Jun 2017 22:05:04 +0800 Subject: [PATCH 02/11] Also add pb_py_library --- cmake/generic.cmake | 31 ++++++++++++++++++++++++------- proto/CMakeLists.txt | 30 ++---------------------------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 0370ab31f3..14b6909829 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -341,13 +341,6 @@ function(pb_cc_library TARGET_NAME) foreach(FIL ${pb_cc_library_SRCS}) get_filename_component(ABS_FIL ${FIL} ABSOLUTE) get_filename_component(FIL_WE ${FIL} NAME_WE) - if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH) - get_filename_component(FIL_DIR ${FIL} DIRECTORY) - if(FIL_DIR) - set(FIL_WE "${FIL_DIR}/${FIL_WE}") - endif() - endif() - list(APPEND proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") list(APPEND proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") @@ -363,4 +356,28 @@ function(pb_cc_library TARGET_NAME) set_source_files_properties(${proto_srcs} ${proto_hdrs} PROPERTIES GENERATED TRUE) include_directories(${CMAKE_CURRENT_BINARY_DIR}) cc_library(${TARGET_NAME} SRCS ${proto_srcs}) +endfunction() + +function(pb_py_library TARGET_NAME) + set(oneValueArgs TARGET_DIR) + set(multiValueArgs SRCS) + cmake_parse_arguments(pb_py_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if (NOT ${pb_py_library_TARGET_DIR}) + set(pb_py_library_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}) + endif() + + set(py_srcs) + foreach(FIL ${pb_py_library_SRCS}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + set(cur_py_src ${pb_py_library_TARGET_DIR}/${FIL_WE}_pb2.py) + list(APPEND py_srcs "${cur_py_src}") + add_custom_command(OUTPUT ${cur_py_src} + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS "--python_out=${pb_py_library_TARGET_DIR}" "-I" ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL} + DEPENDS ${ABS_FIL} protoc + COMMENT "Running Python protocol buffer compiler on ${FIL}") + endforeach() + + add_custom_target(${TARGET_NAME} ALL DEPENDS ${py_srcs}) endfunction() \ No newline at end of file diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 948d7db6b2..9b29d43d73 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -1,29 +1,3 @@ -set(proto_filenames - DataConfig.proto - DataFormat.proto - ModelConfig.proto - ParameterConfig.proto - ParameterService.proto - TrainerConfig.proto - OptimizerConfig.proto - ParameterServerConfig.proto) - -set(PROTO_GEN) -set(PROTO_GEN_PY) - -foreach(filename ${proto_filenames}) - get_filename_component(base_filename ${filename} NAME_WE) - set(CUR_PROTO_GEN_PY - ${PROJ_ROOT}/paddle/python/paddle/proto/${base_filename}_pb2.py) - set(PROTO_GEN_PY - ${CUR_PROTO_GEN_PY} - ${PROTO_GEN_PY}) - add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY} - COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto - --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} - DEPENDS ${filename} ${external_project_dependencies}) -endforeach() - -add_custom_target(gen_proto_py ALL DEPENDS ${PROTO_GEN_PY}) - +file(GLOB proto_filenames . *.proto) pb_cc_library(paddle_proto SRCS ${proto_filenames}) +pb_py_library(gen_proto_py SRCS ${proto_filenames} TARGET_DIR ${CMAKE_CURRENT_SOURCE_DIR}) From 64b78b1656bd023e916447e7ea6c08de3d5c1f88 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 28 Jun 2017 22:27:50 +0800 Subject: [PATCH 03/11] Fix TravisCI --- cmake/generic.cmake | 3 +-- doc/CMakeLists.txt | 7 ------- paddle/gserver/CMakeLists.txt | 2 +- paddle/math/CMakeLists.txt | 2 +- paddle/optimizer/CMakeLists.txt | 2 +- paddle/parameter/CMakeLists.txt | 2 +- paddle/pserver/CMakeLists.txt | 2 +- paddle/trainer/CMakeLists.txt | 3 ++- paddle/utils/CMakeLists.txt | 2 +- proto/CMakeLists.txt | 2 +- 10 files changed, 10 insertions(+), 17 deletions(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 14b6909829..24a07c0a24 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -362,10 +362,9 @@ function(pb_py_library TARGET_NAME) set(oneValueArgs TARGET_DIR) set(multiValueArgs SRCS) cmake_parse_arguments(pb_py_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (NOT ${pb_py_library_TARGET_DIR}) + if (NOT pb_py_library_TARGET_DIR) set(pb_py_library_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}) endif() - set(py_srcs) foreach(FIL ${pb_py_library_SRCS}) get_filename_component(ABS_FIL ${FIL} ABSOLUTE) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 6fa42fd0c7..94dd3457fb 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -27,10 +27,6 @@ sphinx_add_target(paddle_docs ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_EN}) -add_dependencies(paddle_docs - gen_proto_py) - - # configured documentation tools and intermediate build results set(BINARY_BUILD_DIR_CN "${CMAKE_CURRENT_BINARY_DIR}/cn/_build") @@ -51,6 +47,3 @@ sphinx_add_target(paddle_docs_cn ${SPHINX_CACHE_DIR_CN} ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_CN}) - -add_dependencies(paddle_docs_cn - gen_proto_py) diff --git a/paddle/gserver/CMakeLists.txt b/paddle/gserver/CMakeLists.txt index 3bd583773a..0012636b8f 100644 --- a/paddle/gserver/CMakeLists.txt +++ b/paddle/gserver/CMakeLists.txt @@ -58,7 +58,7 @@ endif() add_style_check_target(paddle_gserver ${GSERVER_SOURCES}) add_style_check_target(paddle_gserver ${GSERVER_HEADER}) -add_dependencies(paddle_gserver paddle_proto) +add_dependencies(paddle_gserver paddle_proto ${external_project_dependencies}) if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/paddle/math/CMakeLists.txt b/paddle/math/CMakeLists.txt index 326cdb156c..9981de6160 100644 --- a/paddle/math/CMakeLists.txt +++ b/paddle/math/CMakeLists.txt @@ -33,7 +33,7 @@ endif() add_style_check_target(paddle_math ${MATH_SOURCES}) add_style_check_target(paddle_math ${MATH_HEADERS}) -add_dependencies(paddle_math paddle_proto) # depends +add_dependencies(paddle_math paddle_proto ${external_project_dependencies}) # depends if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/paddle/optimizer/CMakeLists.txt b/paddle/optimizer/CMakeLists.txt index bf878baaf0..9996d01d18 100644 --- a/paddle/optimizer/CMakeLists.txt +++ b/paddle/optimizer/CMakeLists.txt @@ -10,7 +10,7 @@ set(OPITMIZER_SRCS ) add_library(paddle_optimizer STATIC ${OPITMIZER_SRCS}) -add_dependencies(paddle_optimizer paddle_proto) +add_dependencies(paddle_optimizer paddle_proto ${external_project_dependencies}) if(WITH_TESTING) add_simple_unittest(serialization_test) diff --git a/paddle/parameter/CMakeLists.txt b/paddle/parameter/CMakeLists.txt index a9e344afdc..d2ae1c16c6 100644 --- a/paddle/parameter/CMakeLists.txt +++ b/paddle/parameter/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(paddle_parameter STATIC ${PARAMETERS_SOURCES}) add_style_check_target(paddle_parameter ${PARAMETERS_SOURCES}) add_style_check_target(paddle_parameter ${PARAMETERS_HEADERS}) -add_dependencies(paddle_parameter paddle_proto) +add_dependencies(paddle_parameter paddle_proto ${external_project_dependencies}) if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/paddle/pserver/CMakeLists.txt b/paddle/pserver/CMakeLists.txt index 92dd286f04..f2e0b4b76b 100644 --- a/paddle/pserver/CMakeLists.txt +++ b/paddle/pserver/CMakeLists.txt @@ -40,7 +40,7 @@ add_library(paddle_pserver STATIC add_style_check_target(paddle_pserver ${PSERVER_SOURCES}) add_style_check_target(paddle_pserver ${PSERVER_HEADERS}) -add_dependencies(paddle_pserver paddle_proto) +add_dependencies(paddle_pserver paddle_proto ${external_project_dependencies}) set(PSERVER_MAIN_SOURCES ParameterServer2Main.cpp) diff --git a/paddle/trainer/CMakeLists.txt b/paddle/trainer/CMakeLists.txt index b8f03fa7e7..6414c39956 100644 --- a/paddle/trainer/CMakeLists.txt +++ b/paddle/trainer/CMakeLists.txt @@ -41,7 +41,8 @@ add_style_check_target(paddle_trainer_lib add_style_check_target(paddle_trainer_lib ${TRAINER_HEADERS}) add_dependencies(paddle_trainer_lib - paddle_proto) + paddle_proto + ${external_project_dependencies}) macro(add_paddle_exe TARGET_NAME) add_executable(${TARGET_NAME} ${ARGN}) diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index f5c399256a..7a4977935e 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -17,7 +17,7 @@ add_library(paddle_utils STATIC add_style_check_target(paddle_utils ${UTIL_HEADERS}) add_style_check_target(paddle_utils ${UTIL_SOURCES} ${UTIL_ARCH_SOURCES}) -add_dependencies(paddle_utils paddle_proto) +add_dependencies(paddle_utils paddle_proto ${external_project_dependencies}) if(WITH_TESTING) add_subdirectory(tests) endif() diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 9b29d43d73..4402f2c899 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -1,3 +1,3 @@ file(GLOB proto_filenames . *.proto) pb_cc_library(paddle_proto SRCS ${proto_filenames}) -pb_py_library(gen_proto_py SRCS ${proto_filenames} TARGET_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +pb_py_library(gen_proto_py SRCS ${proto_filenames} TARGET_DIR ${PROJ_ROOT}/python/paddle/proto) From 456f9cc89f6ec5a80e08522ff1fafb8f20a21fa6 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 12:11:03 +0800 Subject: [PATCH 04/11] Remove Python protobuf function --- cmake/generic.cmake | 23 ----------------------- proto/CMakeLists.txt | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 24a07c0a24..8736d30059 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -357,26 +357,3 @@ function(pb_cc_library TARGET_NAME) include_directories(${CMAKE_CURRENT_BINARY_DIR}) cc_library(${TARGET_NAME} SRCS ${proto_srcs}) endfunction() - -function(pb_py_library TARGET_NAME) - set(oneValueArgs TARGET_DIR) - set(multiValueArgs SRCS) - cmake_parse_arguments(pb_py_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (NOT pb_py_library_TARGET_DIR) - set(pb_py_library_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}) - endif() - set(py_srcs) - foreach(FIL ${pb_py_library_SRCS}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(FIL_WE ${FIL} NAME_WE) - set(cur_py_src ${pb_py_library_TARGET_DIR}/${FIL_WE}_pb2.py) - list(APPEND py_srcs "${cur_py_src}") - add_custom_command(OUTPUT ${cur_py_src} - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS "--python_out=${pb_py_library_TARGET_DIR}" "-I" ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL} - DEPENDS ${ABS_FIL} protoc - COMMENT "Running Python protocol buffer compiler on ${FIL}") - endforeach() - - add_custom_target(${TARGET_NAME} ALL DEPENDS ${py_srcs}) -endfunction() \ No newline at end of file diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 4402f2c899..1cf39d6944 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -1,3 +1,20 @@ file(GLOB proto_filenames . *.proto) pb_cc_library(paddle_proto SRCS ${proto_filenames}) -pb_py_library(gen_proto_py SRCS ${proto_filenames} TARGET_DIR ${PROJ_ROOT}/python/paddle/proto) + +set(PROTO_GEN) +set(PROTO_GEN_PY) + +foreach(filename ${proto_filenames}) + get_filename_component(base_filename ${filename} NAME_WE) + set(CUR_PROTO_GEN_PY + ${PROJ_ROOT}/paddle/python/paddle/proto/${base_filename}_pb2.py) + set(PROTO_GEN_PY + ${CUR_PROTO_GEN_PY} + ${PROTO_GEN_PY}) + add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY} + COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto + --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} + DEPENDS ${filename} ${external_project_dependencies}) +endforeach() + +add_custom_target(gen_proto_py ALL DEPENDS ${PROTO_GEN_PY}) From 32d6587242e5a2e97fe5b9e675273fa96fd99c5a Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 12:28:48 +0800 Subject: [PATCH 05/11] Use protobuf_generate_cpp --- cmake/external/protobuf.cmake | 7 +++++++ cmake/generic.cmake | 17 +---------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 891fb29118..2f267adc20 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -13,6 +13,10 @@ # limitations under the License. INCLUDE(ExternalProject) +# Always invoke `FIND_PACKAGE(Protobuf)` for importing function protobuf_generate_cpp +FIND_PACKAGE(Protobuf QUIET) +SET(PROTOBUF_FOUND "OFF") + # Print and set the protobuf library information, # finish this cmake process and exit from this file. @@ -44,6 +48,9 @@ macro(PROMPT_PROTOBUF_LIB) ADD_EXECUTABLE(protoc IMPORTED GLOBAL) SET_PROPERTY(TARGET protoc PROPERTY IMPORTED_LOCATION ${PROTOBUF_PROTOC_EXECUTABLE}) + # FIND_Protobuf.cmake uses `Protobuf_PROTOC_EXECUTABLE`. + # make `protobuf_generate_cpp` happy. + SET(Protobuf_PROTOC_EXECUTABLE ${PROTOBUF_PROTOC_EXECUTABLE}) FOREACH(dep ${protobuf_DEPS}) ADD_DEPENDENCIES(protobuf ${dep}) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 8736d30059..cdf917a1e9 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -338,22 +338,7 @@ function(pb_cc_library TARGET_NAME) cmake_parse_arguments(pb_cc_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) set(proto_srcs) set(proto_hdrs) - foreach(FIL ${pb_cc_library_SRCS}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(FIL_WE ${FIL} NAME_WE) - list(APPEND proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") - list(APPEND proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS "--cpp_out=${DLL_EXPORT_DECL}${CMAKE_CURRENT_BINARY_DIR}" "-I" ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL} - DEPENDS ${ABS_FIL} protoc - COMMENT "Running C++ protocol buffer compiler on ${FIL}" - VERBATIM ) - endforeach() - set_source_files_properties(${proto_srcs} ${proto_hdrs} PROPERTIES GENERATED TRUE) + protobuf_generate_cpp(proto_srcs proto_hdrs ${pb_cc_library_SRCS}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) cc_library(${TARGET_NAME} SRCS ${proto_srcs}) endfunction() From b88ca542baa4bb5e5822912dceeb4ceb2080c660 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 12:43:51 +0800 Subject: [PATCH 06/11] Rename pb_cc_library -> proto_library --- cmake/generic.cmake | 6 +++--- proto/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index cdf917a1e9..779f627115 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -332,13 +332,13 @@ function(go_test TARGET_NAME) add_test(${TARGET_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}) endfunction(go_test) -function(pb_cc_library TARGET_NAME) +function(proto_library TARGET_NAME) set(oneValueArgs "") set(multiValueArgs SRCS) - cmake_parse_arguments(pb_cc_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(proto_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) set(proto_srcs) set(proto_hdrs) - protobuf_generate_cpp(proto_srcs proto_hdrs ${pb_cc_library_SRCS}) + protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) cc_library(${TARGET_NAME} SRCS ${proto_srcs}) endfunction() diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 1cf39d6944..436bea53e5 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -1,5 +1,5 @@ file(GLOB proto_filenames . *.proto) -pb_cc_library(paddle_proto SRCS ${proto_filenames}) +proto_library(paddle_proto SRCS ${proto_filenames}) set(PROTO_GEN) set(PROTO_GEN_PY) From 4a4ec31e0d2f07b5e29acfd1b5b0b62d40f7ab91 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 12:53:10 +0800 Subject: [PATCH 07/11] Fix TravisCI --- paddle/cuda/CMakeLists.txt | 2 +- proto/CMakeLists.txt | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt index f9061e96de..73ffa690d9 100755 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -83,7 +83,7 @@ else() ${CUDA_CXX_SOURCES}) endif() -add_dependencies(paddle_cuda ${external_project_dependencies}) +add_dependencies(paddle_cuda paddle_proto ${external_project_dependencies}) add_style_check_target(paddle_cuda ${CUDA_SOURCES} diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 436bea53e5..70dd4d674c 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -5,16 +5,18 @@ set(PROTO_GEN) set(PROTO_GEN_PY) foreach(filename ${proto_filenames}) - get_filename_component(base_filename ${filename} NAME_WE) + get_filename_component(ABS_FIL ${filename} ABSOLUTE) + get_filename_component(FIL_WE ${filename} NAME_WE) set(CUR_PROTO_GEN_PY - ${PROJ_ROOT}/paddle/python/paddle/proto/${base_filename}_pb2.py) + ${PROJ_ROOT}/paddle/python/paddle/proto/${FIL_WE}_pb2.py) set(PROTO_GEN_PY ${CUR_PROTO_GEN_PY} ${PROTO_GEN_PY}) add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY} - COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto - --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} - DEPENDS ${filename} ${external_project_dependencies}) + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS "--python_out=${PROJ_ROOT}/python/paddle/proto" + "-I" ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${external_project_dependencies}) endforeach() add_custom_target(gen_proto_py ALL DEPENDS ${PROTO_GEN_PY}) From 2d3c186a96178132784ecb9ba81403b67c3b7f67 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 14:14:55 +0800 Subject: [PATCH 08/11] Follow comments --- cmake/generic.cmake | 6 ++++-- proto/CMakeLists.txt | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 779f627115..8117dbc53e 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -87,6 +87,9 @@ # go_library(example SHARED) # +# including binary directory for generated headers. +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + if(NOT APPLE) find_package(Threads REQUIRED) link_libraries(${CMAKE_THREAD_LIBS_INIT}) @@ -339,6 +342,5 @@ function(proto_library TARGET_NAME) set(proto_srcs) set(proto_hdrs) protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS}) - include_directories(${CMAKE_CURRENT_BINARY_DIR}) - cc_library(${TARGET_NAME} SRCS ${proto_srcs}) + cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS protobuf) endfunction() diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 70dd4d674c..18584cafe7 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -1,4 +1,5 @@ file(GLOB proto_filenames . *.proto) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) proto_library(paddle_proto SRCS ${proto_filenames}) set(PROTO_GEN) From 3d44fd5bf38b8cd74fccc17081972b5a9a0eaa2e Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 14:53:29 +0800 Subject: [PATCH 09/11] Follow yiqun's comments --- cmake/generic.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 8117dbc53e..61353a4a26 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -88,7 +88,7 @@ # # including binary directory for generated headers. -include_directories(${CMAKE_CURRENT_BINARY_DIR}) +include_directories(${CMAKE_BINARY_DIR}) if(NOT APPLE) find_package(Threads REQUIRED) From 52efb243b82b7b68868cfe5391eafb0376ddb839 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 15:06:46 +0800 Subject: [PATCH 10/11] Fix CI tests --- paddle/testing/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/testing/CMakeLists.txt b/paddle/testing/CMakeLists.txt index 4aa6eae681..4245df5ab7 100644 --- a/paddle/testing/CMakeLists.txt +++ b/paddle/testing/CMakeLists.txt @@ -2,7 +2,7 @@ if(WITH_TESTING) add_library(paddle_test_main STATIC TestMain.cpp) - add_dependencies(paddle_test_main paddle_proto) + add_dependencies(paddle_test_main paddle_proto ${external_project_dependencies}) add_library(paddle_test_util STATIC TestUtil.cpp) - add_dependencies(paddle_test_util paddle_proto) + add_dependencies(paddle_test_util paddle_proto ${external_project_dependencies}) endif() From a1e7284ad139e6578036b7d872563aa1a2642351 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Jun 2017 15:24:12 +0800 Subject: [PATCH 11/11] Fix TravisCI --- paddle/pserver/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/pserver/CMakeLists.txt b/paddle/pserver/CMakeLists.txt index f2e0b4b76b..2245c7d88c 100644 --- a/paddle/pserver/CMakeLists.txt +++ b/paddle/pserver/CMakeLists.txt @@ -17,7 +17,7 @@ add_library(paddle_network STATIC add_style_check_target(paddle_network ${NETWORK_SOURCES}) add_style_check_target(paddle_network ${NETWORK_HEADERS}) -add_dependencies(paddle_network paddle_proto) +add_dependencies(paddle_network paddle_proto ${external_project_dependencies}) ################### paddle_pserver ###################### set(PSERVER_SOURCES