diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b138576fc..4cd8eb12f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,8 +3,8 @@ hooks: - id: remove-crlf files: (?!.*third_party)^.*$ | (?!.*book)^.*$ -- repo: https://github.com/reyoung/mirrors-yapf.git - sha: v0.13.2 +- repo: https://github.com/PaddlePaddle/mirrors-yapf.git + sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37 hooks: - id: yapf files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ diff --git a/README.md b/README.md index bcc24b8412..fa16cc3cf2 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ before looking into the We provide [English](http://www.paddlepaddle.org/develop/doc/) and [Chinese](http://www.paddlepaddle.org/doc_cn/) documentation. -- [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) +- [Deep Learning 101](http://book.paddlepaddle.org/index.html) You might want to start from the this online interactive book that can run in Jupyter Notebook. diff --git a/cmake/cudnn.cmake b/cmake/cudnn.cmake index af9be86961..92dce20c69 100644 --- a/cmake/cudnn.cmake +++ b/cmake/cudnn.cmake @@ -11,11 +11,23 @@ find_path(CUDNN_INCLUDE_DIR cudnn.h get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH) +if(NOT ${CMAKE_HOST_SYSTEM_PROCESSOR}) + execute_process( + COMMAND uname -m COMMAND tr -d '\n' + OUTPUT_VARIABLE HOST_ARCH + RESULT_VARIABLE UNAME_RESULT) + if(${UNAME_RESULT}) + set(HOST_ARCH "x86_64") + endif(${UNAME_RESULT}) +else(NOT ${CMAKE_HOST_SYSTEM_PROCESSOR}) + set(HOST_ARCH ${CMAKE_HOST_SYSTEM_PROCESSOR}) +endif(NOT ${CMAKE_HOST_SYSTEM_PROCESSOR}) + list(APPEND CUDNN_CHECK_LIBRARY_DIRS ${CUDNN_ROOT} ${CUDNN_ROOT}/lib64 ${CUDNN_ROOT}/lib - ${CUDNN_ROOT}/lib/x86_64-linux-gnu + ${CUDNN_ROOT}/lib/${HOST_ARCH}-linux-gnu $ENV{CUDNN_ROOT} $ENV{CUDNN_ROOT}/lib64 $ENV{CUDNN_ROOT}/lib diff --git a/cmake/external/any.cmake b/cmake/external/any.cmake index 8116f235d5..62eea42692 100644 --- a/cmake/external/any.cmake +++ b/cmake/external/any.cmake @@ -18,3 +18,4 @@ ExternalProject_Add( ) add_definitions(-DANY_IMPL_ANY_CAST_MOVEABLE) +LIST(APPEND external_project_dependencies linb_any) \ No newline at end of file diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 0afb3ab9af..30027a336c 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -26,7 +26,7 @@ ENDIF(WIN32) INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) ExternalProject_Add( - gflags + extern_gflags ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/gflags/gflags.git" PREFIX ${GFLAGS_SOURCES_DIR} @@ -44,4 +44,8 @@ ExternalProject_Add( -DCMAKE_BUILD_TYPE:STRING=Release ) +ADD_LIBRARY(gflags STATIC IMPORTED) +SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES}) +ADD_DEPENDENCIES(gflags extern_gflags) + LIST(APPEND external_project_dependencies gflags) diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index 4a9e2ecc6b..fa9a509287 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -27,7 +27,7 @@ ENDIF(WIN32) INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR}) ExternalProject_Add( - glog + extern_glog ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS gflags GIT_REPOSITORY "https://github.com/google/glog.git" @@ -48,4 +48,8 @@ ExternalProject_Add( -DCMAKE_BUILD_TYPE:STRING=Release ) +ADD_LIBRARY(glog STATIC IMPORTED) +SET_PROPERTY(TARGET glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARIES}) +ADD_DEPENDENCIES(glog extern_glog) + LIST(APPEND external_project_dependencies glog) diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index 49c7d71443..386204dc37 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -35,7 +35,7 @@ IF(WITH_TESTING) ENDIF(WIN32) ExternalProject_Add( - gtest + extern_gtest ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/google/googletest.git" GIT_TAG "release-1.8.0" @@ -55,5 +55,14 @@ IF(WITH_TESTING) -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=Release ) - LIST(APPEND external_project_dependencies gtest) + + ADD_LIBRARY(gtest STATIC IMPORTED) + SET_PROPERTY(TARGET gtest PROPERTY IMPORTED_LOCATION ${GTEST_LIBRARIES}) + ADD_DEPENDENCIES(gtest extern_gtest) + + ADD_LIBRARY(gtest_main STATIC IMPORTED) + SET_PROPERTY(TARGET gtest_main PROPERTY IMPORTED_LOCATION ${GTEST_MAIN_LIBRARIES}) + ADD_DEPENDENCIES(gtest_main extern_gtest) + + LIST(APPEND external_project_dependencies gtest gtest_main) ENDIF(WITH_TESTING) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index b6bd24fe8a..cb67793cf9 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -37,11 +37,11 @@ IF(NOT ${CBLAS_FOUND}) SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER} TARGET=ARMV7 USE_THREAD=0 libs) ELSE() SET(OPENBLAS_COMMIT "v0.2.19") - SET(OPENBLAS_ARGS DYNAMIC_ARCH=1 libs) + SET(OPTIONAL_ARGS DYNAMIC_ARCH=1 libs NUM_THREADS=64) ENDIF() ExternalProject_Add( - openblas + extern_openblas ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY https://github.com/xianyi/OpenBLAS.git GIT_TAG ${OPENBLAS_COMMIT} @@ -53,8 +53,14 @@ IF(NOT ${CBLAS_FOUND}) UPDATE_COMMAND "" CONFIGURE_COMMAND "" ) - LIST(APPEND external_project_dependencies openblas) ENDIF(NOT ${CBLAS_FOUND}) MESSAGE(STATUS "BLAS library: ${CBLAS_LIBRARIES}") INCLUDE_DIRECTORIES(${CBLAS_INC_DIR}) + +ADD_LIBRARY(cblas STATIC IMPORTED) +SET_PROPERTY(TARGET cblas PROPERTY IMPORTED_LOCATION ${CBLAS_LIBRARIES}) +IF(NOT ${CBLAS_FOUND}) + ADD_DEPENDENCIES(cblas extern_openblas) + LIST(APPEND external_project_dependencies cblas) +ENDIF(NOT ${CBLAS_FOUND}) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index da46eaff50..7340394b1e 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -17,9 +17,14 @@ INCLUDE(ExternalProject) macro(PROMPT_PROTOBUF_LIB) MESSAGE(STATUS "Protobuf protoc executable: ${PROTOBUF_PROTOC_EXECUTABLE}") MESSAGE(STATUS "Protobuf library: ${PROTOBUF_LIBRARY}") + MESSAGE(STATUS "Protobuf version: ${PROTOBUF_VERSION}") INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) RETURN() endmacro() +macro(SET_PROTOBUF_VERSION) + EXEC_PROGRAM(${PROTOBUF_PROTOC_EXECUTABLE} ARGS --version OUTPUT_VARIABLE PROTOBUF_VERSION) + STRING(REGEX MATCH "[0-9]+.[0-9]+" PROTOBUF_VERSION "${PROTOBUF_VERSION}") +endmacro() set(PROTOBUF_ROOT "" CACHE PATH "Folder contains protobuf") if (NOT "${PROTOBUF_ROOT}" STREQUAL "") @@ -30,6 +35,7 @@ if (NOT "${PROTOBUF_ROOT}" STREQUAL "") find_program(PROTOBUF_PROTOC_EXECUTABLE protoc PATHS ${PROTOBUF_ROOT}/bin) if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOBUF_LITE_LIBRARY AND PROTOBUF_PROTOC_LIBRARY AND PROTOBUF_PROTOC_EXECUTABLE) message(STATUS "Using custom protobuf library in ${PROTOBUF_ROOT}.") + SET_PROTOBUF_VERSION() PROMPT_PROTOBUF_LIB() else() message(WARNING "Cannot find protobuf library in ${PROTOBUF_ROOT}.") @@ -100,8 +106,7 @@ IF(NOT CMAKE_CROSSCOMPILING) FIND_PACKAGE(Protobuf ${PROTOBUF_VERSION}) IF(PROTOBUF_FOUND) - EXEC_PROGRAM(${PROTOBUF_PROTOC_EXECUTABLE} ARGS --version OUTPUT_VARIABLE PROTOBUF_VERSION) - STRING(REGEX MATCH "[0-9]+.[0-9]+" PROTOBUF_VERSION "${PROTOBUF_VERSION}") + SET_PROTOBUF_VERSION() IF("${PROTOBUF_VERSION}" VERSION_LESS "3.1.0") SET(PROTOBUF_FOUND OFF) ENDIF() diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 293070c3cf..367d5b98c7 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -43,7 +43,7 @@ ELSE() ENDIF() ExternalProject_Add( - warpctc + extern_warpctc ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git" PREFIX ${WARPCTC_SOURCES_DIR} @@ -65,4 +65,8 @@ ExternalProject_Add( -DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR} ) +ADD_LIBRARY(warpctc STATIC IMPORTED) +SET_PROPERTY(TARGET warpctc PROPERTY IMPORTED_LOCATION ${WARPCTC_LIBRARIES}) +ADD_DEPENDENCIES(warpctc extern_warpctc) + LIST(APPEND external_project_dependencies warpctc) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index efc49b8fd3..052530608e 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -1,11 +1,11 @@ # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,7 @@ # To simplify the build process of PaddlePaddle, we defined couple of # fundamental abstractions, e.g., how to build library, binary and # test in C++, CUDA and Go. -# +# # ------------------------------------------- # C++ CUDA C++ Go # ------------------------------------------- @@ -29,6 +29,11 @@ # https://cmake.org/cmake/help/v3.0/module/CMakeParseArguments.html # +if(NOT APPLE) + find_package(Threads REQUIRED) + link_libraries(${CMAKE_THREAD_LIBS_INIT}) +endif(NOT APPLE) + # cc_library parses tensor.cc and figures out that target also depend on tensor.h. # cc_library(tensor # SRCS @@ -45,7 +50,9 @@ function(cc_library TARGET_NAME) else() add_library(${TARGET_NAME} STATIC ${cc_library_SRCS}) endif() - add_dependencies(${TARGET_NAME} ${cc_library_DEPS} ${external_project_dependencies}) + if (cc_library_DEPS) + add_dependencies(${TARGET_NAME} ${cc_library_DEPS}) + endif() endfunction(cc_library) # cc_binary parses tensor.cc and figures out that target also depend on tensor.h. @@ -58,8 +65,7 @@ function(cc_binary TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(cc_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${TARGET_NAME} ${cc_binary_SRCS}) - link_paddle_exe(${TARGET_NAME}) - if(cc_binary_DEPS) + if(cc_binary_DEPS) target_link_libraries(${TARGET_NAME} ${cc_binary_DEPS}) add_dependencies(${TARGET_NAME} ${cc_binary_DEPS}) endif() @@ -73,17 +79,16 @@ endfunction(cc_binary) # DEPS # tensor) function(cc_test TARGET_NAME) - set(options "") - set(oneValueArgs "") - set(multiValueArgs SRCS DEPS) - cmake_parse_arguments(cc_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - add_executable(${TARGET_NAME} ${cc_test_SRCS}) - link_paddle_test(${TARGET_NAME}) - if(cc_test_DEPS) - target_link_libraries(${TARGET_NAME} ${cc_test_DEPS}) - add_dependencies(${TARGET_NAME} ${cc_test_DEPS}) + if(WITH_TESTING) + set(options "") + set(oneValueArgs "") + set(multiValueArgs SRCS DEPS) + cmake_parse_arguments(cc_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + add_executable(${TARGET_NAME} ${cc_test_SRCS}) + target_link_libraries(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main) + add_dependencies(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main) + add_test(${TARGET_NAME} ${TARGET_NAME}) endif() - add_test(${TARGET_NAME} ${TARGET_NAME}) endfunction(cc_test) # Suppose that ops.cu includes global functions that take Tensor as @@ -95,28 +100,33 @@ endfunction(cc_test) # DEPS # tensor) function(nv_library TARGET_NAME) - set(options OPTIONAL) - set(oneValueArgs "") - set(multiValueArgs SRCS DEPS) - cmake_parse_arguments(nv_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (${nv_library_OPTIONAL} STREQUAL "SHARED") - cuda_add_library(${TARGET_NAME} SHARED ${nv_library_SRCS}) - else() - cuda_add_library(${TARGET_NAME} STATIC ${nv_library_SRCS}) + if (WITH_GPU) + set(options OPTIONAL) + set(oneValueArgs "") + set(multiValueArgs SRCS DEPS) + cmake_parse_arguments(nv_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if (${nv_library_OPTIONAL} STREQUAL "SHARED") + cuda_add_library(${TARGET_NAME} SHARED ${nv_library_SRCS}) + else() + cuda_add_library(${TARGET_NAME} STATIC ${nv_library_SRCS}) + endif() + if (nv_library_DEPS) + add_dependencies(${TARGET_NAME} ${nv_library_DEPS}) + endif() endif() - add_dependencies(${TARGET_NAME} ${nv_library_DEPS} ${external_project_dependencies}) endfunction(nv_library) function(nv_binary TARGET_NAME) - set(options "") - set(oneValueArgs "") - set(multiValueArgs SRCS DEPS) - cmake_parse_arguments(nv_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - cuda_add_executable(${TARGET_NAME} ${nv_binary_SRCS}) - link_paddle_exe(${TARGET_NAME}) - if(nv_binary_DEPS) - target_link_libraries(${TARGET_NAME} ${nv_binary_DEPS}) - add_dependencies(${TARGET_NAME} ${nv_binary_DEPS}) + if (WITH_GPU) + set(options "") + set(oneValueArgs "") + set(multiValueArgs SRCS DEPS) + cmake_parse_arguments(nv_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cuda_add_executable(${TARGET_NAME} ${nv_binary_SRCS}) + if(nv_binary_DEPS) + target_link_libraries(${TARGET_NAME} ${nv_binary_DEPS}) + add_dependencies(${TARGET_NAME} ${nv_binary_DEPS}) + endif() endif() endfunction(nv_binary) @@ -128,17 +138,16 @@ endfunction(nv_binary) # DEPS # ops) function(nv_test TARGET_NAME) - set(options "") - set(oneValueArgs "") - set(multiValueArgs SRCS DEPS) - cmake_parse_arguments(nv_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - cuda_add_executable(${TARGET_NAME} ${nv_test_SRCS}) - link_paddle_test(${TARGET_NAME}) - if(nv_test_DEPS) - target_link_libraries(${TARGET_NAME} ${nv_test_DEPS}) - add_dependencies(${TARGET_NAME} ${nv_test_DEPS}) + if (WITH_GPU AND WITH_TESTING) + set(options "") + set(oneValueArgs "") + set(multiValueArgs SRCS DEPS) + cmake_parse_arguments(nv_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cuda_add_executable(${TARGET_NAME} ${nv_test_SRCS}) + target_link_libraries(${TARGET_NAME} ${nv_test_DEPS} gtest gtest_main) + add_dependencies(${TARGET_NAME} ${nv_test_DEPS} gtest gtest_main) + add_test(${TARGET_NAME} ${TARGET_NAME}) endif() - add_test(${TARGET_NAME} ${TARGET_NAME}) endfunction(nv_test) set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go") @@ -164,7 +173,7 @@ function(go_library TARGET_NAME) set(LIB_NAME "lib${TARGET_NAME}.dylib") else() set(LIB_NAME "lib${TARGET_NAME}.so") - endif() + endif() else() set(BUILD_MODE "-buildmode=c-archive") set(LIB_NAME "lib${TARGET_NAME}.a") @@ -190,8 +199,8 @@ function(go_binary TARGET_NAME) COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}" ${go_library_SRCS} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) - add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_binary_DEPS}) + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) + add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_binary_DEPS}) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin) endfunction(go_binary) @@ -204,8 +213,8 @@ function(go_test TARGET_NAME) COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} test -c -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}" ${go_test_SRCS} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) - add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS}) + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) + add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS}) add_test(${TARGET_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}) endfunction(go_test) diff --git a/doc/design/cluster_train/pserver_client.md b/doc/design/cluster_train/pserver_client.md index 007285640e..b3e4079010 100644 --- a/doc/design/cluster_train/pserver_client.md +++ b/doc/design/cluster_train/pserver_client.md @@ -136,6 +136,9 @@ int paddle_send_grads(paddle_pserver_client* client, const paddle_gradient* grad /** * @brief paddle_get_params gets parameters from parameter servers. * + * paddle_get_params will block until parameters are initialized on + * the parameter servers. + * * @param names the array of names of the parameters to get. * @param dst the destination array of parameters to save to. * @param len the length of the names array and the paddle_parameter diff --git a/doc/howto/deep_model/rnn/hierarchical_layer_cn.rst b/doc/howto/deep_model/rnn/hierarchical_layer_cn.rst index 79048e9248..e05173c200 100644 --- a/doc/howto/deep_model/rnn/hierarchical_layer_cn.rst +++ b/doc/howto/deep_model/rnn/hierarchical_layer_cn.rst @@ -28,17 +28,17 @@ pooling 的使用示例如下，详细见 :ref:`api_v2.layer_pooling` 配置API seq_pool = pooling(input=layer, pooling_type=pooling.Max(), - agg_level=AggregateLevel.EACH_SEQUENCE) + agg_level=AggregateLevel.TO_SEQUENCE) - `pooling_type` 目前支持两种，分别是：pooling.Max()和pooling.Avg()。 -- `agg_level=AggregateLevel.EACH_TIMESTEP` 时（默认值）： +- `agg_level=AggregateLevel.TO_NO_SEQUENCE` 时（默认值）： - 作用：双层序列经过运算变成一个0层序列，或单层序列经过运算变成一个0层序列 - 输入：一个双层序列，或一个单层序列 - 输出：一个0层序列，即整个输入序列（单层或双层）的平均值（或最大值） -- `agg_level=AggregateLevel.EACH_SEQUENCE` 时： +- `agg_level=AggregateLevel.TO_SEQUENCE` 时： - 作用：一个双层序列经过运算变成一个单层序列 - 输入：必须是一个双层序列 @@ -52,15 +52,15 @@ last_seq 的使用示例如下（ :ref:`api_v2.layer_first_seq` 类似），详 .. code-block:: bash last = last_seq(input=layer, - agg_level=AggregateLevel.EACH_SEQUENCE) + agg_level=AggregateLevel.TO_SEQUENCE) -- `agg_level=AggregateLevel.EACH_TIMESTEP` 时（默认值）： +- `agg_level=AggregateLevel.TO_NO_SEQUENCE` 时（默认值）： - 作用：一个双层序列经过运算变成一个0层序列，或一个单层序列经过运算变成一个0层序列 - 输入：一个双层序列或一个单层序列 - 输出：一个0层序列，即整个输入序列（双层或者单层）最后一个，或第一个元素。 -- `agg_level=AggregateLevel.EACH_SEQUENCE` 时： +- `agg_level=AggregateLevel.TO_SEQUENCE` 时： - 作用：一个双层序列经过运算变成一个单层序列 - 输入：必须是一个双层序列 - 输出：一个单层序列，其中每个元素是双层序列中每个subseq最后一个（或第一个）元素。 @@ -74,9 +74,9 @@ expand 的使用示例如下，详细见 :ref:`api_v2.layer_expand` 配置API。 ex = expand(input=layer1, expand_as=layer2, - expand_level=ExpandLevel.FROM_TIMESTEP) + expand_level=ExpandLevel.FROM_NO_SEQUENCE) -- `expand_level=ExpandLevel.FROM_TIMESTEP` 时（默认值）： +- `expand_level=ExpandLevel.FROM_NO_SEQUENCE` 时（默认值）： - 作用：一个0层序列经过运算扩展成一个单层序列，或者一个双层序列 - 输入：layer1必须是一个0层序列，是待扩展的数据；layer2 可以是一个单层序列，或者是一个双层序列，提供扩展的长度信息 diff --git a/doc/howto/deep_model/rnn/hrnn_rnn_api_compare_cn.rst b/doc/howto/deep_model/rnn/hrnn_rnn_api_compare_cn.rst index 96e52b910a..efdc44455e 100644 --- a/doc/howto/deep_model/rnn/hrnn_rnn_api_compare_cn.rst +++ b/doc/howto/deep_model/rnn/hrnn_rnn_api_compare_cn.rst @@ -81,7 +81,7 @@ * 在本例中，我们将原始数据的每一组，通过\ :code:`recurrent_group`\ 进行拆解，拆解成的每一句话再通过一个LSTM网络。这和单层RNN的配置是等价的。 -* 与单层RNN的配置类似，我们只需要使用LSTM encode成的最后一个向量。所以对\ :code:`recurrent_group`\ 进行了\ :code:`last_seq`\ 操作。但和单层RNN不同，我们是对每一个子序列取最后一个元素，因此\ :code:`agg_level=AggregateLevel.EACH_SEQUENCE`\ 。 +* 与单层RNN的配置类似，我们只需要使用LSTM encode成的最后一个向量。所以对\ :code:`recurrent_group`\ 进行了\ :code:`last_seq`\ 操作。但和单层RNN不同，我们是对每一个子序列取最后一个元素，因此\ :code:`agg_level=AggregateLevel.TO_SEQUENCE`\ 。 * 至此，\ :code:`lstm_last`\ 便和单层RNN配置中的\ :code:`lstm_last`\ 具有相同的结果了。 diff --git a/paddle/go/cclient/cmake/CMakeDetermineGoCompiler.cmake b/go/cmake/CMakeDetermineGoCompiler.cmake similarity index 94% rename from paddle/go/cclient/cmake/CMakeDetermineGoCompiler.cmake rename to go/cmake/CMakeDetermineGoCompiler.cmake index b3f8fbe271..a9bb6906c7 100644 --- a/paddle/go/cclient/cmake/CMakeDetermineGoCompiler.cmake +++ b/go/cmake/CMakeDetermineGoCompiler.cmake @@ -38,7 +38,7 @@ endif() mark_as_advanced(CMAKE_Go_COMPILER) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CMakeGoCompiler.cmake.in +configure_file(${CMAKE_MODULE_PATH}/CMakeGoCompiler.cmake.in ${CMAKE_PLATFORM_INFO_DIR}/CMakeGoCompiler.cmake @ONLY) set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER") diff --git a/paddle/go/cclient/cmake/CMakeGoCompiler.cmake.in b/go/cmake/CMakeGoCompiler.cmake.in similarity index 100% rename from paddle/go/cclient/cmake/CMakeGoCompiler.cmake.in rename to go/cmake/CMakeGoCompiler.cmake.in diff --git a/paddle/go/cclient/cmake/CMakeGoInformation.cmake b/go/cmake/CMakeGoInformation.cmake similarity index 100% rename from paddle/go/cclient/cmake/CMakeGoInformation.cmake rename to go/cmake/CMakeGoInformation.cmake diff --git a/paddle/go/cclient/cmake/CMakeTestGoCompiler.cmake b/go/cmake/CMakeTestGoCompiler.cmake similarity index 100% rename from paddle/go/cclient/cmake/CMakeTestGoCompiler.cmake rename to go/cmake/CMakeTestGoCompiler.cmake diff --git a/paddle/go/cclient/cmake/flags.cmake b/go/cmake/flags.cmake similarity index 95% rename from paddle/go/cclient/cmake/flags.cmake rename to go/cmake/flags.cmake index 062d5ab660..a167c432a9 100644 --- a/paddle/go/cclient/cmake/flags.cmake +++ b/go/cmake/flags.cmake @@ -21,7 +21,7 @@ function(CheckCompilerCXX11Flag) if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.3) message(FATAL_ERROR "Unsupported Clang version. Clang >= 3.3 required.") endif() - endif() + endif() endif() endfunction() @@ -42,4 +42,4 @@ if (CUDA_VERSION VERSION_GREATER "8.0" OR CUDA_VERSION VERSION_EQUAL "8.0") list(APPEND __arch_flags " -gencode arch=compute_60,code=sm_60") endif() -set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS}) \ No newline at end of file +set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS}) diff --git a/go/cmake/golang.cmake b/go/cmake/golang.cmake new file mode 100644 index 0000000000..e73b0c865b --- /dev/null +++ b/go/cmake/golang.cmake @@ -0,0 +1,50 @@ +set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go") +file(MAKE_DIRECTORY ${GOPATH}) +set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle") +file(MAKE_DIRECTORY ${PADDLE_IN_GOPATH}) + +function(GO_LIBRARY NAME BUILD_TYPE) + if(BUILD_TYPE STREQUAL "STATIC") + set(BUILD_MODE -buildmode=c-archive) + set(LIB_NAME "lib${NAME}.a") + else() + set(BUILD_MODE -buildmode=c-shared) + if(APPLE) + set(LIB_NAME "lib${NAME}.dylib") + else() + set(LIB_NAME "lib${NAME}.so") + endif() + endif() + + file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go") + file(RELATIVE_PATH rel ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + + # find Paddle directory. + get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) + get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) + get_filename_component(PADDLE_DIR ${PARENT_DIR} DIRECTORY) + + # automatically get all dependencies specified in the source code + # for given target. + add_custom_target(goGet env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ${rel}/...) + + # make a symlink that references Paddle inside $GOPATH, so go get + # will use the local changes in Paddle rather than checkout Paddle + # in github. + add_custom_target(copyPaddle + COMMAND ln -sf ${PADDLE_DIR} ${PADDLE_IN_GOPATH}) + add_dependencies(goGet copyPaddle) + + add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp + COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE} + -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}" + ${CMAKE_GO_FLAGS} ${GO_SOURCE} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) + + add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN}) + add_dependencies(${NAME} goGet) + + if(NOT BUILD_TYPE STREQUAL "STATIC") + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} DESTINATION bin) + endif() +endfunction(GO_LIBRARY) diff --git a/paddle/go/cmd/master/master.go b/go/cmd/master/master.go similarity index 94% rename from paddle/go/cmd/master/master.go rename to go/cmd/master/master.go index ef1f87c2dd..cc6e45049a 100644 --- a/paddle/go/cmd/master/master.go +++ b/go/cmd/master/master.go @@ -13,8 +13,8 @@ import ( "github.com/namsral/flag" - "github.com/PaddlePaddle/Paddle/paddle/go/master" - "github.com/PaddlePaddle/Paddle/paddle/go/recordio" + "github.com/PaddlePaddle/Paddle/go/master" + "github.com/PaddlePaddle/Paddle/go/recordio" ) func main() { diff --git a/paddle/go/cmd/pserver/.gitignore b/go/cmd/pserver/.gitignore similarity index 100% rename from paddle/go/cmd/pserver/.gitignore rename to go/cmd/pserver/.gitignore diff --git a/paddle/go/cmd/pserver/pserver.go b/go/cmd/pserver/pserver.go similarity index 89% rename from paddle/go/cmd/pserver/pserver.go rename to go/cmd/pserver/pserver.go index bd4bfc7028..f0be251c24 100644 --- a/paddle/go/cmd/pserver/pserver.go +++ b/go/cmd/pserver/pserver.go @@ -8,7 +8,7 @@ import ( "github.com/namsral/flag" - "github.com/PaddlePaddle/Paddle/paddle/go/pserver" + "github.com/PaddlePaddle/Paddle/go/pserver" ) func main() { diff --git a/paddle/go/master/service.go b/go/master/service.go similarity index 98% rename from paddle/go/master/service.go rename to go/master/service.go index 7526648287..50e646b01f 100644 --- a/paddle/go/master/service.go +++ b/go/master/service.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/PaddlePaddle/Paddle/paddle/go/recordio" + "github.com/PaddlePaddle/Paddle/go/recordio" ) const ( diff --git a/paddle/go/master/service_internal_test.go b/go/master/service_internal_test.go similarity index 100% rename from paddle/go/master/service_internal_test.go rename to go/master/service_internal_test.go diff --git a/go/pserver/cclient/CMakeLists.txt b/go/pserver/cclient/CMakeLists.txt new file mode 100644 index 0000000000..c017d74656 --- /dev/null +++ b/go/pserver/cclient/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.0) + +get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") + +project(cxx_go C Go) + +include(golang) +include(flags) + +go_library(client STATIC) +add_subdirectory(test) diff --git a/paddle/go/cclient/cclient.go b/go/pserver/cclient/cclient.go similarity index 82% rename from paddle/go/cclient/cclient.go rename to go/pserver/cclient/cclient.go index dc86d47e8d..0b4aa79806 100644 --- a/paddle/go/cclient/cclient.go +++ b/go/pserver/cclient/cclient.go @@ -39,10 +39,11 @@ import "C" import ( "log" + "strings" "sync" "unsafe" - "github.com/PaddlePaddle/Paddle/paddle/go/pserver" + "github.com/PaddlePaddle/Paddle/go/pserver" ) var nullPtr = unsafe.Pointer(uintptr(0)) @@ -78,34 +79,54 @@ func cArrayToSlice(p unsafe.Pointer, len int) []byte { return nil } - // create a Go clice backed by a C array, - // reference: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices + // create a Go clice backed by a C array, reference: + // https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices + // + // Go garbage collector will not interact with this data, need + // to be freed properly. return (*[1 << 30]byte)(p)[:len:len] } +type selector bool + +func (s selector) Select() bool { + return bool(s) +} + +type lister []pserver.Server + +func (l lister) List() []pserver.Server { + return l +} + //export paddle_new_pserver_client -func paddle_new_pserver_client(addr *C.char) C.client { - c := pserver.NewClient(C.GoString(addr)) +func paddle_new_pserver_client(addrs *C.char, selected int) C.client { + a := C.GoString(addrs) + as := strings.Split(a, ",") + servers := make([]pserver.Server, len(as)) + for i := range as { + servers[i].Index = i + servers[i].Addr = as[i] + } + c := pserver.NewClient(lister(servers), len(as), selector(selected != 0)) return add(c) } +//export paddle_new_etcd_pserver_client +func paddle_new_etcd_pserver_client(etcd_addr *C.char) C.client { + // TODO(helin): fault tolerant pserver client using etcd. + panic("not implemented.") +} + //export paddle_pserver_client_release func paddle_pserver_client_release(client C.client) { - c := remove(client) - c.Cleanup() + remove(client) } //export paddle_begin_init_params -func paddle_begin_init_params(client C.client, pserver_config unsafe.Pointer, config_len C.int) C.int { +func paddle_begin_init_params(client C.client) C.int { c := get(client) - b := cArrayToSlice(pserver_config, int(config_len)) - selected, err := c.BeginInitParams(b) - if err != nil { - log.Println(err) - return -1 - } - - if selected { + if selected := c.BeginInitParams(); selected { return 1 } return 0 @@ -227,7 +248,7 @@ func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter func paddle_save_model(client C.client, path *C.char) C.int { p := C.GoString(path) c := get(client) - err := c.SaveModel(p) + err := c.Save(p) if err != nil { log.Println(err) return -1 diff --git a/paddle/go/cclient/test/CMakeLists.txt b/go/pserver/cclient/test/CMakeLists.txt similarity index 67% rename from paddle/go/cclient/test/CMakeLists.txt rename to go/pserver/cclient/test/CMakeLists.txt index de7ef6a47a..16f84648c1 100644 --- a/paddle/go/cclient/test/CMakeLists.txt +++ b/go/pserver/cclient/test/CMakeLists.txt @@ -4,5 +4,8 @@ include_directories(${CMAKE_BINARY_DIR}) add_executable(main main.c) add_dependencies(main client) -set (CMAKE_EXE_LINKER_FLAGS "-pthread") + +if(APPLE) + set(CMAKE_EXE_LINKER_FLAGS "-framework CoreFoundation -framework Security") +endif() target_link_libraries(main ${CMAKE_BINARY_DIR}/libclient.a) diff --git a/paddle/go/cclient/test/main.c b/go/pserver/cclient/test/main.c similarity index 81% rename from paddle/go/cclient/test/main.c rename to go/pserver/cclient/test/main.c index 28e3d03b7a..f75a2110b9 100644 --- a/paddle/go/cclient/test/main.c +++ b/go/pserver/cclient/test/main.c @@ -1,22 +1,23 @@ -#include "libclient.h" +#include -//#include "gtest/gtest.h" +#include "libclient.h" -void panic() { +void fail() { // TODO(helin): fix: gtest using cmake is not working, using this // hacky way for now. - *(void*)0; + printf("test failed.\n"); + exit(-1); } int main() { char addr[] = "localhost:3000"; - client c = paddle_new_pserver_client(addr); + client c = paddle_new_pserver_client(addr, 1); retry: - if (paddle_begin_init_params(c, NULL, 0)) { + if (paddle_begin_init_params(c)) { paddle_parameter param; char name_a[] = "param_a"; char name_b[] = "param_b"; - char content[] = {0x00, 0x11, 0x22}; + unsigned char content[] = {0x00, 0x11, 0x22}; param.element_type = PADDLE_ELEMENT_TYPE_FLOAT32; param.name = name_a; param.content = content; @@ -35,34 +36,34 @@ retry: goto retry; } } else { - panic(); + fail(); } - char content[] = {0x00, 0x11, 0x22}; + unsigned char content[] = {0x00, 0x11, 0x22}; paddle_gradient grads[2] = { {"param_a", PADDLE_ELEMENT_TYPE_INT32, content, 3}, {"param_b", PADDLE_ELEMENT_TYPE_FLOAT32, content, 3}}; if (!paddle_send_grads(c, grads, 2)) { - panic(); + fail(); } paddle_parameter* params[2] = {NULL, NULL}; char* names[] = {"param_a", "param_b"}; if (!paddle_get_params(c, names, params, 2)) { - panic(); + fail(); } // get parameters again by reusing the allocated parameter buffers. if (!paddle_get_params(c, names, params, 2)) { - panic(); + fail(); } paddle_release_param(params[0]); paddle_release_param(params[1]); if (!paddle_save_model(c, "/tmp/")) { - panic(); + fail(); } return 0; diff --git a/go/pserver/client.go b/go/pserver/client.go new file mode 100644 index 0000000000..f8bd0aa59f --- /dev/null +++ b/go/pserver/client.go @@ -0,0 +1,232 @@ +package pserver + +import ( + "hash/fnv" + "log" + "sort" + "time" + + "github.com/PaddlePaddle/Paddle/go/pserver/internal/connection" +) + +// TODO(helin): add RPC call retry logic + +// Selector selects if the client should initialize parameter servers. +type Selector interface { + Select() bool +} + +// Server is the identification of a parameter Server. +type Server struct { + Index int + Addr string +} + +// Lister lists currently available parameter servers. +type Lister interface { + List() []Server +} + +// Client is the client to parameter servers. +type Client struct { + sel Selector + pservers []*connection.Conn +} + +// NewClient creates a new client. +func NewClient(l Lister, pserverNum int, sel Selector) *Client { + c := &Client{sel: sel} + c.pservers = make([]*connection.Conn, pserverNum) + for i := 0; i < pserverNum; i++ { + c.pservers[i] = connection.New() + } + go c.monitorPservers(l, pserverNum) + return c +} + +// monitorPservers monitors pserver addresses, and updates connection +// when the address changes. +func (c *Client) monitorPservers(l Lister, pserverNum int) { + knownServers := make([]Server, pserverNum) + ticker := time.NewTicker(10 * time.Second) + monitor := func() { + curServers := make([]Server, pserverNum) + list := l.List() + for _, l := range list { + curServers[l.Index] = l + } + + for i := range knownServers { + if knownServers[i].Addr != curServers[i].Addr { + err := c.pservers[i].Connect(curServers[i].Addr) + if err != nil { + log.Println(err) + + // connect to addr failed, set + // to last known addr in order + // to retry next time. + curServers[i].Addr = knownServers[i].Addr + } + } + } + + knownServers = curServers + } + + monitor() + for _ = range ticker.C { + monitor() + } +} + +// BeginInitParams begins to initialize parameters on parameter +// servers. +// +// BeginInitParams will be called from multiple trainers, only one +// trainer will be selected to initialize the parameters on parameter +// servers. Other trainers will be blocked until the initialization is +// done, and they need to get the initialized parameters from +// parameter servers using GetParams. +func (c *Client) BeginInitParams() bool { + return c.sel.Select() +} + +// InitParam initializes the parameter on parameter servers. +func (c *Client) InitParam(paramWithConfigs ParameterWithConfig) error { + var dummy int + return c.pservers[c.partition(paramWithConfigs.Param.Name)].Call("Service.InitParam", paramWithConfigs, &dummy) +} + +// FinishInitParams tells parameter servers client has sent all +// parameters to parameter servers as initialization. +func (c *Client) FinishInitParams() error { + for _, p := range c.pservers { + var dummy int + err := p.Call("Service.FinishInitParams", dummy, &dummy) + if err != nil { + return err + } + } + return nil +} + +// SendGrads sends gradients to parameter servers for updating +// parameters. +func (c *Client) SendGrads(grads []Gradient) error { + errCh := make(chan error, len(grads)) + for _, g := range grads { + go func(g Gradient) { + var dummy int + err := c.pservers[c.partition(g.Name)].Call("Service.SendGrad", g, &dummy) + errCh <- err + }(g) + } + + recv := 0 + for err := range errCh { + if err != nil { + return err + } + + recv++ + if recv == len(grads) { + break + } + } + return nil +} + +type result struct { + idx int + param Parameter + err error +} + +type results []result + +func (r results) Len() int { + return len(r) +} + +func (r results) Less(i int, j int) bool { + return r[i].idx < r[j].idx +} + +func (r results) Swap(i int, j int) { + r[i], r[j] = r[j], r[i] +} + +// GetParams gets parameters from parameter servers. +func (c *Client) GetParams(names []string) ([]Parameter, error) { + rCh := make(chan result, len(names)) + + for idx, name := range names { + go func(name string, idx int) { + var parameter Parameter + err := c.pservers[c.partition(name)].Call("Service.GetParam", name, ¶meter) + rCh <- result{idx: idx, param: parameter, err: err} + }(name, idx) + } + + var rs results + recv := 0 + for r := range rCh { + if r.err != nil { + return nil, r.err + } + rs = append(rs, r) + + recv++ + if recv == len(names) { + break + } + } + sort.Sort(rs) + + ps := make([]Parameter, len(rs)) + for i := range rs { + ps[i] = rs[i].param + } + + return ps, nil +} + +// Save indicates parameters to save the parameter to the given path. +func (c *Client) Save(path string) error { + errCh := make(chan error, len(c.pservers)) + + for _, p := range c.pservers { + var dummy int + err := p.Call("Service.Save", path, &dummy) + errCh <- err + } + + recv := 0 + for err := range errCh { + if err != nil { + return err + } + + recv++ + if recv == len(c.pservers) { + break + } + } + + // TODO(helin): there will be many files under path, need to + // merge them into a single file. + return nil +} + +func strHash(s string) uint32 { + h := fnv.New32a() + h.Write([]byte(s)) + return h.Sum32() +} + +// TODO(helin): now partition only select which parameter server to +// send the entire parameter. We need to partition a parameter into +// small blocks and send to different parameter servers. +func (c *Client) partition(key string) int { + return int(strHash(key) % uint32(len(c.pservers))) +} diff --git a/go/pserver/client_test.go b/go/pserver/client_test.go new file mode 100644 index 0000000000..a9a0948a51 --- /dev/null +++ b/go/pserver/client_test.go @@ -0,0 +1,123 @@ +package pserver_test + +import ( + "net" + "net/http" + "net/rpc" + "strconv" + "strings" + "testing" + + "github.com/PaddlePaddle/Paddle/go/pserver" +) + +const numPserver = 10 + +var port [numPserver]int + +func init() { + for i := 0; i < numPserver; i++ { + l, err := net.Listen("tcp", ":0") + if err != nil { + panic(err) + } + + ss := strings.Split(l.Addr().String(), ":") + p, err := strconv.Atoi(ss[len(ss)-1]) + if err != nil { + panic(err) + } + port[i] = p + + go func(l net.Listener) { + s := pserver.NewService() + server := rpc.NewServer() + err := server.Register(s) + if err != nil { + panic(err) + } + + mux := http.NewServeMux() + mux.Handle(rpc.DefaultRPCPath, server) + err = http.Serve(l, mux) + if err != nil { + panic(err) + } + }(l) + } +} + +type selector bool + +func (s selector) Select() bool { + return bool(s) +} + +type lister []pserver.Server + +func (l lister) List() []pserver.Server { + return l +} + +func TestClientFull(t *testing.T) { + servers := make([]pserver.Server, numPserver) + for i := 0; i < numPserver; i++ { + servers[i] = pserver.Server{Index: i, Addr: ":" + strconv.Itoa(port[i])} + } + c := pserver.NewClient(lister(servers), len(servers), selector(true)) + selected := c.BeginInitParams() + if !selected { + t.Fatal("should be selected.") + } + + const numParameter = 100 + for i := 0; i < numParameter; i++ { + var p pserver.Parameter + p.Name = "p_" + strconv.Itoa(i) + p.ElementType = pserver.Float32 + p.Content = make([]byte, (i+1)*100) + err := c.InitParam(pserver.ParameterWithConfig{Param: p}) + if err != nil { + t.Fatal(err) + } + } + + err := c.FinishInitParams() + if err != nil { + t.Fatal(err) + } + + var grads []pserver.Gradient + for i := 0; i < numParameter/2; i++ { + var g pserver.Gradient + g.Name = "p_" + strconv.Itoa(i) + g.ElementType = pserver.Float32 + g.Content = make([]byte, (i+1)*100) + grads = append(grads, g) + } + + err = c.SendGrads(grads) + if err != nil { + t.Fatal(err) + } + + names := make([]string, numParameter) + for i := 0; i < numParameter; i++ { + names[i] = "p_" + strconv.Itoa(i) + } + + params, err := c.GetParams(names) + if err != nil { + t.Fatal(err) + } + + if len(names) != len(params) { + t.Fatalf("parameter size not match, need: %d, have: %d", len(names), len(params)) + } + + for i := range params { + if names[i] != params[i].Name { + t.Fatalf("order of returned parameter does not required: parameter name: %s, required name: %s", names[i], params[i]) + } + } +} diff --git a/go/pserver/internal/connection/conn.go b/go/pserver/internal/connection/conn.go new file mode 100644 index 0000000000..1c04f11725 --- /dev/null +++ b/go/pserver/internal/connection/conn.go @@ -0,0 +1,84 @@ +package connection + +import ( + "errors" + "net/rpc" + "sync" +) + +// TODO(helin): add TCP re-connect logic + +// Conn is a connection to a parameter server +type Conn struct { + mu sync.Mutex + client *rpc.Client + waitConn chan struct{} +} + +// New creates a new connection. +func New() *Conn { + c := &Conn{} + return c +} + +// Connect connects the connection to a address. +func (c *Conn) Connect(addr string) error { + c.mu.Lock() + if c.client != nil { + err := c.client.Close() + if err != nil { + c.mu.Unlock() + return err + } + + c.client = nil + } + c.mu.Unlock() + + client, err := rpc.DialHTTP("tcp", addr) + if err != nil { + return err + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.client == nil { + c.client = client + if c.waitConn != nil { + close(c.waitConn) + c.waitConn = nil + } + } else { + return errors.New("client already set from a concurrent goroutine") + } + + return nil +} + +// Call make a RPC call. +// +// Call will be blocked until the connection to remote RPC service +// being established. +func (c *Conn) Call(serviceMethod string, args interface{}, reply interface{}) error { + c.mu.Lock() + client := c.client + var waitCh chan struct{} + if client == nil { + if c.waitConn != nil { + waitCh = c.waitConn + } else { + waitCh = make(chan struct{}) + c.waitConn = waitCh + } + } + c.mu.Unlock() + + if waitCh != nil { + // wait until new connection being established + <-waitCh + return c.Call(serviceMethod, args, reply) + } + + return client.Call(serviceMethod, args, reply) +} diff --git a/paddle/go/pserver/optimizer.c b/go/pserver/optimizer.c similarity index 100% rename from paddle/go/pserver/optimizer.c rename to go/pserver/optimizer.c diff --git a/paddle/go/pserver/optimizer.go b/go/pserver/optimizer.go similarity index 77% rename from paddle/go/pserver/optimizer.go rename to go/pserver/optimizer.go index 64bdefe660..417f8c5093 100644 --- a/paddle/go/pserver/optimizer.go +++ b/go/pserver/optimizer.go @@ -29,11 +29,11 @@ func newOptimizer(t optimizerType, learning_rate float64) *optimizer { func (o *optimizer) UpdateParameter(p Parameter, g Gradient) error { if len(p.Content) != len(g.Content) { - return fmt.Errorf("parameter and gradient length not match, parameter: %d, gradient: %d", len(p.Content), len(g.Content)) + return fmt.Errorf("Name: %s, parameter and gradient length not match, parameter: %d, gradient: %d", p.Name, len(p.Content), len(g.Content)) } if p.ElementType != g.ElementType { - return fmt.Errorf("parameter and gradient element type not match, parameter: %v, gradient: %v", p.ElementType, g.ElementType) + return fmt.Errorf("Name: %s, parameter and gradient element type not match, parameter: %v, gradient: %v", p.Name, p.ElementType, g.ElementType) } r := C.paddle_update_parameter(o.opt, unsafe.Pointer(&p.Content[0]), C.paddle_element_type(p.ElementType), unsafe.Pointer(&g.Content[0]), C.int(len(g.Content))) diff --git a/paddle/go/pserver/optimizer.h b/go/pserver/optimizer.h similarity index 100% rename from paddle/go/pserver/optimizer.h rename to go/pserver/optimizer.h diff --git a/paddle/go/pserver/optimizer_test.go b/go/pserver/optimizer_test.go similarity index 100% rename from paddle/go/pserver/optimizer_test.go rename to go/pserver/optimizer_test.go diff --git a/paddle/go/pserver/service.go b/go/pserver/service.go similarity index 55% rename from paddle/go/pserver/service.go rename to go/pserver/service.go index f43e59403a..d5787b9708 100644 --- a/paddle/go/pserver/service.go +++ b/go/pserver/service.go @@ -49,33 +49,12 @@ type Service struct { // NewService creates a new service. func NewService() *Service { - s := &Service{} + s := &Service{opt: newOptimizer(sgd, 0.01)} s.paramMap = make(map[string]Parameter) s.initialized = make(chan struct{}) return s } -// BeginInitParams tells the parameter server that the parameter -// initialization has begun. -func (s *Service) BeginInitParams(config []byte, dummy *int) error { - select { - case <-s.initialized: - return ErrAlreadyInitialized - default: - } - - s.mu.Lock() - defer s.mu.Unlock() - - if s.opt != nil { - s.opt.Cleanup() - } - - // TODO(helin): parse learning rate from config - s.opt = newOptimizer(sgd, 0.01) - return nil -} - // InitParam initializes a parameter. func (s *Service) InitParam(paramWithConfigs ParameterWithConfig, dummy *int) error { select { @@ -109,75 +88,45 @@ func (s *Service) FinishInitParams(dummy0 int, dummy1 *int) error { return nil } -// SendGrads sends gradients to parameter servers for parameter +// SendGrad sends gradient to parameter servers for parameter // optimization. -func (s *Service) SendGrads(grads []Gradient, dummy *int) error { +func (s *Service) SendGrad(g Gradient, dummy *int) error { select { case <-s.initialized: default: return ErrUninitialized } - count := len(grads) - if count == 0 { - return nil - } - s.mu.Lock() defer s.mu.Unlock() - for _, g := range grads { - if _, ok := s.paramMap[g.Name]; !ok { - return fmt.Errorf("parameter: %s does not exist", g.Name) - } - } - - errCh := make(chan error, count) - for _, g := range grads { - go func(p Parameter, g Gradient) { - err := s.opt.UpdateParameter(p, g) - errCh <- err - }(s.paramMap[g.Name], g) + p, ok := s.paramMap[g.Name] + if !ok { + return fmt.Errorf("parameter: %s does not exist", g.Name) } - recv := 0 - for err := range errCh { - if err != nil { - return err - } - - recv++ - if recv == count { - break - } - } - return nil + return s.opt.UpdateParameter(p, g) } -// GetParams gets parameters from the parameter server. -func (s *Service) GetParams(names []string, parameters *[]Parameter) error { +// GetParam gets parameters from the parameter server. +func (s *Service) GetParam(name string, parameter *Parameter) error { <-s.initialized s.mu.Lock() defer s.mu.Unlock() - for _, n := range names { - if _, ok := s.paramMap[n]; !ok { - return fmt.Errorf("parameter: %s does not exist", n) - } - } - - *parameters = make([]Parameter, len(names)) - for i, n := range names { - // The parameter content (a byte slice) may change - // during RPC serialization due to write from other - // goroutine, we allow it since mini-batch based deep - // learning optimization methods are stochastic in - // nature. This race condition is allowed deliberately - // to save the program from making a copy of the - // paramter content. - (*parameters)[i] = s.paramMap[n] + p, ok := s.paramMap[name] + if !ok { + return fmt.Errorf("parameter: %s does not exist", name) } + // The parameter content (a byte slice) may change + // during RPC serialization due to write from other + // goroutine, we allow it since mini-batch based deep + // learning optimization methods are stochastic in + // nature. This race condition is allowed deliberately + // to save the program from making a copy of the + // paramter content. + *parameter = p return nil } diff --git a/paddle/go/pserver/service_test.go b/go/pserver/service_test.go similarity index 60% rename from paddle/go/pserver/service_test.go rename to go/pserver/service_test.go index 10185bd0f2..4c9fac4536 100644 --- a/paddle/go/pserver/service_test.go +++ b/go/pserver/service_test.go @@ -4,23 +4,19 @@ import ( "reflect" "sync" "testing" + "time" - "github.com/PaddlePaddle/Paddle/paddle/go/pserver" + "github.com/PaddlePaddle/Paddle/go/pserver" ) func TestFull(t *testing.T) { s := pserver.NewService() - var dummy int - err := s.BeginInitParams(nil, &dummy) - if err != nil { - t.FailNow() - } - var p pserver.Parameter p.Name = "param_a" p.Content = []byte{1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0} p.ElementType = pserver.Int32 - err = s.InitParam(pserver.ParameterWithConfig{p, nil}, &dummy) + var dummy int + err := s.InitParam(pserver.ParameterWithConfig{p, nil}, &dummy) if err != nil { t.FailNow() } @@ -39,40 +35,39 @@ func TestFull(t *testing.T) { t.FailNow() } - var params []pserver.Parameter - err = s.GetParams([]string{"param_b", "param_a"}, ¶ms) + var param pserver.Parameter + err = s.GetParam("param_b", ¶m) if err != nil { t.FailNow() } - if len(params) != 2 || !reflect.DeepEqual(params[0], p1) || !reflect.DeepEqual(params[0], p1) { + if !reflect.DeepEqual(param, p1) { t.FailNow() } - grads := []pserver.Gradient{pserver.Gradient(p1), pserver.Gradient(p)} - err = s.SendGrads(grads, &dummy) + g1, g2 := pserver.Gradient(p1), pserver.Gradient(p) + err = s.SendGrad(g1, &dummy) if err != nil { t.FailNow() } + err = s.SendGrad(g2, &dummy) - var params1 []pserver.Parameter - err = s.GetParams([]string{"param_b", "param_a"}, ¶ms1) if err != nil { t.FailNow() } - if len(params) != 2 { + var param1 pserver.Parameter + err = s.GetParam("param_a", ¶m1) + if err != nil { t.FailNow() } // don't compare content, since it's already changed by // gradient update. - params1[0].Content = nil - params1[0].Content = nil + param1.Content = nil p.Content = nil - p1.Content = nil - if !reflect.DeepEqual(params1[0], p1) || !reflect.DeepEqual(params1[0], p1) { + if !reflect.DeepEqual(param1, p) { t.FailNow() } } @@ -80,19 +75,7 @@ func TestFull(t *testing.T) { func TestMultipleInit(t *testing.T) { s := pserver.NewService() var dummy int - err := s.BeginInitParams(nil, &dummy) - if err != nil { - t.FailNow() - } - - // this is fine, it's possible for client to call init - // multiple times. - err = s.BeginInitParams(nil, &dummy) - if err != nil { - t.FailNow() - } - - err = s.FinishInitParams(0, &dummy) + err := s.FinishInitParams(0, &dummy) if err != nil { t.FailNow() } @@ -101,17 +84,12 @@ func TestMultipleInit(t *testing.T) { if err != pserver.ErrAlreadyInitialized { t.FailNow() } - - err = s.BeginInitParams(nil, &dummy) - if err != pserver.ErrAlreadyInitialized { - t.FailNow() - } } func TestUninitialized(t *testing.T) { s := pserver.NewService() var dummy int - err := s.SendGrads(nil, &dummy) + err := s.SendGrad(pserver.Gradient{}, &dummy) if err != pserver.ErrUninitialized { t.FailNow() } @@ -123,8 +101,8 @@ func TestBlockUntilInitialized(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - var params []pserver.Parameter - err := s.GetParams(nil, ¶ms) + var param pserver.Parameter + err := s.GetParam("param_a", ¶m) if err != nil { t.FailNow() } @@ -143,11 +121,7 @@ func TestBlockUntilInitialized(t *testing.T) { ch <- struct{}{} }() - var dummy int - err := s.BeginInitParams(nil, &dummy) - if err != nil { - t.FailNow() - } + time.Sleep(50 * time.Millisecond) select { case <-ch: @@ -156,6 +130,16 @@ func TestBlockUntilInitialized(t *testing.T) { default: } + var p pserver.Parameter + p.Name = "param_a" + p.Content = []byte{1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0} + p.ElementType = pserver.Int32 + var dummy int + err := s.InitParam(pserver.ParameterWithConfig{p, nil}, &dummy) + if err != nil { + t.FailNow() + } + err = s.FinishInitParams(0, &dummy) if err != nil { t.FailNow() diff --git a/paddle/go/recordio/README.md b/go/recordio/README.md similarity index 92% rename from paddle/go/recordio/README.md rename to go/recordio/README.md index 8b0b9308b1..50e7e95476 100644 --- a/paddle/go/recordio/README.md +++ b/go/recordio/README.md @@ -8,6 +8,7 @@ w := recordio.NewWriter(f) w.Write([]byte("Hello")) w.Write([]byte("World!")) w.Close() +f.Close() ``` ## Read @@ -18,6 +19,7 @@ w.Close() f, e := os.Open("a_file.recordio") idx, e := recordio.LoadIndex(f) fmt.Println("Total records: ", idx.Len()) + f.Close() ``` 2. Create one or more scanner to read a range of records. The @@ -30,7 +32,8 @@ w.Close() for s.Scan() { fmt.Println(string(s.Record())) } - if s.Err() != nil && s.Err() != io.EOF { + if s.Err() != nil { log.Fatalf("Something wrong with scanning: %v", e) } + f.Close() ``` diff --git a/go/recordio/c/CMakeLists.txt b/go/recordio/c/CMakeLists.txt new file mode 100644 index 0000000000..c300c091f8 --- /dev/null +++ b/go/recordio/c/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.0) + +get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") + +project(cxx_go C Go) + +include(golang) +include(flags) + +go_library(recordio STATIC) +add_subdirectory(test) diff --git a/go/recordio/c/crecordio.go b/go/recordio/c/crecordio.go new file mode 100644 index 0000000000..e5cc302992 --- /dev/null +++ b/go/recordio/c/crecordio.go @@ -0,0 +1,116 @@ +package main + +/* +#include + +typedef int reader; +typedef int writer; +*/ +import "C" + +import ( + "log" + "os" + "strings" + "unsafe" + + "github.com/PaddlePaddle/Paddle/go/recordio" +) + +var nullPtr = unsafe.Pointer(uintptr(0)) + +type writer struct { + w *recordio.Writer + f *os.File +} + +type reader struct { + scanner *recordio.Scanner +} + +func cArrayToSlice(p unsafe.Pointer, len int) []byte { + if p == nullPtr { + return nil + } + + // create a Go clice backed by a C array, reference: + // https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices + // + // Go garbage collector will not interact with this data, need + // to be freed properly. + return (*[1 << 30]byte)(p)[:len:len] +} + +//export create_recordio_writer +func create_recordio_writer(path *C.char) C.writer { + p := C.GoString(path) + f, err := os.Create(p) + if err != nil { + log.Println(err) + return -1 + } + + w := recordio.NewWriter(f, -1, -1) + writer := &writer{f: f, w: w} + return addWriter(writer) +} + +//export recordio_write +func recordio_write(writer C.writer, buf *C.uchar, size C.int) C.int { + w := getWriter(writer) + b := cArrayToSlice(unsafe.Pointer(buf), int(size)) + c, err := w.w.Write(b) + if err != nil { + log.Println(err) + return -1 + } + + return C.int(c) +} + +//export release_recordio_writer +func release_recordio_writer(writer C.writer) { + w := removeWriter(writer) + w.w.Close() + w.f.Close() +} + +//export create_recordio_reader +func create_recordio_reader(path *C.char) C.reader { + p := C.GoString(path) + s, err := recordio.NewScanner(strings.Split(p, ",")...) + if err != nil { + log.Println(err) + return -1 + } + + r := &reader{scanner: s} + return addReader(r) +} + +//export recordio_read +func recordio_read(reader C.reader, record **C.uchar) C.int { + r := getReader(reader) + if r.scanner.Scan() { + buf := r.scanner.Record() + if len(buf) == 0 { + *record = (*C.uchar)(nullPtr) + return 0 + } + + size := C.int(len(buf)) + *record = (*C.uchar)(C.malloc(C.size_t(len(buf)))) + C.memcpy(unsafe.Pointer(*record), unsafe.Pointer(&buf[0]), C.size_t(len(buf))) + return size + } + + return -1 +} + +//export release_recordio_reader +func release_recordio_reader(reader C.reader) { + r := removeReader(reader) + r.scanner.Close() +} + +func main() {} // Required but ignored diff --git a/go/recordio/c/register.go b/go/recordio/c/register.go new file mode 100644 index 0000000000..61dfdbd4ab --- /dev/null +++ b/go/recordio/c/register.go @@ -0,0 +1,61 @@ +package main + +/* +typedef int reader; +typedef int writer; +*/ +import "C" + +import "sync" + +var mu sync.Mutex +var handleMap = make(map[C.reader]*reader) +var curHandle C.reader +var writerMap = make(map[C.writer]*writer) +var curWriterHandle C.writer + +func addReader(r *reader) C.reader { + mu.Lock() + defer mu.Unlock() + reader := curHandle + curHandle++ + handleMap[reader] = r + return reader +} + +func getReader(reader C.reader) *reader { + mu.Lock() + defer mu.Unlock() + return handleMap[reader] +} + +func removeReader(reader C.reader) *reader { + mu.Lock() + defer mu.Unlock() + r := handleMap[reader] + delete(handleMap, reader) + return r +} + +func addWriter(w *writer) C.writer { + mu.Lock() + defer mu.Unlock() + writer := curWriterHandle + curWriterHandle++ + writerMap[writer] = w + return writer +} + +func getWriter(writer C.writer) *writer { + mu.Lock() + defer mu.Unlock() + return writerMap[writer] +} + +func removeWriter(writer C.writer) *writer { + mu.Lock() + defer mu.Unlock() + w := writerMap[writer] + delete(writerMap, writer) + return w +} diff --git a/go/recordio/c/test/CMakeLists.txt b/go/recordio/c/test/CMakeLists.txt new file mode 100644 index 0000000000..bac1006ae1 --- /dev/null +++ b/go/recordio/c/test/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.0) + +include_directories(${CMAKE_BINARY_DIR}) + +add_executable(recordio_test test.c) +add_dependencies(recordio_test recordio) +set (CMAKE_EXE_LINKER_FLAGS "-pthread") +target_link_libraries(recordio_test ${CMAKE_BINARY_DIR}/librecordio.a) diff --git a/go/recordio/c/test/test.c b/go/recordio/c/test/test.c new file mode 100644 index 0000000000..b25536a9d7 --- /dev/null +++ b/go/recordio/c/test/test.c @@ -0,0 +1,56 @@ +#include +#include + +#include "librecordio.h" + +void fail() { + // TODO(helin): fix: gtest using cmake is not working, using this + // hacky way for now. + printf("test failed.\n"); + exit(-1); +} + +int main() { + writer w = create_recordio_writer("/tmp/test_recordio_0"); + recordio_write(w, "hello", 6); + recordio_write(w, "hi", 3); + release_recordio_writer(w); + + w = create_recordio_writer("/tmp/test_recordio_1"); + recordio_write(w, "dog", 4); + recordio_write(w, "cat", 4); + release_recordio_writer(w); + + reader r = create_recordio_reader("/tmp/test_recordio_*"); + unsigned char* item = NULL; + int size = recordio_read(r, &item); + if (strcmp(item, "hello") || size != 6) { + fail(); + } + free(item); + + size = recordio_read(r, &item); + if (strcmp(item, "hi") || size != 3) { + fail(); + } + free(item); + + size = recordio_read(r, &item); + if (strcmp(item, "dog") || size != 4) { + fail(); + } + free(item); + + size = recordio_read(r, &item); + if (strcmp(item, "cat") || size != 4) { + fail(); + } + free(item); + + size = recordio_read(r, &item); + if (size != -1) { + fail(); + } + + release_recordio_reader(r); +} diff --git a/paddle/go/recordio/chunk.go b/go/recordio/chunk.go similarity index 100% rename from paddle/go/recordio/chunk.go rename to go/recordio/chunk.go diff --git a/paddle/go/recordio/header.go b/go/recordio/header.go similarity index 100% rename from paddle/go/recordio/header.go rename to go/recordio/header.go diff --git a/paddle/go/recordio/reader.go b/go/recordio/range_scanner.go similarity index 84% rename from paddle/go/recordio/reader.go rename to go/recordio/range_scanner.go index a12c604f7b..46e2eee68c 100644 --- a/paddle/go/recordio/reader.go +++ b/go/recordio/range_scanner.go @@ -74,8 +74,8 @@ func (r *Index) Locate(recordIndex int) (int, int) { return -1, -1 } -// Scanner scans records in a specified range within [0, numRecords). -type Scanner struct { +// RangeScanner scans records in a specified range within [0, numRecords). +type RangeScanner struct { reader io.ReadSeeker index *Index start, end, cur int @@ -84,10 +84,10 @@ type Scanner struct { err error } -// NewScanner creates a scanner that sequencially reads records in the +// NewRangeScanner creates a scanner that sequencially reads records in the // range [start, start+len). If start < 0, it scans from the // beginning. If len < 0, it scans till the end of file. -func NewScanner(r io.ReadSeeker, index *Index, start, len int) *Scanner { +func NewRangeScanner(r io.ReadSeeker, index *Index, start, len int) *RangeScanner { if start < 0 { start = 0 } @@ -95,7 +95,7 @@ func NewScanner(r io.ReadSeeker, index *Index, start, len int) *Scanner { len = index.NumRecords() - start } - return &Scanner{ + return &RangeScanner{ reader: r, index: index, start: start, @@ -108,7 +108,7 @@ func NewScanner(r io.ReadSeeker, index *Index, start, len int) *Scanner { // Scan moves the cursor forward for one record and loads the chunk // containing the record if not yet. -func (s *Scanner) Scan() bool { +func (s *RangeScanner) Scan() bool { s.cur++ if s.cur >= s.end { @@ -124,12 +124,17 @@ func (s *Scanner) Scan() bool { } // Record returns the record under the current cursor. -func (s *Scanner) Record() []byte { +func (s *RangeScanner) Record() []byte { _, ri := s.index.Locate(s.cur) return s.chunk.records[ri] } -// Error returns the error that stopped Scan. -func (s *Scanner) Error() error { +// Err returns the first non-EOF error that was encountered by the +// Scanner. +func (s *RangeScanner) Err() error { + if s.err == io.EOF { + return nil + } + return s.err } diff --git a/paddle/go/recordio/recordio_internal_test.go b/go/recordio/recordio_internal_test.go similarity index 96% rename from paddle/go/recordio/recordio_internal_test.go rename to go/recordio/recordio_internal_test.go index e0f7dd0407..30e317925d 100644 --- a/paddle/go/recordio/recordio_internal_test.go +++ b/go/recordio/recordio_internal_test.go @@ -68,7 +68,7 @@ func TestWriteAndRead(t *testing.T) { 2*4)}, // two record legnths idx.chunkOffsets) - s := NewScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1) + s := NewRangeScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1) i := 0 for s.Scan() { assert.Equal(data[i], string(s.Record())) diff --git a/paddle/go/recordio/recordio_test.go b/go/recordio/recordio_test.go similarity index 87% rename from paddle/go/recordio/recordio_test.go rename to go/recordio/recordio_test.go index 8bf1b020ab..e4ef835afa 100644 --- a/paddle/go/recordio/recordio_test.go +++ b/go/recordio/recordio_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/PaddlePaddle/Paddle/paddle/go/recordio" + "github.com/PaddlePaddle/Paddle/go/recordio" ) func TestWriteRead(t *testing.T) { @@ -29,7 +29,7 @@ func TestWriteRead(t *testing.T) { t.Fatal("num record does not match:", idx.NumRecords(), total) } - s := recordio.NewScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1) + s := recordio.NewRangeScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1) i := 0 for s.Scan() { if !reflect.DeepEqual(s.Record(), make([]byte, i)) { @@ -66,7 +66,7 @@ func TestChunkIndex(t *testing.T) { for i := 0; i < total; i++ { newIdx := idx.ChunkIndex(i) - s := recordio.NewScanner(bytes.NewReader(buf.Bytes()), newIdx, -1, -1) + s := recordio.NewRangeScanner(bytes.NewReader(buf.Bytes()), newIdx, -1, -1) j := 0 for s.Scan() { if !reflect.DeepEqual(s.Record(), make([]byte, i)) { diff --git a/go/recordio/scanner.go b/go/recordio/scanner.go new file mode 100644 index 0000000000..865228ff65 --- /dev/null +++ b/go/recordio/scanner.go @@ -0,0 +1,140 @@ +package recordio + +import ( + "fmt" + "os" + "path/filepath" +) + +// Scanner is a scanner for multiple recordio files. +type Scanner struct { + paths []string + curFile *os.File + curScanner *RangeScanner + pathIdx int + end bool + err error +} + +// NewScanner creates a new Scanner. +func NewScanner(paths ...string) (*Scanner, error) { + var ps []string + for _, s := range paths { + match, err := filepath.Glob(s) + if err != nil { + return nil, err + } + + ps = append(ps, match...) + } + + if len(ps) == 0 { + return nil, fmt.Errorf("no valid path provided: %v", paths) + } + + return &Scanner{paths: ps}, nil +} + +// Scan moves the cursor forward for one record and loads the chunk +// containing the record if not yet. +func (s *Scanner) Scan() bool { + if s.err != nil { + return false + } + + if s.end { + return false + } + + if s.curScanner == nil { + more, err := s.nextFile() + if err != nil { + s.err = err + return false + } + + if !more { + s.end = true + return false + } + } + + curMore := s.curScanner.Scan() + s.err = s.curScanner.Err() + + if s.err != nil { + return curMore + } + + if !curMore { + err := s.curFile.Close() + if err != nil { + s.err = err + return false + } + s.curFile = nil + + more, err := s.nextFile() + if err != nil { + s.err = err + return false + } + + if !more { + s.end = true + return false + } + + return s.Scan() + } + return true +} + +// Err returns the first non-EOF error that was encountered by the +// Scanner. +func (s *Scanner) Err() error { + return s.err +} + +// Record returns the record under the current cursor. +func (s *Scanner) Record() []byte { + if s.curScanner == nil { + return nil + } + + return s.curScanner.Record() +} + +// Close release the resources. +func (s *Scanner) Close() error { + s.curScanner = nil + if s.curFile != nil { + err := s.curFile.Close() + s.curFile = nil + return err + } + return nil +} + +func (s *Scanner) nextFile() (bool, error) { + if s.pathIdx >= len(s.paths) { + return false, nil + } + + path := s.paths[s.pathIdx] + s.pathIdx++ + f, err := os.Open(path) + if err != nil { + return false, err + } + + idx, err := LoadIndex(f) + if err != nil { + f.Close() + return false, err + } + + s.curFile = f + s.curScanner = NewRangeScanner(f, idx, 0, -1) + return true, nil +} diff --git a/paddle/go/recordio/writer.go b/go/recordio/writer.go similarity index 100% rename from paddle/go/recordio/writer.go rename to go/recordio/writer.go diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index cf31b4a342..9898dc083e 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -9,9 +9,10 @@ add_subdirectory(pserver) add_subdirectory(trainer) add_subdirectory(scripts) -if(CMAKE_Go_COMPILER) - add_subdirectory(go) -endif() +# Do not build go directory until go cmake is working smoothly. +# if(CMAKE_Go_COMPILER) +# add_subdirectory(go) +# endif() find_package(Boost QUIET) diff --git a/paddle/go/CMakeLists.txt b/paddle/go/CMakeLists.txt deleted file mode 100644 index 51c5252d66..0000000000 --- a/paddle/go/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -go_library(adder SRCS adder.go) - -if (WITH_TESTING) - cc_test(cgo_test - SRCS - cgo_test.cc - DEPS - adder) -endif() diff --git a/paddle/go/adder.go b/paddle/go/adder.go deleted file mode 100644 index e14f40fd9f..0000000000 --- a/paddle/go/adder.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import "C" - -//export GoAdder -func GoAdder(x, y int) int { - return x + y -} - -func main() {} // Required but ignored diff --git a/paddle/go/cclient/CMakeLists.txt b/paddle/go/cclient/CMakeLists.txt deleted file mode 100644 index c85ff3db09..0000000000 --- a/paddle/go/cclient/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -cmake_minimum_required(VERSION 3.0) - -if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) - message("-- Found gtest (include: ${GTEST_INCLUDE_DIR}, library: ${GTEST_LIBRARIES})") -else() - # find cmake directory modules - get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) - get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) - get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) - - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") - - # enable c++11 - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") - - # enable gtest - set(THIRD_PARTY_PATH ./third_party) - set(WITH_TESTING ON) - include(external/gtest) -endif() - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - -project(cxx_go CXX C Go) - -include(cmake/golang.cmake) -include(cmake/flags.cmake) - -ExternalGoProject_Add(pserver github.com/PaddlePaddle/Paddle/paddle/go/pserver) -add_go_library(client STATIC pserver) -add_subdirectory(test) diff --git a/paddle/go/cclient/cmake/golang.cmake b/paddle/go/cclient/cmake/golang.cmake deleted file mode 100644 index 5d39868bfd..0000000000 --- a/paddle/go/cclient/cmake/golang.cmake +++ /dev/null @@ -1,46 +0,0 @@ -set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go") -file(MAKE_DIRECTORY ${GOPATH}) - -function(ExternalGoProject_Add TARG) - add_custom_target(${TARG} env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get ${ARGN}) -endfunction(ExternalGoProject_Add) - -function(add_go_executable NAME) - file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go") - add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp - COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build - -o "${CMAKE_CURRENT_BINARY_DIR}/${NAME}" - ${CMAKE_GO_FLAGS} ${GO_SOURCE} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) - - add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN}) - install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${NAME} DESTINATION bin) -endfunction(add_go_executable) - - -function(ADD_GO_LIBRARY NAME BUILD_TYPE) - if(BUILD_TYPE STREQUAL "STATIC") - set(BUILD_MODE -buildmode=c-archive) - set(LIB_NAME "lib${NAME}.a") - else() - set(BUILD_MODE -buildmode=c-shared) - if(APPLE) - set(LIB_NAME "lib${NAME}.dylib") - else() - set(LIB_NAME "lib${NAME}.so") - endif() - endif() - - file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go") - add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp - COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE} - -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}" - ${CMAKE_GO_FLAGS} ${GO_SOURCE} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) - - add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN}) - - if(NOT BUILD_TYPE STREQUAL "STATIC") - install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} DESTINATION bin) - endif() -endfunction(ADD_GO_LIBRARY) diff --git a/paddle/go/cgo_test.cc b/paddle/go/cgo_test.cc deleted file mode 100644 index 64efa606ff..0000000000 --- a/paddle/go/cgo_test.cc +++ /dev/null @@ -1,5 +0,0 @@ -#include -#include "gtest/gtest.h" -#include "libadder.h" - -TEST(Cgo, Invoke) { EXPECT_EQ(GoAdder(30, 12), 42); } diff --git a/paddle/go/pserver/client.go b/paddle/go/pserver/client.go deleted file mode 100644 index 1c98aea6d1..0000000000 --- a/paddle/go/pserver/client.go +++ /dev/null @@ -1,54 +0,0 @@ -package pserver - -// Client is the client to parameter servers. -type Client struct { -} - -// NewClient creates a new client. -func NewClient(addr string) *Client { - return &Client{} -} - -// BeginInitParams begins to initialize parameters on parameter -// servers. -// -// BeginInitParams will be called from multiple trainers, only one -// trainer will be selected to initialize the parameters on parameter -// servers. Other trainers will be blocked until the initialization is -// done, and they need to get the initialized parameters from -// parameter servers using GetParams. -func (c *Client) BeginInitParams(pserverConfigProto []byte) (selected bool, err error) { - return true, nil -} - -// InitParam initializes the parameter on parameter servers. -func (c *Client) InitParam(paramWithConfigs ParameterWithConfig) error { - return nil -} - -// FinishInitParams tells parameter servers client has sent all -// parameters to parameter servers as initialization. -func (c *Client) FinishInitParams() error { - return nil -} - -// SendGrads sends gradients to parameter servers for updating -// parameters. -func (c *Client) SendGrads(grads []Gradient) error { - return nil -} - -// GetParams gets parameters from parameter servers. -func (c *Client) GetParams(names []string) ([]Parameter, error) { - return nil, nil -} - -// SaveModel indicates parameters to save the parameter to the given -// path. -func (c *Client) SaveModel(path string) error { - return nil -} - -// Cleanup cleans up the client states. -func (c *Client) Cleanup() { -} diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 69d5830dd2..6bfdea3c6e 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -217,10 +217,10 @@ void SmoothL1CostLayer::forwardImp(Matrix& output, targetCpu->copyFrom(target); outputCpu->copyFrom(output); labelCpu->copyFrom(*label.value); - targetCpu->smoothL1(*outputCpu, *labelCpu); + targetCpu->smoothL1(*outputCpu, *labelCpu, 1.0); target.copyFrom(*targetCpu); } else { - target.smoothL1(output, *label.value); + target.smoothL1(output, *label.value, 1.0); } } @@ -238,10 +238,10 @@ void SmoothL1CostLayer::backwardImp(Matrix& output, outputGCpu->copyFrom(outputG); outputCpu->copyFrom(output); labelCpu->copyFrom(*label.value); - outputGCpu->smoothL1Bp(*outputCpu, *labelCpu); + outputGCpu->smoothL1Bp(*outputCpu, *labelCpu, 1.0); outputG.copyFrom(*outputGCpu); } else { - outputG.smoothL1Bp(output, *label.value); + outputG.smoothL1Bp(output, *label.value, 1.0); } } diff --git a/paddle/gserver/tests/sequence_nest_layer_group.conf b/paddle/gserver/tests/sequence_nest_layer_group.conf index c01b95f7a2..71ef53d08a 100644 --- a/paddle/gserver/tests/sequence_nest_layer_group.conf +++ b/paddle/gserver/tests/sequence_nest_layer_group.conf @@ -59,7 +59,7 @@ lstm_nest_group = recurrent_group( input=SubsequenceInput(emb_group), step=lstm_group, name="lstm_nest_group") # hasSubseq ->(seqlastins) seq lstm_last = last_seq( - input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) + input=lstm_nest_group, agg_level=AggregateLevel.TO_SEQUENCE) # seq ->(expand) hasSubseq lstm_expand = expand_layer( @@ -71,7 +71,7 @@ lstm_expand = expand_layer( lstm_average = pooling_layer( input=lstm_expand, pooling_type=AvgPooling(), - agg_level=AggregateLevel.EACH_SEQUENCE) + agg_level=AggregateLevel.TO_SEQUENCE) with mixed_layer( size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output: diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt index cb8bece00e..93e5e2c22f 100644 --- a/paddle/majel/CMakeLists.txt +++ b/paddle/majel/CMakeLists.txt @@ -1,6 +1,8 @@ cc_library(place SRCS place.cc) +cc_test(place_test SRCS place_test.cc DEPS place glog gflags) + cc_library(ddim SRCS ddim.cc) +cc_test(ddim_test SRCS ddim_test.cc DEPS ddim) -if(WITH_TESTING) - add_subdirectory(test) -endif() +nv_test(cuda_test SRCS cuda_test.cu) +nv_test(dim_test SRCS dim_test.cu DEPS ddim) diff --git a/paddle/majel/test/cuda_test.cu b/paddle/majel/cuda_test.cu similarity index 100% rename from paddle/majel/test/cuda_test.cu rename to paddle/majel/cuda_test.cu diff --git a/paddle/majel/test/ddim_test.cc b/paddle/majel/ddim_test.cc similarity index 100% rename from paddle/majel/test/ddim_test.cc rename to paddle/majel/ddim_test.cc diff --git a/paddle/majel/test/dim_test.cu b/paddle/majel/dim_test.cu similarity index 100% rename from paddle/majel/test/dim_test.cu rename to paddle/majel/dim_test.cu diff --git a/paddle/majel/test/place_test.cc b/paddle/majel/place_test.cc similarity index 92% rename from paddle/majel/test/place_test.cc rename to paddle/majel/place_test.cc index c5fa65ef6d..6a099ae6b6 100644 --- a/paddle/majel/test/place_test.cc +++ b/paddle/majel/place_test.cc @@ -1,7 +1,6 @@ #include "paddle/majel/place.h" #include #include "gtest/gtest.h" -#include "paddle/utils/Logging.h" TEST(Place, Equality) { majel::CpuPlace cpu; @@ -38,5 +37,4 @@ TEST(Place, Print) { ss << majel::CpuPlace(); EXPECT_EQ("CpuPlace", ss.str()); } - LOG(INFO) << "\n[----------] Done \n"; } diff --git a/paddle/majel/test/CMakeLists.txt b/paddle/majel/test/CMakeLists.txt deleted file mode 100644 index 9d632d568e..0000000000 --- a/paddle/majel/test/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -cc_test(place_test - SRCS place_test.cc - DEPS place) - -cc_test(ddim_test - SRCS ddim_test.cc - DEPS ddim) - -if(WITH_GPU) - nv_test(cuda_test SRCS cuda_test.cu) - nv_test(dim_test SRCS dim_test.cu DEPS ddim) -endif() diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 6ac61be0bf..c910146164 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -3606,7 +3606,7 @@ void CpuMatrix::sumOfSquaresBp(Matrix& output, Matrix& label) { } } -void CpuMatrix::smoothL1(Matrix& output, Matrix& label) { +void CpuMatrix::smoothL1(Matrix& output, Matrix& label, real destScale) { CHECK(output.useGpu_ == false && label.useGpu_ == false) << "Matrix type are not equal"; @@ -3624,6 +3624,7 @@ void CpuMatrix::smoothL1(Matrix& output, Matrix& label) { for (size_t i = 0; i < numSamples; ++i, out += dim, lbl += dim) { for (size_t j = 0; j < dim; ++j) { real absVal = std::fabs(out[j] - lbl[j]); + cost[i] *= destScale; if (absVal < 1.0) cost[i] += 0.5 * absVal * absVal; else @@ -3632,7 +3633,7 @@ void CpuMatrix::smoothL1(Matrix& output, Matrix& label) { } } -void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label) { +void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label, real destScale) { CHECK(output.useGpu_ == false && label.useGpu_ == false) << "Matrix type are not equal"; @@ -3650,6 +3651,7 @@ void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label) { for (size_t i = 0; i < numSamples; ++i, out += dim, grad += dim, lbl += dim) { for (size_t j = 0; j < dim; ++j) { real val = out[j] - lbl[j]; + grad[j] *= destScale; if (std::fabs(val) < 1) { grad[j] += val; } else { diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 3252adb19e..748be850b4 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -789,11 +789,11 @@ public: LOG(FATAL) << "Not implemented"; } - virtual void smoothL1(Matrix& output, Matrix& label) { + virtual void smoothL1(Matrix& output, Matrix& label, real destScale) { LOG(FATAL) << "Not implemented"; } - virtual void smoothL1Bp(Matrix& outputV, Matrix& label) { + virtual void smoothL1Bp(Matrix& outputV, Matrix& label, real destScale) { LOG(FATAL) << "Not implemented"; } @@ -1736,8 +1736,8 @@ public: /// gradient of sumOfSquares. void sumOfSquaresBp(Matrix& outputV, Matrix& label); - void smoothL1(Matrix& output, Matrix& label); - void smoothL1Bp(Matrix& output, Matrix& label); + void smoothL1(Matrix& output, Matrix& label, real destScale); + void smoothL1Bp(Matrix& output, Matrix& label, real destScale); void tanh(Matrix& output); void tanhDerivative(Matrix& output); diff --git a/paddle/parameter/Parameter.h b/paddle/parameter/Parameter.h index d77486ce42..0bac76f068 100644 --- a/paddle/parameter/Parameter.h +++ b/paddle/parameter/Parameter.h @@ -324,6 +324,7 @@ protected: std::vector> updaterHooks_; public: + void setSharedCount(int cnt) { sharedCount_ = cnt; } int getSharedCount() { return sharedCount_; } bool isSparse() { return config_.is_sparse(); } diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 2dfa712427..9f0f9f2d74 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -21,6 +21,8 @@ cd /paddle/build # build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true +# delete previous built whl packages +rm -rf /paddle/paddle/dist 2>/dev/null || true cat < /// for GCC/Clang #define CPUID(info, x) __cpuid_count(x, 0, info[0], info[1], info[2], info[3]) @@ -32,7 +32,7 @@ limitations under the License. */ namespace paddle { SIMDFlags::SIMDFlags() { -#if defined(__arm__) +#if defined(__arm__) || defined(__aarch64__) simd_flags_ = SIMD_NEON; #else unsigned int cpuInfo[4]; diff --git a/paddle/utils/arch/linux/Locks.cpp b/paddle/utils/arch/linux/Locks.cpp index 310c9a6542..3a0903d1f2 100644 --- a/paddle/utils/arch/linux/Locks.cpp +++ b/paddle/utils/arch/linux/Locks.cpp @@ -55,8 +55,11 @@ public: }; #else - +// clang-format off +#include #include +// clang-format on + class SpinLockPrivate { public: inline void lock() { diff --git a/paddle/utils/tests/test_SIMDFlags.cpp b/paddle/utils/tests/test_SIMDFlags.cpp index 185789c927..a808d456a6 100644 --- a/paddle/utils/tests/test_SIMDFlags.cpp +++ b/paddle/utils/tests/test_SIMDFlags.cpp @@ -19,7 +19,7 @@ using namespace paddle; // NOLINT TEST(SIMDFlags, gccTest) { #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__)) && \ - !defined(__arm__) + !defined(__arm__) && !defined(__aarch64__) // clang-format off CHECK(!__builtin_cpu_supports("sse") != HAS_SSE); CHECK(!__builtin_cpu_supports("sse2") != HAS_SSE2); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index d80590210f..3775375c9b 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -3378,7 +3378,7 @@ def make_importer(config_dir, config_args): return Import -settings = dict( +DEFAULT_SETTING = dict( batch_size=None, mini_batch_size=None, algorithm='async_sgd', @@ -3411,6 +3411,8 @@ settings = dict( adam_beta2=0.999, adam_epsilon=1e-8, ) +settings = copy.deepcopy(DEFAULT_SETTING) + settings_deprecated = dict(usage_ratio=1., ) trainer_settings = dict( @@ -3551,10 +3553,8 @@ def update_g_config(): return g_config -def parse_config(trainer_config, config_arg_str): +def begin_parse(config_arg_str=''): ''' - @param trainer_config: can be a string of config file name or a function name - with config logic @param config_arg_str: a string of the form var1=val1,var2=val2. It will be passed to config script as a dictionary CONFIG_ARGS ''' @@ -3562,12 +3562,23 @@ def parse_config(trainer_config, config_arg_str): for hook in _parse_config_hooks: hook() - config_args = {} - logger.findCaller = find_caller logger.fatal = my_fatal g_config.model_config.type = "nn" + + global g_current_submodel, g_root_submodel + g_root_submodel = g_config.model_config.sub_models.add() + g_root_submodel.name = 'root' + g_root_submodel.is_recurrent_layer_group = False + g_current_submodel = g_root_submodel + + +def parse_config(trainer_config, config_arg_str): + begin_parse(config_arg_str) + + config_args = {} + if config_arg_str: config_args = dict([f.split('=') for f in config_arg_str.split(',')]) @@ -3580,14 +3591,6 @@ def parse_config(trainer_config, config_arg_str): extension_module = importlib(extension_module_name) g_extended_config_funcs = extension_module.get_config_funcs(g_config) - g_config.model_config.type = 'nn' - - global g_current_submodel, g_root_submodel - g_root_submodel = g_config.model_config.sub_models.add() - g_root_submodel.name = 'root' - g_root_submodel.is_recurrent_layer_group = False - g_current_submodel = g_root_submodel - if hasattr(trainer_config, '__call__'): trainer_config.func_globals.update( make_config_environment("", config_args)) diff --git a/python/paddle/trainer_config_helpers/config_parser.py b/python/paddle/trainer_config_helpers/config_parser.py deleted file mode 100644 index 4b91b8d282..0000000000 --- a/python/paddle/trainer_config_helpers/config_parser.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import paddle.trainer.config_parser as config_parser -''' -This file is a wrapper of formal config_parser. The main idea of this file is to -separete different config logic into different function, such as network configuration - and optimizer configuration. -''' - -__all__ = [ - "parse_trainer_config", "parse_network_config", "parse_optimizer_config" -] - - -def parse_trainer_config(trainer_conf, config_arg_str): - return config_parser.parse_config(trainer_conf, config_arg_str) - - -def parse_network_config(network_conf): - config = config_parser.parse_config(network_conf, '') - return config.model_config - - -def parse_optimizer_config(optimizer_conf): - config = config_parser.parse_config(optimizer_conf, '') - return config.opt_config diff --git a/python/paddle/trainer_config_helpers/config_parser_utils.py b/python/paddle/trainer_config_helpers/config_parser_utils.py index 681b177a55..ee5bbbfb2d 100644 --- a/python/paddle/trainer_config_helpers/config_parser_utils.py +++ b/python/paddle/trainer_config_helpers/config_parser_utils.py @@ -12,15 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import paddle.trainer.config_parser as config_parser +from paddle.proto.TrainerConfig_pb2 import OptimizationConfig ''' -This file is a wrapper of formal config_parser. The main idea of this file is to +This file is a wrapper of formal config_parser. The main idea of this file is to separete different config logic into different function, such as network configuration and optimizer configuration. ''' __all__ = [ - "parse_trainer_config", "parse_network_config", "parse_optimizer_config" + "parse_trainer_config", "parse_network_config", "parse_optimizer_config", + "reset_parser" ] @@ -34,5 +37,15 @@ def parse_network_config(network_conf, config_arg_str=''): def parse_optimizer_config(optimizer_conf, config_arg_str=''): - config = config_parser.parse_config(optimizer_conf, config_arg_str) - return config.opt_config + config_parser.settings = copy.deepcopy(config_parser.DEFAULT_SETTING) + optimizer_conf() + opt_config = OptimizationConfig() + for k, v in config_parser.settings.iteritems(): + if v is None: + continue + opt_config.__setattr__(k, v) + return opt_config + + +def reset_parser(): + config_parser.begin_parse() diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index ec81e1dc3d..81cce31fec 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -119,6 +119,7 @@ __all__ = [ 'eos_layer', 'smooth_l1_cost', 'layer_support', + 'multiplex_layer', ] @@ -185,6 +186,7 @@ class LayerType(object): MAXOUT = "maxout" SPP_LAYER = "spp" PAD_LAYER = "pad" + MULTIPLEX_LAYER = "multiplex" PRINT_LAYER = "print" PRIORBOX_LAYER = "priorbox" @@ -235,16 +237,19 @@ class AggregateLevel(object): Accordingly, AggregateLevel supports two modes: - - :code:`AggregateLevel.EACH_TIMESTEP` means the aggregation acts on each + - :code:`AggregateLevel.TO_NO_SEQUENCE` means the aggregation acts on each timestep of a sequence, both :code:`SUB_SEQUENCE` and :code:`SEQUENCE` will be aggregated to :code:`NO_SEQUENCE`. - - :code:`AggregateLevel.EACH_SEQUENCE` means the aggregation acts on each + - :code:`AggregateLevel.TO_SEQUENCE` means the aggregation acts on each sequence of a nested sequence, :code:`SUB_SEQUENCE` will be aggregated to :code:`SEQUENCE`. """ - EACH_TIMESTEP = 'non-seq' - EACH_SEQUENCE = 'seq' + TO_NO_SEQUENCE = 'non-seq' + TO_SEQUENCE = 'seq' + # compatible with previous configuration + EACH_TIMESTEP = TO_NO_SEQUENCE + EACH_SEQUENCE = TO_SEQUENCE class LayerOutput(object): @@ -285,6 +290,7 @@ class LayerOutput(object): assert size is not None assert LayerType.is_layer_type(layer_type) self.name = name + self.full_name = MakeLayerNameInSubmodel(name) self.layer_type = layer_type if parents is not None and type(parents) != list: parents = [parents] @@ -479,7 +485,7 @@ def table_projection(input, size=0, param_attr=None): return proj -def identity_projection(input, offset=None): +def identity_projection(input, offset=None, size=None): """ 1. IdentityProjection if offset=None. It performs: @@ -520,8 +526,10 @@ def identity_projection(input, offset=None): proj = IdentityProjection(input_layer_name=input.name) proj.origin = input else: + if size is None: + size = input.size - offset proj = IdentityOffsetProjection( - input_layer_name=input.name, offset=offset) + input_layer_name=input.name, offset=offset, size=size) proj.origin = input return proj @@ -1078,7 +1086,7 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, - agg_level=AggregateLevel.EACH_TIMESTEP, + agg_level=AggregateLevel.TO_NO_SEQUENCE, layer_attr=None): """ Pooling layer for sequence inputs, not used for Image. @@ -1089,10 +1097,10 @@ def pooling_layer(input, seq_pool = pooling_layer(input=layer, pooling_type=AvgPooling(), - agg_level=AggregateLevel.EACH_SEQUENCE) + agg_level=AggregateLevel.TO_NO_SEQUENCE) - :param agg_level: AggregateLevel.EACH_TIMESTEP or - AggregateLevel.EACH_SEQUENCE + :param agg_level: AggregateLevel.TO_NO_SEQUENCE or + AggregateLevel.TO_SEQUENCE :type agg_level: AggregateLevel :param name: layer name. :type name: basestring @@ -1362,7 +1370,7 @@ def grumemory(input, @layer_support() def last_seq(input, name=None, - agg_level=AggregateLevel.EACH_TIMESTEP, + agg_level=AggregateLevel.TO_NO_SEQUENCE, stride=-1, layer_attr=None): """ @@ -1397,7 +1405,7 @@ def last_seq(input, " series information at all. Maybe you want to use" " first_seq instead.") - if agg_level == AggregateLevel.EACH_SEQUENCE: + if agg_level == AggregateLevel.TO_SEQUENCE: assert stride == -1 Layer( @@ -1418,7 +1426,7 @@ def last_seq(input, @layer_support() def first_seq(input, name=None, - agg_level=AggregateLevel.EACH_TIMESTEP, + agg_level=AggregateLevel.TO_NO_SEQUENCE, stride=-1, layer_attr=None): """ @@ -1454,7 +1462,7 @@ def first_seq(input, ' time series information at all. Maybe you want to use' ' last_seq instead.') - if agg_level == AggregateLevel.EACH_SEQUENCE: + if agg_level == AggregateLevel.TO_SEQUENCE: assert stride == -1 Layer( @@ -1477,16 +1485,18 @@ class ExpandLevel(object): ExpandLevel supports two modes: - - :code:`ExpandLevel.FROM_TIMESTEP` means the expandation acts on each - timestep of a sequence, :code:`NO_SEQUENCE` will be expanded to + - :code:`ExpandLevel.FROM_NO_SEQUENCE` means the expansion acts on + :code:`NO_SEQUENCE`, which will be expanded to :code:`SEQUENCE` or :code:`SUB_SEQUENCE`. - - :code:`ExpandLevel.FROM_SEQUENCE` means the expandation acts on each - sequence of a nested sequence, :code:`SEQUENCE` will be expanded to + - :code:`ExpandLevel.FROM_SEQUENCE` means the expansion acts on + :code:`SEQUENCE`, which will be expanded to :code:`SUB_SEQUENCE`. """ - FROM_TIMESTEP = AggregateLevel.EACH_TIMESTEP - FROM_SEQUENCE = AggregateLevel.EACH_SEQUENCE + FROM_NO_SEQUENCE = AggregateLevel.TO_NO_SEQUENCE + FROM_SEQUENCE = AggregateLevel.TO_SEQUENCE + # compatible with previous configuration + FROM_TIMESTEP = FROM_NO_SEQUENCE @wrap_name_default() @@ -1495,7 +1505,7 @@ def expand_layer(input, expand_as, name=None, bias_attr=False, - expand_level=ExpandLevel.FROM_TIMESTEP, + expand_level=ExpandLevel.FROM_NO_SEQUENCE, layer_attr=None): """ A layer for "Expand Dense data or (sequence data where the length of each @@ -1507,7 +1517,7 @@ def expand_layer(input, expand = expand_layer(input=layer1, expand_as=layer2, - expand_level=ExpandLevel.FROM_TIMESTEP) + expand_level=ExpandLevel.FROM_NO_SEQUENCE) :param input: Input layer :type input: LayerOutput @@ -2794,7 +2804,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): if layer_type == LayerType.CONCAT_LAYER: assert not bias_attr - Layer( + layer = Layer( name=name, type=layer_type, inputs=[x.name for x in input] if is_concat_layer else input, @@ -2802,13 +2812,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): bias=ParamAttr.to_bias(bias_attr), **ExtraLayerAttribute.to_kwargs(layer_attr)) - sz = 0 - for each_input in input: - if each_input.size is not None: - sz += each_input.size - else: - sz = None - break + sz = layer.config.size return LayerOutput( name, @@ -2976,7 +2980,7 @@ def memory(name, @layer_support() def lstm_step_layer(input, state, - size, + size=None, act=None, name=None, gate_act=None, @@ -3042,6 +3046,9 @@ def lstm_step_layer(input, :return: LayerOutput object. :rtype: LayerOutput """ + + assert size is None or state.size == size + size = state.size Layer( name=name, type=LayerType.LSTM_STEP_LAYER, @@ -3049,7 +3056,7 @@ def lstm_step_layer(input, active_gate_type=gate_act.name, active_state_type=state_act.name, bias=ParamAttr.to_bias(bias_attr), - size=size, + size=state.size, inputs=[input.name, state.name], **ExtraLayerAttribute.to_kwargs(layer_attr)) @@ -3489,6 +3496,11 @@ def recurrent_group(step, RecurrentLayerGroupEnd(name=name) + for layer_out in layer_outs: + # Thee previous full_name is the name is the rnn group + # We need a full_name outside the rnn group + layer_out.full_name = MakeLayerNameInSubmodel(layer_out.name) + if len(layer_outs) == 1: return layer_outs[0] else: @@ -5465,3 +5477,54 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput( name, LayerType.SMOOTH_L1, parents=[input, label], size=1) + + +@wrap_name_default() +def multiplex_layer(input, name=None, layer_attr=None): + """ + This layer multiplex multiple layers according to the index, + which is provided by the first input layer. + inputs[0]: the index of the layer to output of size batchSize. + inputs[1:N]; the candidate output data. + For each index i from 0 to batchSize -1, the output is the i-th row of the + (index[i] + 1)-th layer. + + For each i-th row of output: + .. math:: + y[i][j] = x_{x_{0}[i] + 1}[i][j], j = 0,1, ... , (x_{1}.width - 1) + + where, y is output. :math:`x_{k}` is the k-th input layer and + :math:`k = x_{0}[i] + 1`. + + .. code-block:: python + + maxid = multiplex_layer(input=layers) + + :param input: Input layers. + :type input: list of LayerOutput + :param name: Layer name. + :type name: basestring + :param layer_attr: extra layer attributes. + :type layer_attr: ExtraLayerAttribute. + :return: LayerOutput object. + :rtype: LayerOutput + """ + + assert isinstance(input, collections.Sequence) + assert len(input) > 2, 'multiplex_layer should have more than 2 inputs' + for i in range(1, len(input)): + assert isinstance(input[i], LayerOutput) + assert input[i].size == input[1].size, \ + "All the input layers except the first one should have the same size" + + l = Layer( + name=name, + type='multiplex', + inputs=[x.name for x in input], + size=input[1].size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.MULTIPLEX_LAYER, + parents=input, + size=l.config.size) diff --git a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh index c5dc8e1aab..981ccbf248 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh @@ -5,6 +5,6 @@ last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers img_trans_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm shared_gru test_cost_layers_with_weight test_spp_layer test_bilinear_interp test_maxout test_bi_grumemory math_ops -test_seq_concat_reshape test_pad test_smooth_l1) +test_seq_concat_reshape test_pad test_smooth_l1 test_multiplex_layer) export whole_configs=(test_split_datasource) diff --git a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py index 3c6dbc95e5..f87237f9b5 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py +++ b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py @@ -6,7 +6,7 @@ din = data_layer(name='data', size=30) seq_op = [first_seq, last_seq] -agg_level = [AggregateLevel.EACH_SEQUENCE, AggregateLevel.EACH_TIMESTEP] +agg_level = [AggregateLevel.TO_SEQUENCE, AggregateLevel.TO_NO_SEQUENCE] opts = [] @@ -15,6 +15,7 @@ for op in seq_op: opts.append(op(input=din, agg_level=al)) for op in seq_op: - opts.append(op(input=din, agg_level=AggregateLevel.EACH_TIMESTEP, stride=5)) + opts.append( + op(input=din, agg_level=AggregateLevel.TO_NO_SEQUENCE, stride=5)) outputs(opts) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_multiplex_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_multiplex_layer.protostr new file mode 100644 index 0000000000..379842ba8d --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_multiplex_layer.protostr @@ -0,0 +1,63 @@ +type: "nn" +layers { + name: "index" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "data1" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "data2" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "data3" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "__multiplex_layer_0__" + type: "multiplex" + size: 30 + active_type: "" + inputs { + input_layer_name: "index" + } + inputs { + input_layer_name: "data1" + } + inputs { + input_layer_name: "data2" + } + inputs { + input_layer_name: "data3" + } +} +input_layer_names: "index" +input_layer_names: "data1" +input_layer_names: "data2" +input_layer_names: "data3" +output_layer_names: "__multiplex_layer_0__" +sub_models { + name: "root" + layer_names: "index" + layer_names: "data1" + layer_names: "data2" + layer_names: "data3" + layer_names: "__multiplex_layer_0__" + input_layer_names: "index" + input_layer_names: "data1" + input_layer_names: "data2" + input_layer_names: "data3" + output_layer_names: "__multiplex_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py index 81e5161ebc..c53f10e0a4 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py @@ -9,4 +9,6 @@ outputs( expand_layer( input=din, expand_as=data_seq, expand_level=ExpandLevel.FROM_SEQUENCE), expand_layer( - input=din, expand_as=data_seq, expand_level=ExpandLevel.FROM_TIMESTEP)) + input=din, + expand_as=data_seq, + expand_level=ExpandLevel.FROM_NO_SEQUENCE)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py new file mode 100644 index 0000000000..d250001932 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py @@ -0,0 +1,12 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=1000, learning_rate=1e-5) + +index = data_layer(name='index', size=1) +din1 = data_layer(name='data1', size=30) +din2 = data_layer(name='data2', size=30) +din3 = data_layer(name='data3', size=30) + +dout = multiplex_layer([index, din1, din2, din3]) + +outputs(dout) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py index f67b6364d8..3c49eb56c1 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py @@ -6,7 +6,7 @@ din = data_layer(name='dat_in', size=100) POOL_TYPE = [MaxPooling, AvgPooling, SumPooling] -AGG_LEVEL = [AggregateLevel.EACH_SEQUENCE, AggregateLevel.EACH_TIMESTEP] +AGG_LEVEL = [AggregateLevel.TO_SEQUENCE, AggregateLevel.TO_NO_SEQUENCE] opts = [] diff --git a/python/paddle/utils/make_model_diagram.py b/python/paddle/utils/make_model_diagram.py index 1370ea83a4..40f99075de 100644 --- a/python/paddle/utils/make_model_diagram.py +++ b/python/paddle/utils/make_model_diagram.py @@ -39,6 +39,10 @@ def make_layer_label(layer_config): def make_diagram(config_file, dot_file, config_arg_str): config = parse_config(config_file, config_arg_str) + make_diagram_from_proto(config.model_config, dot_file) + + +def make_diagram_from_proto(model_config, dot_file): # print >> sys.stderr, config name2id = {} f = open(dot_file, 'w') @@ -59,12 +63,12 @@ def make_diagram(config_file, dot_file, config_arg_str): print >> f, 'digraph graphname {' print >> f, 'node [width=0.375,height=0.25];' - for i in xrange(len(config.model_config.layers)): - l = config.model_config.layers[i] + for i in xrange(len(model_config.layers)): + l = model_config.layers[i] name2id[l.name] = i i = 0 - for sub_model in config.model_config.sub_models: + for sub_model in model_config.sub_models: if sub_model.name == 'root': continue print >> f, 'subgraph cluster_%s {' % i @@ -78,18 +82,18 @@ def make_diagram(config_file, dot_file, config_arg_str): for layer_name in sub_model.layer_names: submodel_layers.add(layer_name) lid = name2id[layer_name] - layer_config = config.model_config.layers[lid] + layer_config = model_config.layers[lid] label = make_layer_label(layer_config) print >> f, 'l%s [label="%s", shape=box];' % (lid, label) print >> f, '}' - for i in xrange(len(config.model_config.layers)): - l = config.model_config.layers[i] + for i in xrange(len(model_config.layers)): + l = model_config.layers[i] if l.name not in submodel_layers: label = make_layer_label(l) print >> f, 'l%s [label="%s", shape=box];' % (i, label) - for sub_model in config.model_config.sub_models: + for sub_model in model_config.sub_models: if sub_model.name == 'root': continue for link in sub_model.in_links: @@ -99,8 +103,8 @@ def make_diagram(config_file, dot_file, config_arg_str): for mem in sub_model.memories: print >> f, make_mem(mem) - for i in xrange(len(config.model_config.layers)): - for l in config.model_config.layers[i].inputs: + for i in xrange(len(model_config.layers)): + for l in model_config.layers[i].inputs: print >> f, 'l%s -> l%s [label="%s"];' % ( name2id[l.input_layer_name], i, l.input_parameter_name) diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 851fe7060f..b9d0a7f291 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -27,6 +27,7 @@ from . import dataset from . import reader from . import plot import attr +import op import pooling import inference import networks diff --git a/python/paddle/v2/config_base.py b/python/paddle/v2/config_base.py index acda778e0a..d9613e001a 100644 --- a/python/paddle/v2/config_base.py +++ b/python/paddle/v2/config_base.py @@ -14,206 +14,55 @@ import collections import re -from paddle.trainer_config_helpers.default_decorators import wrap_name_default import paddle.trainer_config_helpers as conf_helps -from topology import Topology - - -class LayerType(type): - def __new__(cls, name, bases, attrs): - method_name = attrs.get('METHOD_NAME', None) - if method_name is not None: - method = getattr(conf_helps, method_name) - if method.__doc__ is not None: - mapper = attrs.get("__map_docstr__", None) - if mapper is not None: - attrs['__doc__'] = LayerType.__map_docstr__( - mapper(method.__doc__), - method_name=method_name, - name=name) - else: - attrs['__doc__'] = LayerType.__map_docstr__( - method.__doc__, method_name=method_name, name=name) - return super(LayerType, cls).__new__(cls, name, bases, attrs) - - @staticmethod - def __map_docstr__(doc, name, method_name): - assert isinstance(doc, basestring) - - # replace LayerOutput to paddle.v2.config_base.Layer - doc = doc.replace("LayerOutput", "paddle.v2.config_base.Layer") - - doc = doc.replace('ParameterAttribute', - 'paddle.v2.attr.ParameterAttribute') - - doc = re.sub(r'ExtraLayerAttribute[^\s]?', - 'paddle.v2.attr.ExtraAttribute', doc) - - # xxx_layer to xxx - doc = re.sub(r"(?P[a-z]+)_layer", r"\g", doc) - - # XxxxActivation to paddle.v2.Activation.Xxxx - doc = re.sub(r"(?P[A-Z][a-zA-Z]+)Activation", - r"paddle.v2.Activation.\g", doc) - - # TODO(yuyang18): Add more rules if needed. + +__layer_map__ = {} + + +def __map_docstr__(doc, name): + if doc is None: return doc + assert isinstance(doc, basestring) + + # replace LayerOutput to paddle.v2.config_base.Layer + doc = doc.replace("LayerOutput", "paddle.v2.config_base.Layer") + + doc = doc.replace('ParameterAttribute', 'paddle.v2.attr.ParameterAttribute') + + doc = re.sub(r'ExtraLayerAttribute[^\s]?', 'paddle.v2.attr.ExtraAttribute', + doc) + + # xxx_layer to xxx + doc = re.sub(r"(?P[a-z]+)_layer", r"\g", doc) + + # XxxxActivation to paddle.v2.activation.Xxxx + doc = re.sub(r"(?P[A-Z][a-zA-Z]+)Activation", + r"paddle.v2.activation.\g", doc) + + # xxx_evaluator to paddle.v2.evaluator.xxx + doc = re.sub(r"(?P[a-z]+)_evaluator", r"evaluator.\g", doc) + + # TODO(yuyang18): Add more rules if needed. + return doc + + +def __convert_to_v2__(f, name, module): + def wrapped(*args, **xargs): + out = f(*args, **xargs) + outs = out + if not isinstance(out, collections.Sequence): + outs = [out] + for l in outs: + if isinstance(l, conf_helps.LayerOutput): + __layer_map__[l.full_name] = l + return out + + wrapped.__doc__ = __map_docstr__(f.__doc__, name) + wrapped.__name__ = name + wrapped.__module__ = module + + return wrapped + -class Layer(object): - __metaclass__ = LayerType - - def __init__(self, name=None, parent_layers=None): - assert isinstance(parent_layers, dict) - self.name = name - self.__context__ = {} - self.__parent_layers__ = parent_layers - # some layer may have some extra parent layer - self.__extra_parent__ = [] - # used for evaluator. - self.__children_layers__ = [] - - def extra_parent(self): - return self.__extra_parent__ - - def append_extra_parent(self, parent): - self.__extra_parent__.append(parent) - - def append_child(self, layer, parent_names): - self.__children_layers__.append((layer, parent_names)) - - def to_proto(self, context): - """ - function to set proto attribute - """ - self.__context__ = context - - # STEP: short cut if this layer is parsed before. - if self.context_name() in context: - if self.use_context_name(): - return context[self.context_name()] - else: - return context[self.name] - - # STEP: parse extra_parent that is not used by this layer but must - # be parsed before this layer. - for p in self.__extra_parent__: - p.to_proto(context=context) - - # STEP: parse parent that is used by this layer, get the result and - # insert into kwargs of the next layer's to_proto_impl method. - kwargs = dict() - for layer_name in self.__parent_layers__: - if not isinstance(self.__parent_layers__[layer_name], - collections.Sequence): - v1_layer = self.__parent_layers__[layer_name].to_proto( - context=context) - else: - v1_layer = map(lambda x: x.to_proto(context=context), - self.__parent_layers__[layer_name]) - kwargs[layer_name] = v1_layer - - # STEP: parse myself and add myself into context. - ret_val = self.to_proto_impl(**kwargs) - if self.context_name() is not None \ - and self.context_name() not in context: - context[self.context_name()] = ret_val - - # STEP: parse children that should be pased after this layer. - for layer, pnames in self.__children_layers__: - drop = False - - # child will only be parsed if all parents are in context. - for pname in pnames: - if pname not in context: - drop = True - break - if drop: - continue - layer.to_proto(context=context) - - # STEP: return v1 layer result - if self.context_name() is None: - return ret_val - elif self.use_context_name(): - return context[self.context_name()] - else: - return context[self.name] - - def to_proto_impl(self, **kwargs): - raise NotImplementedError() - - def context_name(self): - """ - Context name means the context which stores `to_proto_impl` result. - If multiple layer share same context_name, the `to_proto_impl` of them - will be invoked only once. - """ - return self.name - - def use_context_name(self): - return False - - def calculate_size(self): - """ - lazy calculate size of the layer, should be called when to_proto_impl of - this layer is called. - :return: - """ - return self.__context__[self.context_name()].size - - def attr(self): - topo = Topology(self) - return topo.get_layer_proto(self.name) - - -def __convert_to_v2__(method_name, - parent_names, - is_default_name=True, - attach_parent=False): - if is_default_name: - wrapper = wrap_name_default(name_prefix=method_name) - else: - wrapper = None - - class V2LayerImpl(Layer): - METHOD_NAME = method_name - - def __init__(self, **kwargs): - parent_layers = dict() - other_kwargs = dict() - for pname in parent_names: - if pname in kwargs: - parent_layers[pname] = kwargs[pname] - - if attach_parent: - pnames = [x.context_name() for x in parent_layers.values()] - - for pname in parent_layers: - layers = kwargs[pname] - if not isinstance(layers, collections.Sequence): - layers = [layers] - - for layer in layers: - layer.append_child(self, pnames) - - for key in kwargs.keys(): - if key not in parent_names: - other_kwargs[key] = kwargs[key] - - name = kwargs.get('name', None) - super(V2LayerImpl, self).__init__(name, parent_layers) - self.__other_kwargs__ = other_kwargs - - if wrapper is not None: - __init__ = wrapper(__init__) - - def to_proto_impl(self, **kwargs): - args = dict() - for each in kwargs: - args[each] = kwargs[each] - for each in self.__other_kwargs__: - args[each] = self.__other_kwargs__[each] - return getattr(conf_helps, method_name)(**args) - - return V2LayerImpl +Layer = conf_helps.LayerOutput diff --git a/python/paddle/v2/evaluator.py b/python/paddle/v2/evaluator.py index 588eefa391..eaaadbe53b 100644 --- a/python/paddle/v2/evaluator.py +++ b/python/paddle/v2/evaluator.py @@ -13,8 +13,8 @@ # limitations under the License. import paddle.trainer_config_helpers.evaluators as evs -import inspect from config_base import __convert_to_v2__ +import inspect __all__ = [] @@ -25,21 +25,10 @@ def initialize(): for __ev_name__ in filter(lambda x: x.endswith('_evaluator'), evs.__all__): __ev__ = getattr(evs, __ev_name__) - if hasattr(__ev__, 'argspec'): - argspec = __ev__.argspec - else: - argspec = inspect.getargspec(__ev__) - parent_names = filter(lambda x: x in ['input', 'label', 'weight'], - argspec.args) - v2_ev = __convert_to_v2__( - __ev_name__, - parent_names=parent_names, - is_default_name='name' in argspec.args, - attach_parent=True) - __new_name__ = convert_to_new_name(__ev_name__) - globals()[__new_name__] = v2_ev + globals()[__new_name__] = __convert_to_v2__(__ev__, __new_name__, + __name__) globals()[__new_name__].__name__ = __new_name__ __all__.append(__new_name__) diff --git a/python/paddle/v2/inference.py b/python/paddle/v2/inference.py index 139339902e..34b7308601 100644 --- a/python/paddle/v2/inference.py +++ b/python/paddle/v2/inference.py @@ -12,9 +12,9 @@ class Inference(object): """ Inference combines neural network output and parameters together to do inference. - + .. code-block:: python - + inferer = Inference(output_layer=prediction, parameters=parameters) for data_batch in batches: print inferer.infer(data_batch) @@ -92,8 +92,8 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): .. code-block:: python - result = paddle.infer(output_layer=prediction, - parameters=parameters, + result = paddle.infer(output_layer=prediction, + parameters=parameters, input=SomeData) print result @@ -101,14 +101,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): .. code-block:: python - result = paddle.infer(output_layer=[prediction1, prediction2], - parameters=parameters, + result = paddle.infer(output_layer=[prediction1, prediction2], + parameters=parameters, input=SomeData, field=[id, value]]) print result :param output_layer: output of the neural network that would be inferred - :type output_layer: paddle.v2.config_base.Layer or a list of + :type output_layer: paddle.v2.config_base.Layer or a list of paddle.v2.config_base.Layer :param parameters: parameters of the neural network. :type parameters: paddle.v2.parameters.Parameters @@ -117,14 +117,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): :type input: collections.Iterable :param feeding: Reader dictionary. Default could generate from input value. - :param field: The prediction field. It should in [`value`, `id`, `prob`]. - `value` and `prob` mean return the prediction probabilities, + :param field: The prediction field. It should in [`value`, `id`, `prob`]. + `value` and `prob` mean return the prediction probabilities, `id` means return the prediction labels. Default is `value`. - Note that `prob` only used when output_layer is beam_search + Note that `prob` only used when output_layer is beam_search or max_id. :type field: str - :return: The prediction result. If there are multiple outout_layers and fields, - the return order is outout_layer1.field1, outout_layer2.field1, ..., + :return: The prediction result. If there are multiple outout_layers and fields, + the return order is outout_layer1.field1, outout_layer2.field1, ..., outout_layer1.field2, outout_layer2.field2 ... :rtype: numpy.ndarray """ diff --git a/python/paddle/v2/layer.py b/python/paddle/v2/layer.py index 919c531d18..da2abdd2d1 100644 --- a/python/paddle/v2/layer.py +++ b/python/paddle/v2/layer.py @@ -32,392 +32,29 @@ The primary usage shows below. """ import collections -import inspect +import copy import re +import paddle.trainer_config_helpers.layers as v1_layers +import paddle.trainer.config_parser as cp +from paddle.proto.ModelConfig_pb2 import ModelConfig, SubModelConfig +from config_base import __convert_to_v2__ +import config_base -import paddle.trainer_config_helpers as conf_helps -from paddle.trainer.config_parser import \ - RecurrentLayerGroupWithoutOutLinksBegin, RecurrentLayerGroupSetOutLink, \ - RecurrentLayerGroupEnd, model_type -from paddle.trainer_config_helpers.config_parser_utils import \ - parse_network_config as __parse__ -from paddle.trainer_config_helpers.default_decorators import wrap_act_default -from paddle.trainer_config_helpers.default_decorators import \ - wrap_bias_attr_default -from paddle.trainer_config_helpers.default_decorators import wrap_name_default -from paddle.trainer_config_helpers.layers import RecurrentLayerGroupSetGenerator, Generator -from paddle.trainer_config_helpers.layers import layer_support +__all__ = ['data', 'parse_network'] -import activation -import attr -import data_type -from config_base import Layer, __convert_to_v2__ -__all__ = ['parse_network', 'data'] +def __need_to_keep__(name): + if name in ['StaticInput', 'LayerType', 'layer_support']: + return False + return True -def parse_network(output_layers, extra_layers=None): - """ - Parse all layers in the neural network graph and - then generate a ModelConfig object. - - .. note:: - - This function is used internally in paddle.v2 module. User should never - invoke this method. - - :param output_layers: Output layers. - :type output_layers: Layer - :param extra_layers: Some layers in the neural network graph are not in the - path of output_layers. - :type extra_layers: Layer - :return: A ModelConfig object instance. - :rtype: ModelConfig - """ - if not isinstance(output_layers, collections.Sequence): - output_layers = [output_layers] - if extra_layers is not None and not isinstance(extra_layers, - collections.Sequence): - extra_layers = [extra_layers] - - def __real_func__(): - """ - __real_func__ is the function that config_parser.parse invoked. It is - the plain old paddle configuration function. - """ - context = dict() - real_output = [each.to_proto(context=context) for each in output_layers] - if extra_layers is not None: - extra_output = [ - each.to_proto(context=context) for each in extra_layers - ] - conf_helps.outputs(real_output) +def __need_to_wrap__(name): + return name not in ['AggregateLevel', 'ExpandLevel'] - return __parse__(__real_func__) - -""" -Some layer may need some special config, and can not use __convert_to_v2__ to convert. -So we also need to implement some special LayerV2. -""" - - -class DataLayerV2(Layer): - METHOD_NAME = 'data_layer' - - def __init__(self, name, type, **kwargs): - assert isinstance(type, data_type.InputType) - - self.type = type - self.__method_name__ = 'data_layer' - self.__kwargs__ = kwargs - - super(DataLayerV2, self).__init__(name=name, parent_layers=dict()) - - def to_proto_impl(self, **kwargs): - args = dict() - args['size'] = self.type.dim - for each in kwargs: - args[each] = kwargs[each] - for each in self.__kwargs__: - args[each] = self.__kwargs__[each] - return getattr(conf_helps, self.__method_name__)(name=self.name, **args) - - def __map_docstr__(doc): - doc = re.sub(r'(data = [^\)]+)\).*', - "data = paddle.layer.data(name=\"input\", " - "type=paddle.data_type.dense_vector(1000))", doc) - - doc = re.sub(r':param size:.*', - ':param type: Data type of this data layer', doc) - doc = re.sub(r':type size:.*', - ":type size: paddle.v2.data_type.InputType", doc) - return doc - - -class MemoryV2(Layer): - def __init__(self, name, extra_input=None, **kwargs): - """ - Init memory object, if memory is inited inside recurrent_group step - function, it may depend on a boot_layer that should be initialized - outside recurrent_group, so we: - 1. add RecurrentLayerInput to extra_parent of self. - 2. add boot_layer to the extra_parent of RecurrentLayerInput. - - :param extra_input: list of RecurrentLayerInput - :type extra_input: [RecurrentLayerInput] - """ - self.name = name - super(MemoryV2, self).__init__(name=name, parent_layers=dict()) - self.__kwargs__ = kwargs - self.__boot_layer_name__ = None - - if 'boot_layer' in kwargs: - begin_of_current_rnn = [] - # TODO(yuyang18): Fix inspect, it could be wrong when user invoke a - # function inside step. - st = inspect.stack() - for i in xrange(len(st)): - locs = inspect.stack()[i][0].f_locals - keys = locs.keys() - for key in keys: - val = locs[key] - if isinstance(val, RecurrentLayerInput): - begin_of_current_rnn.append(val) - elif isinstance(val, collections.Sequence): - for v in val: - if isinstance(v, RecurrentLayerInput): - begin_of_current_rnn.append(v) - - if begin_of_current_rnn: - break - assert begin_of_current_rnn is not None - for extra in begin_of_current_rnn: - self.append_extra_parent(extra) - extra.append_extra_parent(kwargs['boot_layer']) - self.__boot_layer_name__ = kwargs['boot_layer'].name - - def to_proto_impl(self, **kwargs): - args = dict() - for each in kwargs: - args[each] = kwargs[each] - for each in self.__kwargs__: - args[each] = self.__kwargs__[each] - - if self.__boot_layer_name__ is not None: - args['boot_layer'] = self.__context__[self.__boot_layer_name__] - - size = args.get('size', None) - if size is not None: - if callable(size): - real_size = size() - else: - real_size = size - args['size'] = real_size - return conf_helps.memory(name=self.name, **args) - - def context_name(self): - return self.name + "#memory" - - def use_context_name(self): - """ - memory layer will have the same name with some layer - :return: - """ - return True - - -class StaticInputV2(object): - def __init__(self, input, is_seq=False, size=None): - assert isinstance(input, LayerV2) - self.name = input.name - self.input = input - self.is_seq = is_seq - self.size = size - # TODO(add size check) - # assert input.size is not None or size is not None - - -class BaseGeneratedInputV2(object): - def __init__(self): - self.bos_id = None - self.eos_id = None - - def before_real_step(self): - raise NotImplementedError() - - def after_real_step(self, *args): - raise NotImplementedError() - - -class GeneratedInputV2(BaseGeneratedInputV2): - def __init__(self, size, embedding_name, embedding_size): - super(GeneratedInputV2, self).__init__() - self.size = size - self.embedding_name = embedding_name - self.embedding_size = embedding_size - - def after_real_step(self, input): - return max_id(input=input, name='__beam_search_predict__') - - def before_real_step(self): - predict_id = memory( - name='__beam_search_predict__', - size=self.size, - boot_with_const_id=self.bos_id) - - trg_emb = embedding( - input=predict_id, - size=self.embedding_size, - param_attr=attr.ParamAttr(name=self.embedding_name)) - return trg_emb - - -class RecurrentLayerGroupSetGeneratorV2(Layer): - def __init__(self, eos_name, max_length, beam_size, num_results_per_sample): - self.eos_name = eos_name - self.max_length = max_length - self.beam_size = beam_size - self.num_results_per_sample = num_results_per_sample - super(RecurrentLayerGroupSetGeneratorV2, self).__init__( - name=eos_name, parent_layers={}) - - def to_proto_impl(self, **kwargs): - RecurrentLayerGroupSetGenerator( - Generator( - eos_layer_name=self.eos_name, - max_num_frames=self.max_length, - beam_size=self.beam_size, - num_results_per_sample=self.num_results_per_sample)) - return self - - def context_name(self): - return self.eos_name + ".fake" - - def use_context_name(self): - return True - - -class MixedLayerV2(Layer): - """ - This class is use to support `with` grammar. If not, the following code - could convert mixed_layer simply. - - mixed = __convert_to_v2__( - 'mixed_layer', name_prefix='mixed', parent_names=['input']) - """ - - class AddToSealedMixedLayerExceptionV2(Exception): - pass - - def __init__(self, - size=0, - input=None, - name=None, - act=None, - bias_attr=None, - layer_attr=None): - self.__method_name__ = 'mixed_layer' - self.finalized = False - self.__inputs__ = [] - if input is not None: - self.__inputs__ = input - - other_kwargs = dict() - other_kwargs['name'] = name - other_kwargs['size'] = size - other_kwargs['act'] = act - other_kwargs['bias_attr'] = bias_attr - other_kwargs['layer_attr'] = layer_attr - parent_layers = {"input": self.__inputs__} - super(MixedLayerV2, self).__init__(name, parent_layers) - self.__other_kwargs__ = other_kwargs - - def __iadd__(self, other): - if not self.finalized: - self.__inputs__.append(other) - return self - else: - raise MixedLayerV2.AddToSealedMixedLayerExceptionV2() - - def __enter__(self): - assert len(self.__inputs__) == 0 - return self - - def __exit__(self, *args, **kwargs): - self.finalized = True - - def to_proto_impl(self, **kwargs): - args = dict() - for each in kwargs: - args[each] = kwargs[each] - for each in self.__other_kwargs__: - args[each] = self.__other_kwargs__[each] - size = args.get('size', None) - if size is not None: - if callable(size): - real_size = size() - else: - real_size = size - args['size'] = real_size - return getattr(conf_helps, self.__method_name__)(**args) - - -@wrap_name_default("mixed") -@wrap_act_default(act=activation.Linear()) -@wrap_bias_attr_default(has_bias=False) -@layer_support(conf_helps.layers.ERROR_CLIPPING, conf_helps.layers.DROPOUT) -def mixed(size=0, - name=None, - input=None, - act=None, - bias_attr=False, - layer_attr=None): - return MixedLayerV2(size, input, name, act, bias_attr, layer_attr) - - -mixed.__doc__ = conf_helps.mixed_layer.__doc__ - - -class RecurrentLayerInput(Layer): - def __init__(self, recurrent_name, index, parent_layers, reverse): - parents_len = len(parent_layers) - assert parents_len <= 1 - if parents_len == 0: - self.__parents__ = [] - else: - self.__parents__ = parent_layers.values()[0] - self.__recurrent_name__ = recurrent_name - self.__reverse__ = reverse - name = self.__parents__[ - index].name if index >= 0 else self.context_name() - super(RecurrentLayerInput, self).__init__( - name=name, parent_layers=parent_layers) - - def context_name(self): - return self.__recurrent_name__ + ".begin" - - def to_proto_impl(self, **kwargs): - model_type('recurrent_nn') - RecurrentLayerGroupWithoutOutLinksBegin( - name=self.__recurrent_name__, - in_links=map(lambda x: x.name, self.__parents__), - seq_reversed=self.__reverse__) - return self - - -class RecurrentLayerOutput(Layer): - def __init__(self, recurrent_name, index, parent_layers): - assert len(parent_layers) == 1 - self.__parents__ = parent_layers.values()[0] - super(RecurrentLayerOutput, self).__init__( - name=self.__parents__[index].name, parent_layers=parent_layers) - self.__recurrent_name__ = recurrent_name - - def context_name(self): - return self.__recurrent_name__ + ".end" - - def to_proto_impl(self, **kwargs): - for l in self.__parents__: - RecurrentLayerGroupSetOutLink(l.name) - RecurrentLayerGroupEnd(name=self.__recurrent_name__) - - -LayerV2 = Layer -data = DataLayerV2 -data.__name__ = 'data' -AggregateLevel = conf_helps.AggregateLevel -ExpandLevel = conf_helps.ExpandLevel -memory = MemoryV2 -memory.__name__ = 'memory' -memory.__doc__ = conf_helps.memory.__doc__ - - -def __layer_name_mapping__(inname): - if inname in ['data_layer', 'memory', 'mixed_layer', 'recurrent_group']: - # Do Not handle these layers - return - elif inname == 'maxid_layer': +def __convert_name__(inname): + if inname == 'maxid_layer': return 'max_id' elif inname.endswith('memory') or inname.endswith( '_seq') or inname.endswith('_sim') or inname == 'hsigmoid': @@ -431,187 +68,220 @@ def __layer_name_mapping__(inname): return inname elif inname.endswith("_layer"): return inname[:-len("_layer")] + else: + return inname -def __layer_name_mapping_parent_names__(inname): - all_args = getattr(conf_helps, inname).argspec.args - return filter( - lambda x: x in ['input1', 'input2', 'label', 'input', 'a', 'b', - 'expand_as', - 'weights', 'vectors', 'weight', 'score', 'left', - 'right', 'output_mem'], - all_args) - - -def __convert_layer__(_new_name_, _old_name_, _parent_names_): - global __all__ - __all__.append(_new_name_) - globals()[new_name] = __convert_to_v2__(_old_name_, _parent_names_) - globals()[new_name].__name__ = new_name - - -for each_layer_name in dir(conf_helps): - new_name = __layer_name_mapping__(each_layer_name) - if new_name is not None: - parent_names = __layer_name_mapping_parent_names__(each_layer_name) - assert len(parent_names) != 0, each_layer_name - __convert_layer__(new_name, each_layer_name, parent_names) - -del parent_names -del new_name -del each_layer_name - - -@wrap_name_default() -def recurrent_group(step, input, reverse=False, name=None): - if not isinstance(input, collections.Sequence): - input = [input] - - non_static_inputs = filter(lambda x: not isinstance(x, StaticInputV2), - input) - actual_input = [ - RecurrentLayerInput( - recurrent_name=name, - index=i, - parent_layers={'recurrent_inputs': non_static_inputs}, - reverse=reverse) for i in xrange(len(non_static_inputs)) - ] - - extra_input = None - if len(non_static_inputs) == 0: - extra_input = RecurrentLayerInput( - recurrent_name=name, index=-1, parent_layers={}, reverse=reverse) - - def __real_step__(*args): - rnn_input = list(args) - static_inputs = filter(lambda x: isinstance(x, StaticInputV2), input) - for static_input in static_inputs: - mem_name = "__%s_memory__" % static_input.input.name - mem = memory( - name=mem_name, - extra_input=extra_input, - is_seq=static_input.is_seq, - size=static_input.input.calculate_size, - boot_layer=static_input.input) - with mixed( - name=mem_name, - size=static_input.input.calculate_size, - act=activation.Identity()) as mix: - mix += identity_projection(input=mem) - rnn_input.insert(input.index(static_input), mix) - return step(*rnn_input) - - actual_output = __real_step__(*actual_input) - - if not isinstance(actual_output, collections.Sequence): - actual_output = [actual_output] - - retv = [ - RecurrentLayerOutput( - recurrent_name=name, - index=i, - parent_layers={'recurrent_outputs': actual_output}) - for i in xrange(len(actual_output)) - ] - if len(retv) == 1: - return retv[0] +for name in v1_layers.__all__: + obj = getattr(v1_layers, name) + if not __need_to_keep__(name): + continue + new_name = __convert_name__(name) + if callable(obj) and __need_to_wrap__(name): + globals()[new_name] = __convert_to_v2__(obj, new_name, __name__) else: - return retv - - -recurrent_group.__doc__ = conf_helps.recurrent_group.__doc__ - - -@wrap_name_default() -def beam_search(step, - input, - bos_id, - eos_id, - beam_size, - max_length=500, - name=None, - num_results_per_sample=None): - if num_results_per_sample is None: - num_results_per_sample = beam_size - assert num_results_per_sample <= beam_size - # logger.warning("num_results_per_sample should be less than beam_size") - - if isinstance(input, StaticInputV2) or isinstance(input, - BaseGeneratedInputV2): - input = [input] - - generated_input_index = -1 - - real_input = [] - for i, each_input in enumerate(input): - assert isinstance(each_input, StaticInputV2) or isinstance( - each_input, BaseGeneratedInputV2) - if isinstance(each_input, BaseGeneratedInputV2): - assert generated_input_index == -1 - generated_input_index = i - else: - real_input.append(each_input) + globals()[new_name] = obj + __all__.append(new_name) - assert generated_input_index != -1 - gipt = input[generated_input_index] - assert isinstance(gipt, BaseGeneratedInputV2) +def __data_layer__(name, type, **kwargs): + l = v1_layers.data_layer(name, type.dim, **kwargs) + l.data_type = type + return l - gipt.bos_id = bos_id - gipt.eos_id = eos_id - def __real_step__(*args): - eos_name = "__%s_eos_layer__" % name - generator = RecurrentLayerGroupSetGeneratorV2( - eos_name, max_length, beam_size, num_results_per_sample) +def __map_data_docstr__(doc): + doc = re.sub(r'(data = [^\)]+)\).*', + "data = paddle.layer.data(name=\"input\", " + "type=paddle.data_type.dense_vector(1000))", doc) - args = list(args) - before_step_layer = gipt.before_real_step() - before_step_layer.append_child( - layer=generator, parent_names=[before_step_layer.name]) - args.insert(generated_input_index, before_step_layer) + doc = re.sub(r':param size:.*', ':param type: Data type of this data layer', + doc) + doc = re.sub(r':type size:.*', ":type size: paddle.v2.data_type.InputType", + doc) + return doc + + +__data_layer__.__doc__ = __map_data_docstr__(v1_layers.data_layer.__doc__) + +data = __convert_to_v2__(__data_layer__, 'name', __name__) + + +def __get_used_layers__(output_layers, extra_layers=None): + layer_names = set() + parents = {} + + def add_parent(child, parent): + if child in parents: + parents[child].append(parent) + else: + parents[child] = [parent] + + def add_additional_parents(): + for sub_model in cp.g_config.model_config.sub_models: + if sub_model.name == 'root': + continue + for link in sub_model.in_links: + add_parent(link.link_name, link.layer_name) + add_parent(sub_model.name, link.layer_name) + for link in sub_model.out_links: + add_parent(link.link_name, link.layer_name) + add_parent(link.link_name, sub_model.name) + for mem in sub_model.memories: + if mem.boot_layer_name: + add_parent(mem.layer_name, mem.boot_layer_name) + add_parent(mem.link_name, mem.layer_name) + + def dfs_travel(layer_name): + if layer_name in layer_names: + return + layer_names.add(layer_name) + layer = cp.g_layer_map[layer_name] + + for inp in layer.inputs: + dfs_travel(inp.input_layer_name) + if layer.name in parents: + for p in parents[layer.name]: + dfs_travel(p) + + add_additional_parents() + + for layer in output_layers: + dfs_travel(layer.full_name) + + return layer_names + + +def __get_used_parameters__(layer_names, sub_models): + parameter_names = set() + for name in layer_names: + l = cp.g_layer_map[name] + for inp in l.inputs: + if inp.input_parameter_name: + parameter_names.add(inp.input_parameter_name) + if l.bias_parameter_name: + parameter_names.add(l.bias_parameter_name) + + for sub_model in sub_models: + for mem in sub_model.memories: + if mem.HasField("boot_bias_parameter_name"): + parameter_names.add(mem.boot_bias_parameter_name) + + return parameter_names + + +def __get_used_submodels__(layer_names): + submodel_names = set() + for submodel in cp.g_config.model_config.sub_models: + if submodel.name in layer_names: + submodel_names.add(submodel.name) + return submodel_names + + +def __get_used_evaluators__(layer_names): + evaluator_names = set() + for e in cp.g_config.model_config.evaluators: + used = True + for name in e.input_layers: + if name not in layer_names: + used = False + break + if used: + evaluator_names.add(e.name) + return evaluator_names + + +def __trim_submodel__(old_submodel, layer_names, input_layer_names, + output_layer_names, evaluator_names): + + submodel = SubModelConfig() + submodel.name = old_submodel.name + submodel.layer_names.extend( + filter(lambda x: x in layer_names, old_submodel.layer_names)) + submodel.input_layer_names.extend( + filter(lambda x: x in input_layer_names, submodel.layer_names)) + submodel.output_layer_names.extend( + filter(lambda x: x in output_layer_names, submodel.layer_names)) + submodel.evaluator_names.extend( + filter(lambda x: x in evaluator_names, old_submodel.evaluator_names)) + + submodel.is_recurrent_layer_group = old_submodel.is_recurrent_layer_group + submodel.reversed = old_submodel.reversed + + submodel.memories.extend( + filter(lambda x: x.link_name in layer_names, old_submodel.memories)) + target_inlinkid = (old_submodel.target_inlinkid + if old_submodel.HasField('target_inlinkid') else -1) + in_links = [] + for i, link in enumerate(old_submodel.in_links): + if link.link_name in layer_names or i == target_inlinkid: + in_links.append(link) + if i == target_inlinkid: + target_inlinkid = len(in_links) - 1 + submodel.in_links.extend(in_links) + + submodel.out_links.extend( + filter(lambda x: x.link_name in layer_names, old_submodel.out_links)) + if old_submodel.HasField('generator'): + submodel.generator.CopyFrom(old_submodel.generator) + + if old_submodel.HasField('target_inlinkid'): + submodel.target_inlinkid = target_inlinkid + return submodel + + +def parse_network(output_layers, extra_layers=None): + if not isinstance(output_layers, collections.Sequence): + output_layers = [output_layers] + if extra_layers is not None and not isinstance(extra_layers, + collections.Sequence): + extra_layers = [extra_layers] + else: + extra_layers = [] - predict = gipt.after_real_step(step(*args)) + layer_names = __get_used_layers__(output_layers + extra_layers) + submodel_names = __get_used_submodels__(layer_names) + submodel_names.add('root') + evaluator_names = __get_used_evaluators__(layer_names) + input_layer_names = set() + output_layer_names = set() - eos_layer = eos(input=predict, eos_id=eos_id, name=eos_name) - predict.append_child(layer=eos_layer, parent_names=[predict.name]) + model_config = ModelConfig() + model_config.type = cp.g_config.model_config.type + for l in cp.g_config.model_config.layers: + if l.name not in layer_names: + continue + model_config.layers.extend([l]) + if l.type == 'data': + model_config.input_layer_names.append(l.name) + input_layer_names.add(l.name) - return predict + for layer in output_layers: + model_config.output_layer_names.append(layer.full_name) + output_layer_names.add(layer.full_name) - # tmp = paddle.layer.recurrent_group( - # step=__real_step__, - # input=real_input, - # reverse=False, - # name=name, - # is_generating=True) - tmp = recurrent_group(step=__real_step__, input=real_input, name=name) + for e in cp.g_config.model_config.evaluators: + if e.name in evaluator_names: + model_config.evaluators.extend([e]) - return tmp + for s in cp.g_config.model_config.sub_models: + if s.name in submodel_names: + s = __trim_submodel__(s, layer_names, input_layer_names, + output_layer_names, evaluator_names) + model_config.sub_models.extend([s]) + parameter_names = __get_used_parameters__(layer_names, + model_config.sub_models) -beam_search.__doc__ = conf_helps.beam_search.__doc__ + for p in cp.g_config.model_config.parameters: + if p.name in parameter_names: + model_config.parameters.extend([p]) -__projection_names__ = filter(lambda x: x.endswith('_projection'), - dir(conf_helps)) + return model_config -__all__ += __projection_names__ -__operator_names__ = filter(lambda x: x.endswith('_operator'), dir(conf_helps)) -__all__ += __operator_names__ +def get_layer(name): + return config_base.__layer_map__.get(name) -# convert projection -for prj in __projection_names__: - globals()[prj] = __convert_to_v2__( - prj, parent_names=['input'], is_default_name=False) - globals()[prj].__name__ = prj -# convert operator -operator_list = [ - # [V1_method_name, parent_names], - ['dotmul_operator', ['a', 'b']], - ['conv_operator', ['img', 'filter']] -] -for op in operator_list: - globals()[op[0]] = __convert_to_v2__( - op[0], parent_names=op[1], is_default_name=False) - globals()[op[0]].__name__ = op[0] +cp.begin_parse() diff --git a/python/paddle/v2/networks.py b/python/paddle/v2/networks.py index 9e6644196c..8ae9f3b202 100644 --- a/python/paddle/v2/networks.py +++ b/python/paddle/v2/networks.py @@ -24,20 +24,7 @@ def __initialize__(): if each_subnetwork in ['inputs', 'outputs']: continue func = getattr(conf_nw, each_subnetwork) - if hasattr(func, 'argspec'): - argspec = func.argspec - else: - argspec = inspect.getargspec(func) - if each_subnetwork == 'simple_attention': - parents = ['encoded_sequence', 'encoded_proj', 'decoder_state'] - else: - parents = filter(lambda x: x.startswith('input'), argspec.args) - assert len(parents) != 0, each_subnetwork - v2_subnet = __convert_to_v2__( - each_subnetwork, - parent_names=parents, - is_default_name='name' in argspec.args) - globals()[each_subnetwork] = v2_subnet + globals()[each_subnetwork] = func globals()[each_subnetwork].__name__ = each_subnetwork global __all__ __all__.append(each_subnetwork) diff --git a/python/paddle/v2/op.py b/python/paddle/v2/op.py new file mode 100644 index 0000000000..03f3b9b9ef --- /dev/null +++ b/python/paddle/v2/op.py @@ -0,0 +1,120 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import layer +import activation as act +from config_base import Layer +from paddle.trainer_config_helpers.attrs import is_compatible_with +from paddle.trainer_config_helpers.default_decorators import wrap_name_default + +__all__ = [] + + +def __register_unary_math_op__(op_name, act): + def op(input, name=None): + return layer.mixed( + input=[layer.identity_projection(input=input)], name=name, act=act) + + op = wrap_name_default(op_name)(op) + op.__doc__ = type(act).__doc__ + globals()[op_name] = op + __all__.append(op_name) + + +__register_unary_math_op__('exp', act.Exp()) +__register_unary_math_op__('log', act.Log()) +__register_unary_math_op__('abs', act.Abs()) +__register_unary_math_op__('sigmoid', act.Sigmoid()) +__register_unary_math_op__('tanh', act.Tanh()) +__register_unary_math_op__('square', act.Square()) +__register_unary_math_op__('relu', act.Relu()) +__register_unary_math_op__('sqrt', act.Sqrt()) +__register_unary_math_op__('reciprocal', act.Reciprocal()) +__register_unary_math_op__('softmax', act.Softmax()) + + +def __add__(layeroutput, other): + if is_compatible_with(other, float): + return layer.slope_intercept(input=layeroutput, intercept=other) + if not isinstance(other, Layer): + raise TypeError("Layer can only be added with" + " another Layer or a number") + if layeroutput.size == other.size: + return layer.mixed(input=[ + layer.identity_projection(input=layeroutput), + layer.identity_projection(input=other) + ]) + if other.size != 1 and layeroutput.size != 1: + raise TypeError("Two Layer can be added only if they have equal size" + " or one of their sizes is 1. sizes are %s and %s" % + (layeroutput.size, other.size)) + elif layeroutput.size == 1: + tmp = layeroutput + layeroutput = other + other = tmp + other = layer.repeat(other, layeroutput.size) + return layer.mixed(input=[ + layer.identity_projection(input=layeroutput), + layer.identity_projection(input=other) + ]) + + +Layer.__radd__ = __add__ +Layer.__add__ = __add__ + + +def __neg__(layeroutput): + return layer.slope_intercept(input=layeroutput, slope=-1.0) + + +Layer.__neg__ = __neg__ + + +def __sub__(layeroutput, other): + if is_compatible_with(other, float): + return layer.slope_intercept(input=layeroutput, intercept=other) + if not isinstance(other, Layer): + raise TypeError("Layer can only be subtracted with" + " another Layeroutput or a number") + return __add__(layeroutput, -other) + + +Layer.__sub__ = __sub__ + + +def __rsub__(layeroutput, other): + neg = layer.slope_intercept(input=layeroutput, slope=-1.0) + return __add__(neg, other) + + +Layer.__rsub__ = __rsub__ + + +def __mul__(layeroutput, other): + if is_compatible_with(other, float): + return layer.slope_intercept(input=layeroutput, slope=other) + if not isinstance(other, Layer): + raise TypeError("Layer can only be multiplied with" + " another Layer or a number") + elif layeroutput.size == 1: + return layer.scaling(input=other, weight=layeroutput) + elif other.size == 1: + return layer.scaling(input=layeroutput, weight=other) + else: + raise TypeError("At least one of the operand of '*' must be a number" + " or a Layer with size=1") + + +Layer.__mul__ = __mul__ +Layer.__rmul__ = __mul__ diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt index eb02e53706..058f22befd 100644 --- a/python/paddle/v2/tests/CMakeLists.txt +++ b/python/paddle/v2/tests/CMakeLists.txt @@ -1,2 +1,2 @@ -add_python_test(test_v2_api test_data_feeder.py test_parameters.py +add_python_test(test_v2_api test_data_feeder.py test_op.py test_parameters.py test_layer.py test_rnn_layer.py test_topology.py test_image.py) diff --git a/python/paddle/v2/tests/test_layer.py b/python/paddle/v2/tests/test_layer.py index c67f3b84d9..2d25b1a9dc 100644 --- a/python/paddle/v2/tests/test_layer.py +++ b/python/paddle/v2/tests/test_layer.py @@ -73,7 +73,7 @@ class AggregateLayerTest(unittest.TestCase): pool = layer.pooling( input=pixel, pooling_type=pooling.Avg(), - agg_level=layer.AggregateLevel.EACH_SEQUENCE) + agg_level=layer.AggregateLevel.TO_SEQUENCE) last_seq = layer.last_seq(input=pixel) first_seq = layer.first_seq(input=pixel) concat = layer.concat(input=[last_seq, first_seq]) @@ -109,7 +109,7 @@ class ReshapeLayerTest(unittest.TestCase): expand = layer.expand( input=weight, expand_as=pixel, - expand_level=layer.ExpandLevel.FROM_TIMESTEP) + expand_level=layer.ExpandLevel.FROM_NO_SEQUENCE) repeat = layer.repeat(input=pixel, num_repeats=4) reshape = layer.seq_reshape(input=pixel, reshape_size=4) rotate = layer.rotate(input=pixel, height=16, width=49) @@ -173,9 +173,9 @@ class OtherLayerTest(unittest.TestCase): class ProjOpTest(unittest.TestCase): def test_projection(self): - input = layer.data(name='data', type=data_type.dense_vector(784)) + input = layer.data(name='data2', type=data_type.dense_vector(784)) word = layer.data( - name='word', type=data_type.integer_value_sequence(10000)) + name='word2', type=data_type.integer_value_sequence(10000)) fc0 = layer.fc(input=input, size=100, act=activation.Sigmoid()) fc1 = layer.fc(input=input, size=200, act=activation.Sigmoid()) mixed0 = layer.mixed( @@ -204,8 +204,8 @@ class ProjOpTest(unittest.TestCase): dotmul1 += dotmul context = layer.context_projection(input=fc0, context_len=5) - context0 = layer.mixed(size=100, input=context) - with layer.mixed(size=100) as context1: + context0 = layer.mixed(size=500, input=context) + with layer.mixed(size=500) as context1: context1 += context conv = layer.conv_projection( @@ -231,8 +231,8 @@ class ProjOpTest(unittest.TestCase): print layer.parse_network(conv1) def test_operator(self): - ipt0 = layer.data(name='data', type=data_type.dense_vector(784)) - ipt1 = layer.data(name='word', type=data_type.dense_vector(128)) + ipt0 = layer.data(name='data1', type=data_type.dense_vector(784)) + ipt1 = layer.data(name='word1', type=data_type.dense_vector(128)) fc0 = layer.fc(input=ipt0, size=100, act=activation.Sigmoid()) fc1 = layer.fc(input=ipt0, size=100, act=activation.Sigmoid()) @@ -261,7 +261,7 @@ class ProjOpTest(unittest.TestCase): class NetworkTests(unittest.TestCase): def test_vgg(self): - img = layer.data(name='pixel', type=data_type.dense_vector(784)) + img = layer.data(name='pixel1', type=data_type.dense_vector(784)) vgg_out = networks.small_vgg( input_image=img, num_channels=1, num_classes=2) print layer.parse_network(vgg_out) @@ -269,12 +269,12 @@ class NetworkTests(unittest.TestCase): class EvaluatorTest(unittest.TestCase): def test_evaluator(self): - img = layer.data(name='pixel', type=data_type.dense_vector(784)) + img = layer.data(name='pixel2', type=data_type.dense_vector(784)) output = layer.fc(input=img, size=10, act=activation.Softmax(), name='fc_here') - lbl = layer.data(name='label', type=data_type.integer_value(10)) + lbl = layer.data(name='label2', type=data_type.integer_value(10)) cost = layer.cross_entropy_cost(input=output, label=lbl) evaluator.classification_error(input=output, label=lbl) diff --git a/python/paddle/v2/tests/test_op.py b/python/paddle/v2/tests/test_op.py new file mode 100644 index 0000000000..69acccddf4 --- /dev/null +++ b/python/paddle/v2/tests/test_op.py @@ -0,0 +1,50 @@ +# Copyright PaddlePaddle contributors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import paddle.v2.data_type as data_type +import paddle.v2.layer as layer +import paddle.v2.op as op + + +class OpTest(unittest.TestCase): + def test_op(self): + x = layer.data(name='data', type=data_type.dense_vector(128)) + x = op.exp(x) + x = op.sqrt(x) + x = op.reciprocal(x) + x = op.log(x) + x = op.abs(x) + x = op.sigmoid(x) + x = op.tanh(x) + x = op.square(x) + x = op.relu(x) + y = 1 + x + y = y + 1 + y = x + y + y = y - x + y = y - 2 + y = 2 - y + y = 2 * y + y = y * 3 + z = layer.data(name='data_2', type=data_type.dense_vector(1)) + y = y * z + y = z * y + y = y + z + y = z + y + print layer.parse_network(y) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/tests/test_rnn_layer.py b/python/paddle/v2/tests/test_rnn_layer.py index 845277c012..192b0ee678 100644 --- a/python/paddle/v2/tests/test_rnn_layer.py +++ b/python/paddle/v2/tests/test_rnn_layer.py @@ -20,6 +20,8 @@ import paddle.v2.data_type as data_type import paddle.v2.layer as layer from paddle.trainer_config_helpers.config_parser_utils import \ parse_network_config as parse_network +from paddle.trainer_config_helpers.config_parser_utils import \ + reset_parser class RNNTest(unittest.TestCase): @@ -29,6 +31,8 @@ class RNNTest(unittest.TestCase): hidden_dim = 8 def parse_old_rnn(): + reset_parser() + def step(y): mem = conf_helps.memory(name="rnn_state", size=hidden_dim) out = conf_helps.fc_layer( @@ -48,6 +52,8 @@ class RNNTest(unittest.TestCase): return str(parse_network(test)) def parse_new_rnn(): + reset_parser() + def new_step(y): mem = layer.memory(name="rnn_state", size=hidden_dim) out = layer.fc(input=[y, mem], @@ -75,6 +81,8 @@ class RNNTest(unittest.TestCase): label_dim = 3 def parse_old_rnn(): + reset_parser() + def test(): data = conf_helps.data_layer(name="word", size=dict_dim) label = conf_helps.data_layer(name="label", size=label_dim) @@ -114,6 +122,7 @@ class RNNTest(unittest.TestCase): return str(parse_network(test)) def parse_new_rnn(): + reset_parser() data = layer.data( name="word", type=data_type.dense_vector(dict_dim)) label = layer.data( diff --git a/python/paddle/v2/tests/test_topology.py b/python/paddle/v2/tests/test_topology.py index 5c6dbcdb4f..7fd2ee82fd 100644 --- a/python/paddle/v2/tests/test_topology.py +++ b/python/paddle/v2/tests/test_topology.py @@ -46,8 +46,8 @@ class TestTopology(unittest.TestCase): self.assertEqual(label_data_type[1].dim, 10) def test_get_layer(self): - pixel = layer.data(name='pixel', type=data_type.dense_vector(784)) - label = layer.data(name='label', type=data_type.integer_value(10)) + pixel = layer.data(name='pixel2', type=data_type.dense_vector(784)) + label = layer.data(name='label2', type=data_type.integer_value(10)) hidden = layer.fc(input=pixel, size=100, act=conf_helps.SigmoidActivation()) @@ -56,14 +56,14 @@ class TestTopology(unittest.TestCase): act=conf_helps.SoftmaxActivation()) cost = layer.classification_cost(input=inference, label=label) topo = topology.Topology(cost) - pixel_layer = topo.get_layer("pixel") - label_layer = topo.get_layer("label") + pixel_layer = topo.get_layer("pixel2") + label_layer = topo.get_layer("label2") self.assertEqual(pixel_layer, pixel) self.assertEqual(label_layer, label) def test_parse(self): - pixel = layer.data(name='pixel', type=data_type.dense_vector(784)) - label = layer.data(name='label', type=data_type.integer_value(10)) + pixel = layer.data(name='pixel3', type=data_type.dense_vector(784)) + label = layer.data(name='label3', type=data_type.integer_value(10)) hidden = layer.fc(input=pixel, size=100, act=conf_helps.SigmoidActivation()) diff --git a/python/paddle/v2/topology.py b/python/paddle/v2/topology.py index 1e46e4973f..f3bb4d5f10 100644 --- a/python/paddle/v2/topology.py +++ b/python/paddle/v2/topology.py @@ -15,36 +15,13 @@ import collections from paddle.proto.ModelConfig_pb2 import ModelConfig - +import paddle.trainer_config_helpers as conf_helps import layer as v2_layer +import config_base __all__ = ['Topology'] -def __flatten__(lis): - """ - Given a list, possibly nested to any level, return it flattened. - """ - new_lis = [] - for item in lis: - if isinstance(item, collections.Sequence): - new_lis.extend(__flatten__(item)) - else: - new_lis.append(item) - return new_lis - - -def __bfs_travel__(callback, *layers): - layers = __flatten__(layers) - for each_layer in layers: - __break__ = callback(each_layer) - if __break__: - return - __layers__ = each_layer.__parent_layers__.values() + \ - each_layer.extra_parent() - __bfs_travel__(callback, *__layers__) - - class Topology(object): """ Topology is used to store the information about all layers @@ -94,31 +71,18 @@ class Topology(object): :param name: :return: """ - result_layer = [None] - - def __impl__(l): - if l.name == name: - result_layer[0] = l - return True # break - return False - - __bfs_travel__(__impl__, *self.layers) - if result_layer[0] is None: - raise ValueError("No such layer %s" % name) - return result_layer[0] + return v2_layer.get_layer(name) def data_layers(self): """ get all data layer :return: """ - data_layers = dict() - - def __impl__(l): - if isinstance(l, v2_layer.DataLayerV2): - data_layers[l.name] = l - - __bfs_travel__(__impl__, *self.layers) + data_layers = {} + for layer in self.proto().layers: + l = v2_layer.get_layer(layer.name) + if l and l.layer_type == conf_helps.LayerType.DATA: + data_layers[layer.name] = l return data_layers def data_type(self): @@ -127,7 +91,7 @@ class Topology(object): [('image', dense_vector(768)), ('label', integer_value(10))] """ data_layers = self.data_layers() - return [(nm, data_layers[nm].type) + return [(nm, data_layers[nm].data_type) for nm in self.proto().input_layer_names] def get_layer_proto(self, name): @@ -138,5 +102,5 @@ class Topology(object): def __check_layer_type__(layer): - if not isinstance(layer, v2_layer.LayerV2): - raise ValueError('layer should have type paddle.layer.Layer') + if not isinstance(layer, config_base.Layer): + raise ValueError('layer should have type paddle.v2.config_base.Layer')