Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into remove_evaluator

7 years ago · cf87f39b2b
parent f8029403a0 457d012a8e
commit cf87f39b2b
118 changed files with 2665 additions and 638 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -4,13 +4,17 @@ cache:
    - $HOME/.ccache
    - $HOME/.cache/pip
    - $TRAVIS_BUILD_DIR/build/third_party
+    - $TRAVIS_BUILD_DIR/build_android/third_party
 sudo: required
 dist: trusty
+services:
+  - docker
 os:
  - linux
 env:
  - JOB=build_doc
  - JOB=check_style
+  - JOB=build_android
 addons:
  apt:
    packages:
@ -41,8 +45,10 @@ before_install:
    function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }
 script:
  - |
-    timeout 2580 paddle/scripts/travis/${JOB}.sh # 43min timeout
-    RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true ;else exit 1; fi;
+    # 43min timeout
+    if [[ "$JOB" == "build_android" ]]; then timeout 2580 docker run -it --rm -v "$TRAVIS_BUILD_DIR:/paddle" paddlepaddle/paddle:latest-dev-android;
+    else timeout 2580 paddle/scripts/travis/${JOB}.sh; fi;
+    RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else exit 1; fi;
  - |
    if [[ "$JOB" != "build_doc" ]]; then exit 0; fi;
    if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi;
--- a/AUTHORS.md
+++ b/AUTHORS.md
@ -1,5 +1,6 @@
 | Github account | name |
 |---|---|
+| abhinavarora | Abhinav Arora |
 | backyes | Yan-Fei Wang |
 | beckett1124 | Bin Qi |
 | JiayiFeng | Jia-Yi Feng |
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -166,11 +166,11 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}/proto")
 include_directories("${CMAKE_CURRENT_BINARY_DIR}/go/pserver/client/c")

 set(EXTERNAL_LIBS
-    ${GFLAGS_LIBRARIES}
-    ${GLOG_LIBRARIES}
+    gflags
+    glog
    ${CBLAS_LIBRARIES}
-    ${PROTOBUF_LIBRARY}
-    ${ZLIB_LIBRARIES}
+    protobuf
+    zlib
    ${PYTHON_LIBRARIES}
 )

--- a/benchmark/cluster/vgg16/vgg16_fluid.py
+++ b/benchmark/cluster/vgg16/vgg16_fluid.py
@ -106,10 +106,10 @@ def vgg16_bn_drop(input):
    conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0])

    drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5)
-    fc1 = fluid.layers.fc(input=drop, size=512, act=None)
+    fc1 = fluid.layers.fc(input=drop, size=4096, act=None)
    bn = fluid.layers.batch_norm(input=fc1, act='relu')
    drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5)
-    fc2 = fluid.layers.fc(input=drop2, size=512, act=None)
+    fc2 = fluid.layers.fc(input=drop2, size=4096, act=None)
    return fc2


--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@ -59,7 +59,6 @@ endif(NOT WITH_GOLANG)

 if(NOT WITH_GPU)
    add_definitions(-DHPPL_STUB_FUNC)
-    add_definitions("-DCUPTI_LIB_PATH=\"\"")

    list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu)
 else()
@ -77,9 +76,7 @@ else()
    if(CUPTI_FOUND)
        include_directories(${CUPTI_INCLUDE_DIR})
        add_definitions(-DPADDLE_WITH_CUPTI)
-        add_definitions("-DCUPTI_LIB_PATH=\"${CUPTI_LIBRARY_PATH}\"")
    else()
-        add_definitions("-DCUPTI_LIB_PATH=\"\"")
        message(STATUS "Cannot find CUPTI, GPU Profiling is incorrect.")
    endif()
    set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SIMD_FLAG}")
--- a/cmake/external/zlib.cmake
+++ b/cmake/external/zlib.cmake
@ -28,7 +28,7 @@ ENDIF(WIN32)
 INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR})

 ExternalProject_Add(
-    zlib
+    extern_zlib
    ${EXTERNAL_PROJECT_LOG_ARGS}
    GIT_REPOSITORY  "https://github.com/madler/zlib.git"
    GIT_TAG         "v1.2.8"
@ -49,9 +49,11 @@ ExternalProject_Add(
                     -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
 )

+ADD_LIBRARY(zlib STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET zlib PROPERTY IMPORTED_LOCATION ${ZLIB_LIBRARIES})
+ADD_DEPENDENCIES(zlib extern_zlib)
+
 LIST(APPEND external_project_dependencies zlib)
-ADD_LIBRARY(zlib_target STATIC IMPORTED GLOBAL)
-SET_PROPERTY(TARGET zlib_target PROPERTY IMPORTED_LOCATION ${ZLIB_LIBRARIES})

 IF(WITH_C_API)
  INSTALL(DIRECTORY ${ZLIB_INCLUDE_DIR} DESTINATION third_party/zlib)
--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@ -104,7 +104,9 @@ function(merge_static_libs TARGET_NAME)
  foreach(lib ${libs})
    list(APPEND libs_deps ${${lib}_LIB_DEPENDS})
  endforeach()
-  list(REMOVE_DUPLICATES libs_deps)
+  if(libs_deps)
+    list(REMOVE_DUPLICATES libs_deps)
+  endif()

  # To produce a library we need at least one source file.
  # It is created by add_custom_command below and will helps
@ -191,10 +193,13 @@ function(cc_library TARGET_NAME)
        list(REMOVE_ITEM cc_library_DEPS warpctc)
        add_dependencies(${TARGET_NAME} warpctc)
      endif()
-      # Support linking flags: --whole-archive (Linux) / -force_load (MacOS)
-      target_circle_link_libraries(${TARGET_NAME} ${cc_library_DEPS})
      if("${cc_library_DEPS}" MATCHES "ARCHIVE_START")
+        # Support linking flags: --whole-archive (Linux) / -force_load (MacOS).
+        # WARNING: Please don't use ARCHIVE_START&ARCHIVE_END if TARGET_NAME will be linked by other libraries.
+        target_circle_link_libraries(${TARGET_NAME} ${cc_library_DEPS})
        list(REMOVE_ITEM cc_library_DEPS ARCHIVE_START ARCHIVE_END)
+      else()
+        target_link_libraries(${TARGET_NAME} ${cc_library_DEPS})
      endif()
      add_dependencies(${TARGET_NAME} ${cc_library_DEPS})
    endif()
--- a/cmake/inference_lib.cmake
+++ b/cmake/inference_lib.cmake
@ -72,7 +72,7 @@ copy(inference_lib DEPENDS paddle_fluid_shared
 )

 set(module "platform")
-copy(platform_lib
+copy(platform_lib DEPS profiler_py_proto
  SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/dynload/*.h ${src_dir}/${module}/details/*.h
  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/dynload ${dst_dir}/${module}/details
 )
--- a/doc/api/overview.rst
+++ b/doc/api/overview.rst
@ -7,10 +7,6 @@ it proposes some high-level concepts such as `Layers <http://www.paddlepaddle.or
 A model is composed of the computation described by a group of `Layers`, with `Evaluator` to define the error, `Optimizer` to update the parameters and `Data Reader` to feed in the data.

 We also provide the `interface for Training and Inference <http://www.paddlepaddle.org/docs/develop/api/en/v2/run_logic.html>`_ to help control the training and inference phrase,
-it has several easy to use methods
+it has several easy to use methods to better expose the internal running details, different `events <http://www.paddlepaddle.org/docs/develop/api/en/v2/run_logic.html#event>`_ are available to users by writing some callbacks.

- `paddle.train` 
- `paddle.test`
- `paddle.infer`
-
-to better expose the internal running details, different `events <http://www.paddlepaddle.org/docs/develop/api/en/v2/run_logic.html#event>`_ are available to users by writing some callbacks.
+All in all, the V2 API gives a higher abstraction and make PaddlePaddle programs require fiew lines of code.
--- a/doc/build_and_install/build_from_source_cn.rst
+++ b/doc/build_and_install/build_from_source_cn.rst
@ -9,7 +9,7 @@
 为了编译PaddlePaddle，我们需要

 1. 一台电脑，可以装的是 Linux, Windows 或者 MacOS 操作系统
-1. Docker
+2. Docker

 不需要依赖其他任何软件了。即便是 Python 和 GCC 都不需要，因为我们会把所有编译工具都安装进一个 Docker 镜像里。

--- a/doc/build_and_install/build_from_source_en.rst
+++ b/doc/build_and_install/build_from_source_en.rst
@ -9,7 +9,7 @@ Requirements
 To build PaddlePaddle, you need

 1. A computer -- Linux, Windows, MacOS.
-1. Docker.
+2. Docker.

 Nothing else.  Not even Python and GCC, because you can install all build tools into a Docker image. 
 We run all the tools by running this image.
--- a/doc/build_and_install/pip_install_en.rst
+++ b/doc/build_and_install/pip_install_en.rst
@ -1,4 +1,4 @@
-Install Using pip
+Install using pip
 ================================

 You can use current widely used Python package management
@ -8,7 +8,7 @@ most of current Linux systems or MacOS.

 .. _pip_install:

-Install Using pip
+Install using pip
 ------------------------------

 Run the following command to install PaddlePaddle on the current
--- a/doc/design/parallel_do.md
+++ b/doc/design/parallel_do.md
@ -39,15 +39,16 @@ In the backward pass
 This implementation allows to write mixed device program like this

 ```python
-# get embedding feature on CPU
-feature = some_cpu_only_op(data)
+W1 = fluid.tensor(size=[100,20], parameter=true)
+W2 = fluid.tensor(size=[20,15], parameter=true)

-gpu_places = get_place(use_gpu=True)
+data = layers.data()
+
+gpu_places = layers.get_place(use_gpu=True)
 # parallel processing on multiple GPUs
 pd = ParallelDo(gpu_places)
-with pd.do():
-    read_input(feature)
-    prediction = my_net(feature)
+with pd.do(input=data):
+    prediction = softmax(fc(fc(data, W1), W2))
    write_output(prediction)
 prediction = pd()
 loss = cross_entropy(prediction, label)
@ -66,20 +67,20 @@ start_program
 main_program
 {
 block0 {
-  vars: data, places, w1, w2
+  vars: data, places, w1, w2, w1_grad, w2_grad,
  ops: data, get_place, parallel_do(block1),
       parallel_do_grad(block2),
       sgd(w2, w2_grad),
       sgd(w1, w1_grad)
 }
-block1 {
+block1 { # the forward pass
  parent_block: 0
  vars: data, h1, h2, loss
  ops: fc, fc, softmax
 }
-block2 {
+block2 { # the backward pass
  parent_block: 1
-  vars: data_grad, h1_grad, h2_grad, loss_gard, w1_grad, w2_grad
+  vars: data_grad, h1_grad, h2_grad, loss_gard, local_w1_grad, local_w2_grad
  ops: softmax_grad,
       fc_grad
       fc_grad
--- a/doc/dev/contribute_to_paddle_cn.md
+++ b/doc/dev/contribute_to_paddle_cn.md
@ -222,6 +222,7 @@ upstream
 ## 提交代码的一些约定

 为了使评审人在评审代码时更好地专注于代码本身，请您每次提交代码时，遵守以下约定：
+
 1. 请保证Travis-CI 中单元测试能顺利通过。如果没过，说明提交的代码存在问题，评审人一般不做评审。
 2. 提交PUll Request前：
   - 请注意commit的数量：
@ -231,6 +232,7 @@ upstream
 3. 如果解决了某个Issue的问题，请在该PUll Request的**第一个**评论框中加上：`fix #issue_number`，这样当该PUll Request被合并后，会自动关闭对应的Issue。关键词包括：close, closes, closed, fix, fixes, fixed, resolve, resolves, resolved，请选择合适的词汇。详细可参考[Closing issues via commit messages](https://help.github.com/articles/closing-issues-via-commit-messages)。

 此外，在回复评审人意见时，请您遵守以下约定：
+
 1. 评审人的每个意见都必须回复（这是开源社区的基本礼貌，别人帮了忙，应该说谢谢）：
   - 对评审意见同意且按其修改完的，给个简单的`Done`即可；
   - 对评审意见不同意的，请给出您自己的反驳理由。
--- a/doc/dev/index_en.rst
+++ b/doc/dev/index_en.rst
@ -4,6 +4,5 @@ Development
 ..  toctree::
  :maxdepth: 1

-  new_layer_en.rst
  contribute_to_paddle_en.md
  write_docs_en.rst
--- a/doc/faq/build_and_install/index_en.rst
+++ b/doc/faq/build_and_install/index_en.rst
@ -0,0 +1,5 @@
+############################
+Install, Build and Unit test
+############################
+
+TBD
--- a/doc/faq/cluster/index_en.rst
+++ b/doc/faq/cluster/index_en.rst
@ -0,0 +1,5 @@
+###############################
+Cluster Training and Prediction
+###############################
+
+TBD
--- a/doc/faq/index_en.rst
+++ b/doc/faq/index_en.rst
@ -0,0 +1,12 @@
+FAQ
+====
+
+ 
+..  toctree::
+  :maxdepth: 1
+
+  build_and_install/index_en.rst
+  model/index_en.rst
+  parameter/index_en.rst
+  local/index_en.rst
+  cluster/index_en.rst
--- a/doc/faq/local/index_en.rst
+++ b/doc/faq/local/index_en.rst
@ -0,0 +1,5 @@
+#############################
+Local Training and Prediction
+#############################
+
+TBD
--- a/doc/faq/model/index_en.rst
+++ b/doc/faq/model/index_en.rst
@ -0,0 +1,5 @@
+###################
+Model Configuration
+###################
+
+TBD
--- a/doc/faq/parameter/index_en.rst
+++ b/doc/faq/parameter/index_en.rst
@ -0,0 +1,5 @@
+#################
+Parameter Setting
+#################
+
+TBD
--- a/doc/getstarted/concepts/use_concepts_en.rst
+++ b/doc/getstarted/concepts/use_concepts_en.rst
@ -0,0 +1,3 @@
+Basic Concept
+=============
+TBD
--- a/doc/getstarted/index_en.rst
+++ b/doc/getstarted/index_en.rst
@ -5,3 +5,4 @@ GET STARTED
  :maxdepth: 1

  quickstart_en.rst
+  concepts/use_concepts_en.rst
--- a/doc/howto/capi/compile_paddle_lib_en.md
+++ b/doc/howto/capi/compile_paddle_lib_en.md
@ -0,0 +1,3 @@
+## Install and Build
+
+TBD
--- a/doc/howto/capi/index_en.rst
+++ b/doc/howto/capi/index_en.rst
@ -0,0 +1,9 @@
+C-API Prediction Library
+========================
+
+..  toctree::
+  :maxdepth: 1
+
+  compile_paddle_lib_en.md
+  organization_of_the_inputs_en.md
+  workflow_of_capi_en.md
--- a/Show More
+++ b/Show More