Merge branch 'develop' of github.com:baidu/Paddle into feature/travis_pre_commit_checks

.gitignore

@@ -9,3 +9,6 @@ build/
 .pydevproject
 Makefile
 .test_env/
+
+*~
+bazel-*

.pre-commit-config.yaml

@@ -6,7 +6,8 @@
 -   repo: https://github.com/reyoung/mirrors-yapf.git
     sha: v0.13.2
     hooks:
-    -   id: yapf
+    - id: yapf
+      files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$  # Bazel BUILD files follow Python syntax.
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     sha: 7539d8bd1a00a3c1bfd34cdb606d3a6372e83469
     hooks:

.travis.yml

@@ -46,16 +46,19 @@ addons:
 before_install:
   - |
     if [ ${JOB} == "BUILD_AND_TEST" ]; then
-      if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)|(\.rst$)|(\.jpg$)|(\.png$)'
-      then
-        echo "Only markdown docs were updated, stopping build process."
-        exit
+      local change_list=`git diff --name-only $TRAVIS_COMMIT_RANGE`
+      if [ $? -eq 0 ]; then  # if git diff return no zero, then rerun unit test.
+        if ! echo ${change_list} | grep -qvE '(\.md$)|(\.rst$)|(\.jpg$)|(\.png$)'
+        then
+          echo "Only markdown docs were updated, stopping build process."
+          exit
+        fi
       fi
     fi
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi
   - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
-  - pip install wheel protobuf sphinx breathe recommonmark virtualenv numpy sphinx_rtd_theme pre-commit
+  - pip install wheel protobuf sphinx recommonmark virtualenv numpy sphinx_rtd_theme pre-commit
 script:
   - paddle/scripts/travis/main.sh
 notifications:

WORKSPACE

@@ -0,0 +1,17 @@
+# External dependency to Google protobuf.
+http_archive(
+    name = "protobuf",
+    url = "http://github.com/google/protobuf/archive/v3.1.0.tar.gz",
+    sha256 = "0a0ae63cbffc274efb573bdde9a253e3f32e458c41261df51c5dbc5ad541e8f7",
+    strip_prefix = "protobuf-3.1.0",
+)
+
+# External dependency to gtest 1.7.0.  This method comes from
+# https://www.bazel.io/versions/master/docs/tutorial/cpp.html.
+new_http_archive(
+    name = "gtest",
+    url = "https://github.com/google/googletest/archive/release-1.7.0.zip",
+    sha256 = "b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0",
+    build_file = "third_party/gtest.BUILD",
+    strip_prefix = "googletest-release-1.7.0",
+)

cmake/check_packages.cmake

@@ -30,7 +30,6 @@ if(WITH_DOC)
   find_package(Sphinx REQUIRED)
   find_package(Doxygen REQUIRED)
   find_python_module(recommonmark REQUIRED)
-  find_python_module(breathe REQUIRED)
 endif()
 
 if(WITH_SWIG_PY)

demo/sentiment/predict.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+import os, sys
 import numpy as np
 from optparse import OptionParser
 from py_paddle import swig_paddle, DataProviderConverter
@@ -66,35 +66,27 @@ class SentimentPrediction():
         for v in open(label_file, 'r'):
             self.label[int(v.split('\t')[1])] = v.split('\t')[0]
 
-    def get_data(self, data_file):
+    def get_index(self, data):
         """
-        Get input data of paddle format.
+        transform word into integer index according to the dictionary.
         """
-        with open(data_file, 'r') as fdata:
-            for line in fdata:
-                words = line.strip().split()
-                word_slot = [
-                    self.word_dict[w] for w in words if w in self.word_dict
-                ]
-                if not word_slot:
-                    print "all words are not in dictionary: %s", line
-                    continue
-                yield [word_slot]
+        words = data.strip().split()
+        word_slot = [
+            self.word_dict[w] for w in words if w in self.word_dict
+        ]
+        return word_slot
 
-    def predict(self, data_file):
-        """
-        data_file: file name of input data.
-        """
-        input = self.converter(self.get_data(data_file))
+    def batch_predict(self, data_batch):
+        input = self.converter(data_batch)
         output = self.network.forwardTest(input)
         prob = output[0]["value"]
-        lab = np.argsort(-prob)
-        if self.label is None:
-            print("%s: predicting label is %d" % (data_file, lab[0][0]))
-        else:
-            print("%s: predicting label is %s" %
-                  (data_file, self.label[lab[0][0]]))
-
+        labs = np.argsort(-prob)
+        for idx, lab in enumerate(labs):
+            if self.label is None:
+                print("predicting label is %d" % (lab[0]))
+            else:
+                print("predicting label is %s" %
+                      (self.label[lab[0]]))
 
 def option_parser():
     usage = "python predict.py -n config -w model_dir -d dictionary -i input_file "
@@ -119,11 +111,13 @@ def option_parser():
         default=None,
         help="dictionary file")
     parser.add_option(
-        "-i",
-        "--data",
+        "-c",
+        "--batch_size",
+        type="int",
         action="store",
-        dest="data",
-        help="data file to predict")
+        dest="batch_size",
+        default=1,
+        help="the batch size for prediction")
     parser.add_option(
         "-w",
         "--model",
@@ -137,14 +131,21 @@ def option_parser():
 def main():
     options, args = option_parser()
     train_conf = options.train_conf
-    data = options.data
+    batch_size = options.batch_size
     dict_file = options.dict_file
     model_path = options.model_path
     label = options.label
     swig_paddle.initPaddle("--use_gpu=0")
     predict = SentimentPrediction(train_conf, dict_file, model_path, label)
-    predict.predict(data)
 
+    batch = []
+    for line in sys.stdin:
+        batch.append([predict.get_index(line)])
+        if len(batch) == batch_size:
+            predict.batch_predict(batch)
+            batch=[]
+    if len(batch) > 0:
+        predict.batch_predict(batch)
 
 if __name__ == '__main__':
     main()

demo/sentiment/predict.sh

@@ -19,9 +19,9 @@ set -e
 model=model_output/pass-00002/
 config=trainer_config.py
 label=data/pre-imdb/labels.list
-python predict.py \
-     -n $config\
-     -w $model \
-     -b $label \
-     -d ./data/pre-imdb/dict.txt \
-     -i ./data/aclImdb/test/pos/10007_10.txt 
+cat ./data/aclImdb/test/pos/10007_10.txt | python predict.py \
+     --tconf=$config\
+     --model=$model \
+     --label=$label \
+     --dict=./data/pre-imdb/dict.txt \
+     --batch_size=1

doc/api/data_provider/pydataprovider2_en.rst

@@ -1,5 +1,7 @@
+..  _api_pydataprovider:
+
 PyDataProvider2
-=================
+===============
 
 We highly recommand users to use PyDataProvider2 to provide training or testing
 data to PaddlePaddle. The user only needs to focus on how to read a single

doc/api/index_en.rst

@@ -1,35 +1,37 @@
 API
-====
+===
 
 DataProvider API
 ----------------
 
 ..  toctree::
-  :maxdepth: 1
+    :maxdepth: 1
 
-  data_provider/index_en.rst
-  data_provider/pydataprovider2_en.rst
+    data_provider/index_en.rst
+    data_provider/pydataprovider2_en.rst
+
+..  _api_trainer_config:
 
 Model Config API
 ----------------
 
 ..  toctree::
-  :maxdepth: 1
+    :maxdepth: 1
 
-  trainer_config_helpers/optimizers.rst
-  trainer_config_helpers/data_sources.rst
-  trainer_config_helpers/layers.rst
-  trainer_config_helpers/activations.rst 
-  trainer_config_helpers/poolings.rst
-  trainer_config_helpers/networks.rst
-  trainer_config_helpers/evaluators.rst
-  trainer_config_helpers/attrs.rst
+    trainer_config_helpers/optimizers.rst
+    trainer_config_helpers/data_sources.rst
+    trainer_config_helpers/layers.rst
+    trainer_config_helpers/activations.rst 
+    trainer_config_helpers/poolings.rst
+    trainer_config_helpers/networks.rst
+    trainer_config_helpers/evaluators.rst
+    trainer_config_helpers/attrs.rst
 
 
 Applications API
 ----------------
 
 ..  toctree::
-  :maxdepth: 1
+    :maxdepth: 1
 
-  predict/swig_py_paddle_en.rst
+    predict/swig_py_paddle_en.rst

doc/api/trainer_config_helpers/data_sources.rst

@@ -1,3 +1,5 @@
+..  _api_trainer_config_helpers_data_sources:
+
 DataSources
 ===========
 

doc/api/trainer_config_helpers/layers.rst

@@ -20,6 +20,8 @@ LayerOutput
 Data layer
 ===========
 
+..  _api_trainer_config_helpers_layers_data_layer:
+
 data_layer
 ----------
 ..  automodule:: paddle.trainer_config_helpers.layers
@@ -29,6 +31,8 @@ data_layer
 Fully Connected Layers
 ======================
 
+..  _api_trainer_config_helpers_layers_fc_layer:
+
 fc_layer
 --------
 ..  automodule:: paddle.trainer_config_helpers.layers
@@ -68,6 +72,8 @@ img_conv_layer
     :members: img_conv_layer
     :noindex:
 
+..  _api_trainer_config_helpers_layers_context_projection:
+
 context_projection 
 ------------------
 ..  automodule:: paddle.trainer_config_helpers.layers
@@ -185,6 +191,8 @@ mixed_layer
     :members: mixed_layer
     :noindex:
 
+..  _api_trainer_config_helpers_layers_embedding_layer:
+
 embedding_layer
 ---------------
 ..  automodule:: paddle.trainer_config_helpers.layers
@@ -237,6 +245,8 @@ trans_full_matrix_projection
 Aggregate Layers
 ================
 
+..  _api_trainer_config_helpers_layers_pooling_layer:
+
 pooling_layer
 -------------
 ..  automodule:: paddle.trainer_config_helpers.layers
@@ -333,6 +343,8 @@ tensor_layer
     :members: tensor_layer
     :noindex:
 
+..  _api_trainer_config_helpers_layers_cos_sim:
+
 cos_sim
 -------
 ..  automodule:: paddle.trainer_config_helpers.layers

doc/api/trainer_config_helpers/networks.rst

@@ -13,6 +13,8 @@ sequence_conv_pool
     :members: sequence_conv_pool
     :noindex:
 
+..  _api_trainer_config_helpers_network_text_conv_pool:
+
 text_conv_pool
 --------------
 ..  automodule:: paddle.trainer_config_helpers.networks

doc/conf.py.in

@@ -144,5 +144,6 @@ def setup(app):
     # no c++ API for now
     app.add_config_value('recommonmark_config', {
             'url_resolver': lambda url: github_doc_root + url,
+	    'enable_eval_rst': True,
             }, True)
     app.add_transform(AutoStructify)

doc/getstarted/build_and_install/build_from_source_en.md

@@ -79,7 +79,7 @@ As a simple example, consider the following:
 
     ```bash
     pip install 'sphinx>=1.4.0'
-    pip install sphinx_rtd_theme breathe recommonmark
+    pip install sphinx_rtd_theme recommonmark
 
     # install doxygen on Ubuntu
     sudo apt-get install doxygen 

doc/getstarted/build_and_install/docker_install_en.rst

@@ -104,3 +104,70 @@ container:
 
 Then we can direct our Web browser to the HTML version of source code
 at http://localhost:8088/paddle/
+
+
+Development Using Docker
+------------------------
+
+Develpers can work on PaddlePaddle using Docker.  This allows
+developers to work on different platforms -- Linux, Mac OS X, and
+Windows -- in a consistent way.
+
+The general development workflow with Docker and Bazel is as follows:
+
+1. Get the source code of Paddle:
+
+   .. code-block:: bash
+
+      git clone --recursive https://github.com/paddlepaddle/paddle
+
+
+2. Build a development Docker image `paddle:dev` from the source code.
+   This image contains all the development tools and dependencies of
+   PaddlePaddle.
+
+
+   .. code-block:: bash
+
+      cd paddle
+      docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile .
+
+
+3. Run the image as a container and mounting local source code
+   directory into the container.  This allows us to change the code on
+   the host and build it within the container.
+
+   .. code-block:: bash
+
+      docker run \
+       -d # run the container in background mode \
+       --name paddle # we can run a nginx container to serve documents \
+       -p 2022:22    # so we can SSH into this container \
+       -v $PWD:/paddle # mount the source code \
+       -v $HOME/.cache/bazel:/root/.cache/bazel # mount Bazel cache \
+       paddle:dev
+
+4. SSH into the container:
+
+   .. code-block:: bash
+
+      ssh root@localhost -p 2022
+
+5. We can edit the source code in the container or on this host.  Then
+   we can build using cmake
+
+   .. code-block:: bash
+
+      cd /paddle # where paddle source code has been mounted into the container
+      mkdir -p build
+      cd build
+      cmake -DWITH_TESTING=ON ..
+      make -j `nproc`
+      CTEST_OUTPUT_ON_FAILURE=1 ctest
+
+   or Bazel in the container:
+
+   .. code-block:: bash
+
+      cd /paddle
+      bazel test ...

doc/howto/cmd_parameter/index_en.md

@@ -1,3 +1,6 @@
+```eval_rst
+..  _cmd_line_index_en:
+```
 # How to Set Command-line Parameters
 
 * [Use Case](use_case_en.md)

doc/howto/source/api.rst

@@ -1,5 +0,0 @@
-API
-===
-
-.. doxygenfile:: paddle/api/PaddleAPI.h
-.. doxygenfile:: paddle/api/Internal.h

doc/howto/source/cuda/index.rst

@@ -1,9 +0,0 @@
-CUDA
-====
-
-.. toctree::
-  :maxdepth: 2
-
-  matrix.rst
-  nn.rst
-  utils.rst

doc/howto/source/cuda/matrix.rst

@@ -1,59 +0,0 @@
-Matrix
-======
-
-Base
-----
-
-hl_matrix.h
-```````````
-.. doxygenfile:: paddle/cuda/include/hl_matrix.h
-
-hl_matrix_base.h
-````````````````
-.. doxygenfile:: paddle/cuda/include/hl_matrix_base.cuh
-
-hl_matrix_apply.cuh
-```````````````````
-.. doxygenfile:: paddle/cuda/include/hl_matrix_apply.cuh
-
-hl_matrix_ops.cuh
-`````````````````
-.. doxygenfile:: paddle/cuda/include/hl_matrix_ops.cuh
-
-hl_matrix_type.cuh
-``````````````````
-.. doxygenfile:: paddle/cuda/include/hl_matrix_type.cuh
-
-hl_sse_matrix_kernel.cuh
-````````````````````````
-.. doxygenfile:: paddle/cuda/include/hl_sse_matrix_kernel.cuh
-
-Matrix Function 
----------------
-
-hl_batch_transpose.h
-````````````````````
-.. doxygenfile:: paddle/cuda/include/hl_batch_transpose.h
-
-hl_aggregate.h
-``````````````
-.. doxygenfile:: paddle/cuda/include/hl_aggregate.h
-
-hl_top_k.h
-``````````
-.. doxygenfile:: paddle/cuda/include/hl_top_k.h
-
-hl_table_apply.h
-````````````````
-.. doxygenfile:: paddle/cuda/include/hl_table_apply.h
-
-Sparse Matrix
--------------
-
-hl_sparse.h
-```````````
-.. doxygenfile:: paddle/cuda/include/hl_sparse.h
-
-hl_sparse.ph
-````````````
-.. doxygenfile:: paddle/cuda/include/hl_sparse.ph

doc/howto/source/cuda/nn.rst

@@ -1,39 +0,0 @@
-Neural Network
-==============
-
-Base
-----
-
-.. doxygenfile:: paddle/cuda/include/hl_gpu.h
-.. doxygenfile:: paddle/cuda/include/hl_functions.h
-.. doxygenfile:: paddle/cuda/include/hl_avx_functions.h
-.. doxygenfile:: paddle/cuda/include/hl_gpu_functions.cuh
-.. doxygenfile:: paddle/cuda/include/hl_activation_functions.h
-
-
-CNN Related APIs
-----------------
-.. doxygenfile:: paddle/cuda/include/hl_cnn.h
-.. doxygenfile:: paddle/cuda/include/hl_cuda_cudnn.h
-.. doxygenfile:: paddle/cuda/include/hl_cuda_cudnn.ph
-
-RNN Related APIs
-----------------
-
-.. doxygenfile:: paddle/cuda/include/hl_recurrent_apply.cuh
-.. doxygenfile:: paddle/cuda/include/hl_sequence.h
-
-LSTM Model
-``````````
-
-.. doxygenfile:: paddle/cuda/include/hl_lstm.h
-.. dpxygenfile:: paddle/cuda/include/hl_cpu_lstm.cuh
-.. doxygenfile:: paddle/cuda/include/hl_gpu_lstm.cuh
-.. doxygenfile:: paddle/cuda/include/hl_lstm_ops.cuh
-
-GRU Model
-`````````
-
-.. doxygenfile:: paddle/cuda/include/hl_gru_ops.cuh
-.. doxygenfile:: paddle/cuda/include/hl_cpu_gru.cuh
-.. doxygenfile:: paddle/cuda/include/hl_gpu_gru.cuh

doc/howto/source/cuda/utils.rst

@@ -1,37 +0,0 @@
-Utils
-=====
-
-Dynamic Link Libs
------------------
-.. doxygenfile:: paddle/cuda/include/hl_dso_loader.h
-
-GPU Resources
--------------
-
-hl_cuda.ph
-``````````
-.. doxygenfile:: paddle/cuda/include/hl_cuda.ph
-
-hl_cuda.h
-`````````
-.. doxygenfile:: paddle/cuda/include/hl_cuda.h
-
-HPPL Base
----------
-.. doxygenfile:: paddle/cuda/include/hl_base.h
-
-CUBLAS Wrapper
---------------
-.. doxygenfile:: paddle/cuda/include/hl_cuda_cublas.h
-
-Timer
------
-.. doxygenfile:: paddle/cuda/include/hl_time.h
-
-Thread Resource
----------------
-.. doxygenfile:: paddle/cuda/include/hl_thread.ph
-
-Device Function
----------------
-.. doxygenfile:: paddle/cuda/include/hl_device_functions.cuh

doc/howto/source/gserver/activations.rst

@@ -1,5 +0,0 @@
-Activations
-===========
-
-..  doxygenclass:: paddle::ActivationFunction
-    :members:

doc/howto/source/gserver/dataproviders.rst

@@ -1,87 +0,0 @@
-==============
-Data Providers
-==============
-
-DataProviders
-=============
-
-Base
-----
-..  doxygenclass:: paddle::DataProvider
-    :members:
-
-DataProviderGroup
------------------
-..  doxygenclass:: paddle::DataProviderGroup
-    :members:
-
-MultiDataProvider
------------------
-..  doxygenclass:: paddle::MultiDataProvider
-    :members:
-
-PyDataProvider
-==============
-
-IFieldScanner
--------------
-..  doxygenclass:: paddle::IFieldScanner
-    :members:
-
-DenseScanner
--------------
-..  doxygenclass:: paddle::DenseScanner
-    :members:
-
-IndexScanner
--------------
-..  doxygenclass:: paddle::IndexScanner
-    :members:
-
-SparseNonValueScanner
----------------------
-..  doxygenclass:: paddle::SparseNonValueScanner
-    :members:
-
-SparseValueScanner
-------------------
-..  doxygenclass:: paddle::SparseValueScanner
-    :members:
-
-SequenceScanner
----------------
-..  doxygenclass:: paddle::SparseValueScanner
-    :members:
-
-IPyDataProviderCache
---------------------
-..  doxygenclass:: paddle::IPyDataProviderCache
-    :members:
-
-NoCacheStrategy
----------------
-..  doxygenclass:: paddle::NoCacheStrategy
-    :members:
-
-CacheOnePassInMemory
---------------------
-..  doxygenclass:: paddle::CacheOnePassInMemory
-    :members:
-
-IPyDataProvider
----------------
-..  doxygenclass:: paddle::PyDataProvider2
-    :members:
-
-ProtoDataProvider
-=================
-
-ProtoDataProvider
-----------------
-..  doxygenclass:: paddle::ProtoDataProvider
-    :members:
-
-ProtoSequenceDataProvider
--------------------------
-..  doxygenclass:: paddle::ProtoSequenceDataProvider
-    :members:

doc/howto/source/gserver/evaluators.rst

@@ -1,103 +0,0 @@
-==========
-Evaluators
-==========
-
-Base
-====
-
-..  doxygenclass:: paddle::Evaluator
-    :members:
-
-Sum
-===
-
-SumEvaluator
-------------
-..  doxygenclass:: paddle::SumEvaluator
-    :members:
-
-ColumnSumEvaluator
-------------------
-..  doxygenclass:: paddle::ColumnSumEvaluator
-    :members:
-
-Classification
-==============
-
-ClassificationErrorEvaluator
----------------------------
-..  doxygenclass:: paddle::ClassificationErrorEvaluator
-    :members:
-
-SequenceClassificationErrorEvaluator
-------------------------------------
-..  doxygenclass:: paddle::SequenceClassificationErrorEvaluator
-    :members:
-
-AucEvaluator
--------------
-..  doxygenclass:: paddle::AucEvaluator
-    :members:
-
-PrecisionRecallEvaluator
-------------------------
-..  doxygenclass:: paddle::PrecisionRecallEvaluator
-    :members:
-
-ChunkEvaluator
---------------
-..  doxygenclass:: paddle::ChunkEvaluator
-    :members:
-
-CTCEvaluator
-------------
-..  doxygenclass:: paddle::CTCErrorEvaluator
-    :members:
-
-
-Rank
-====
-
-PnpairEvaluator
--------------
-..  doxygenclass:: paddle::PnpairEvaluator
-    :members:
-
-AucEvaluator
--------------
-..  doxygenclass:: paddle::RankAucEvaluator
-    :members:
-
-
-Printer
-=======
-
-ValuePrinter
--------------
-..  doxygenclass:: paddle::ValuePrinter
-    :members:
-
-GradientPrinter
----------------
-..  doxygenclass:: paddle::GradientPrinter
-    :members:
-
-MaxIdPrinter
-------------
-..  doxygenclass:: paddle::MaxIdPrinter
-    :members:
-
-MaxFramePrinter
----------------
-..  doxygenclass:: paddle::MaxFramePrinter
-    :members:
-
-SequenceTextPrinter
-------------------
-..  doxygenclass:: paddle::SequenceTextPrinter
-    :members:
-
-ClassificationErrorPrinter
---------------------------
-..  doxygenclass:: paddle::ClassificationErrorPrinter
-    :members:

doc/howto/source/gserver/gradientmachines.rst

@@ -1,27 +0,0 @@
-Gradient Machines
-=================
-
-GradientMachine
----------------
-..  doxygenclass:: paddle::GradientMachine
-    :members:
-
-GradientMachineMode
--------------------
-..  doxygenclass:: paddle::IGradientMachineMode
-    :members:
-
-MultiGradientMachine
---------------------
-..  doxygenclass:: paddle::MultiGradientMachine
-    :members:
-
-TrainerThread
-`````````````
-..  doxygenclass:: paddle::TrainerThread
-    :members:
-
-RecurrentGradientMachine
-------------------------
-..  doxygenclass:: paddle::RecurrentGradientMachine
-    :members: