From 994b52fc2b2a061436729b94cae0e3a35ac17366 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 2 Mar 2018 17:53:35 +0800 Subject: [PATCH 1/9] Add layers for save/load op --- python/paddle/fluid/layers/nn.py | 70 +++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index a10463b52c..1acc74f757 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3206,7 +3206,7 @@ def one_hot(input, depth): operator. Args: - input(Tensor/LodTensor): A Tensor/LodTensor of indices, last dimension must be 1. + input(variable): A Tensor/LodTensor of indices, last dimension must be 1. depth(scalar): an interger defining the depth of the one hot dimension. Returns: @@ -3265,3 +3265,71 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): counter.stop_gradient = True return counter + + +def save(x, file_path, overwrite=True): + """ + Saves a variable as a file. + + Args: + x(variable): The Tensor/LoDTensor to be saved. + file_path(str): The file path where the variable will be saved. + overwrite(bool): Whether or not cover the given file when it has already + existed. If it's set 'False' and the file is existed, a runtime + error will be thrown. + """ + helper = LayerHelper("save", **locals()) + helper.append_op( + type="save", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, + "overwrite": overwrite}) + + +def save_combine(x, file_path, overwrite=True): + """ + Saves a variable as a file. + + Args: + x(list): A list of Tensor/LoDTensor to be saved together in a single file. + file_path(str): The file path where variables will be saved. + overwrite(bool): Whether or not cover the given file when it has already + existed. If it's set 'False' and the file is existed, a runtime + error will be thrown. + """ + helper = LayerHelper("save_combine", **locals()) + helper.append_op( + type="save_combine", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, + "overwrite": overwrite}) + + +def load(out, file_path): + """ + Args: + out(variable): The variable to be read from the disk file. + file_path(str): The path of the disk file. + """ + helper = LayerHelper("load", **locals()) + helper.append_op( + type="load", + inputs={}, + output={"Out": out}, + args={"file_path": file_path}) + + +def load_combine(out, file_path): + """ + Args: + out(list): The list of variables to be read from the disk file. + file_path(str): The path of the disk file. + """ + helper = LayerHelper("load_combine", **locals()) + helper.append_op( + type="load_combine", + inputs={}, + output={"Out": out}, + args={"file_path": file_path}) From 15231f0c71e5ce932041ca4761dd15e39c779dc9 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Sat, 3 Mar 2018 20:27:54 +0800 Subject: [PATCH 2/9] Refine the doc for build_and_install. --- doc/build_and_install/index_cn.rst | 55 ++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/doc/build_and_install/index_cn.rst b/doc/build_and_install/index_cn.rst index c0b60f5589..f6611330ac 100644 --- a/doc/build_and_install/index_cn.rst +++ b/doc/build_and_install/index_cn.rst @@ -3,30 +3,55 @@ .. _install_steps: -安装流程 -++++++++ +PaddlePaddle针对不同的用户群体提供了多种安装方式。 -PaddlePaddle提供pip和Docker的安装方式： +专注深度学习模型开发 +----------------- + +PaddlePaddle提供了多种python wheel包，可通过pip一键安装： + +.. toctree:: + :maxdepth: 1 + + pip_install_cn.rst + +这是最便捷的安装方式，请根据机器配置和系统选择对应的安装包。 + +关注底层框架 +---------- + +PaddlePaddle提供了基于Docker的安装方式，请参照以下教程： + +.. toctree:: + :maxdepth: 1 + + docker_install_cn.rst + +我们推荐在Docker中运行PaddlePaddle，该方式具有以下优势： + +- 无需单独安装第三方依赖 +- 方便分享运行时环境，易于问题的复现 + +对于有定制化二进制文件需求的用户，我们同样提供了从源码编译安装PaddlePaddle的方法： .. toctree:: :maxdepth: 1 - pip_install_cn.rst - docker_install_cn.rst + build_from_source_cn.rst + + +.. warning:: -编译流程 -++++++++ + 需要提醒的是，这种安装方式会涉及到一些第三方库的下载、编译及安装，安装过程较长。 -.. warning:: - 建议直接使用上述安装流程，方便快速安装。只有在遇到需要独立定制的二进制时才需要编译。 +常见问题汇总 +----------- -.. toctree:: - :maxdepth: 1 +如果在安装过程中遇到了问题，请先尝试在下面的页面寻找答案： - build_from_source_cn.rst +:ref:`faq/build_and_install/index_cn.rst` -常见问题解答 -++++++++++ +如果问题没有得到解决，欢迎向PaddlePaddle社区反馈问题： -`常见问题解答 `_ +`创建issue `_ From 5fc1e38273860317ad1ab0fe322d2326d235325a Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Mon, 5 Mar 2018 12:00:35 +0800 Subject: [PATCH 3/9] Add guideline for the doc of cmd parameters --- doc/howto/cmd_parameter/index_cn.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/howto/cmd_parameter/index_cn.rst b/doc/howto/cmd_parameter/index_cn.rst index 17b379f629..6900bb1443 100644 --- a/doc/howto/cmd_parameter/index_cn.rst +++ b/doc/howto/cmd_parameter/index_cn.rst @@ -2,10 +2,25 @@ 命令行参数设置 =============== +深度学习算法的实现有着多样化的特点，运行环境、运行阶段、模型结构、训练策略等等这些都是常见的变化因素。PaddlePaddle支持用户灵活地设置各种命令行参数，以实现对模型训练或预测流程的控制。 + +在这一部分，首先以几个实际场景为例，展示了部分命令行参数的使用: .. toctree:: :maxdepth: 1 use_case_cn.md + +接着对所有参数的使用场合进行概述和分类: + +.. toctree:: + :maxdepth: 1 + arguments_cn.md + +最后给出细节描述，详细解释这些参数的属性和意义: + +.. toctree:: + :maxdepth: 1 + detail_introduction_cn.md From 1b28739cb9eba76d96d517ead3d65475628ff6ce Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 5 Mar 2018 13:31:39 +0800 Subject: [PATCH 4/9] update doc string --- python/paddle/fluid/layers/nn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 1acc74f757..836ddbbeb0 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3289,7 +3289,7 @@ def save(x, file_path, overwrite=True): def save_combine(x, file_path, overwrite=True): """ - Saves a variable as a file. + Saves a list of variables into a single file. Args: x(list): A list of Tensor/LoDTensor to be saved together in a single file. @@ -3309,6 +3309,8 @@ def save_combine(x, file_path, overwrite=True): def load(out, file_path): """ + Loads a variable from a given file. + Args: out(variable): The variable to be read from the disk file. file_path(str): The path of the disk file. @@ -3323,6 +3325,8 @@ def load(out, file_path): def load_combine(out, file_path): """ + Loads a list of vairables from a single file. + Args: out(list): The list of variables to be read from the disk file. file_path(str): The path of the disk file. From bbf98a017677082a436f07da47f04103b7bb0aa9 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 5 Mar 2018 13:47:53 +0800 Subject: [PATCH 5/9] Refine the doc in detection_output API. (#8689) * Refine the doc in detection_output API. * Refine the doc. --- python/paddle/fluid/layers/detection.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index a077c0ce33..d16b4dc3a4 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -54,11 +54,17 @@ def detection_output(loc, score_threshold=0.01, nms_eta=1.0): """ - **Detection Output Layer** + **Detection Output Layer for Single Shot Multibox Detector (SSD).** - This layer applies the NMS to the output of network and computes the - predict bounding box location. The output's shape of this layer could - be zero if there is no valid bounding box. + This operation is to get the detection results by performing following + two steps: + + 1. Decode input bounding box predictions according to the prior boxes. + 2. Get the final detection results by applying multi-class non maximum + suppression (NMS). + + Please note, this operation doesn't clip the final output bounding boxes + to the image window. Args: loc(Variable): A 3-D Tensor with shape [N, M, 4] represents the @@ -91,7 +97,15 @@ def detection_output(loc, nms_eta(float): The parameter for adaptive NMS. Returns: - The detected bounding boxes which are a Tensor. + Variable: The detection outputs is a LoDTensor with shape [No, 6]. + Each row has six values: [label, confidence, xmin, ymin, xmax, ymax]. + `No` is the total number of detections in this mini-batch. For each + instance, the offsets in first dimension are called LoD, the offset + number is N + 1, N is the batch size. The i-th image has + `LoD[i + 1] - LoD[i]` detected results, if it is 0, the i-th image + has no detected results. If all images have not detected results, + all the elements in LoD are 0, and output tensor only contains one + value, which is -1. Examples: .. code-block:: python From 649f9c19638e377fa165aa3c9d257702c99422be Mon Sep 17 00:00:00 2001 From: yangyaming Date: Mon, 5 Mar 2018 13:48:12 +0800 Subject: [PATCH 6/9] Refine the document for build_and_install. --- doc/build_and_install/index_cn.rst | 9 ++++----- doc/faq/build_and_install/index_cn.rst | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/build_and_install/index_cn.rst b/doc/build_and_install/index_cn.rst index f6611330ac..e079bb661f 100644 --- a/doc/build_and_install/index_cn.rst +++ b/doc/build_and_install/index_cn.rst @@ -35,14 +35,13 @@ PaddlePaddle提供了基于Docker的安装方式，请参照以下教程： 对于有定制化二进制文件需求的用户，我们同样提供了从源码编译安装PaddlePaddle的方法： .. toctree:: - :maxdepth: 1 - - build_from_source_cn.rst + :maxdepth: 1 + build_from_source_cn.rst .. warning:: - 需要提醒的是，这种安装方式会涉及到一些第三方库的下载、编译及安装，安装过程较长。 + 需要提醒的是，这种安装方式会涉及到一些第三方库的下载、编译及安装，整个安装过程耗时较长。 常见问题汇总 @@ -50,7 +49,7 @@ PaddlePaddle提供了基于Docker的安装方式，请参照以下教程： 如果在安装过程中遇到了问题，请先尝试在下面的页面寻找答案： -:ref:`faq/build_and_install/index_cn.rst` +:ref:`常见问题解答 ` 如果问题没有得到解决，欢迎向PaddlePaddle社区反馈问题： diff --git a/doc/faq/build_and_install/index_cn.rst b/doc/faq/build_and_install/index_cn.rst index 31d2252eb5..7c7e896d18 100644 --- a/doc/faq/build_and_install/index_cn.rst +++ b/doc/faq/build_and_install/index_cn.rst @@ -1,3 +1,5 @@ +.. _install_faq: + ################### 编译安装与单元测试 ################### From 457d012a8e043afa43365e84ef25565a8720e01e Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 5 Mar 2018 13:48:40 +0800 Subject: [PATCH 7/9] Enable device automatically switching in mine_hard_examples_op. (#8706) --- paddle/fluid/operators/mine_hard_examples_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/operators/mine_hard_examples_op.cc b/paddle/fluid/operators/mine_hard_examples_op.cc index b7e9f4e224..0e81d60878 100644 --- a/paddle/fluid/operators/mine_hard_examples_op.cc +++ b/paddle/fluid/operators/mine_hard_examples_op.cc @@ -247,7 +247,7 @@ class MineHardExamplesOp : public framework::OperatorWithKernel { const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( framework::ToDataType(ctx.Input("ClsLoss")->type()), - ctx.device_context()); + platform::CPUPlace()); } }; From bd8440921c6dcf4df26e236d2b0698d87499c05c Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 5 Mar 2018 14:30:24 +0800 Subject: [PATCH 8/9] move save/load from nn.py to tensor.py --- python/paddle/fluid/layers/nn.py | 72 ---------------------------- python/paddle/fluid/layers/tensor.py | 72 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 72 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 836ddbbeb0..e10a01a5d7 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3265,75 +3265,3 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): counter.stop_gradient = True return counter - - -def save(x, file_path, overwrite=True): - """ - Saves a variable as a file. - - Args: - x(variable): The Tensor/LoDTensor to be saved. - file_path(str): The file path where the variable will be saved. - overwrite(bool): Whether or not cover the given file when it has already - existed. If it's set 'False' and the file is existed, a runtime - error will be thrown. - """ - helper = LayerHelper("save", **locals()) - helper.append_op( - type="save", - inputs={"input": x}, - outputs={}, - args={"file_path": file_path, - "overwrite": overwrite}) - - -def save_combine(x, file_path, overwrite=True): - """ - Saves a list of variables into a single file. - - Args: - x(list): A list of Tensor/LoDTensor to be saved together in a single file. - file_path(str): The file path where variables will be saved. - overwrite(bool): Whether or not cover the given file when it has already - existed. If it's set 'False' and the file is existed, a runtime - error will be thrown. - """ - helper = LayerHelper("save_combine", **locals()) - helper.append_op( - type="save_combine", - inputs={"input": x}, - outputs={}, - args={"file_path": file_path, - "overwrite": overwrite}) - - -def load(out, file_path): - """ - Loads a variable from a given file. - - Args: - out(variable): The variable to be read from the disk file. - file_path(str): The path of the disk file. - """ - helper = LayerHelper("load", **locals()) - helper.append_op( - type="load", - inputs={}, - output={"Out": out}, - args={"file_path": file_path}) - - -def load_combine(out, file_path): - """ - Loads a list of vairables from a single file. - - Args: - out(list): The list of variables to be read from the disk file. - file_path(str): The path of the disk file. - """ - helper = LayerHelper("load_combine", **locals()) - helper.append_op( - type="load_combine", - inputs={}, - output={"Out": out}, - args={"file_path": file_path}) diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 8100e8f034..da066c34bd 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -362,3 +362,75 @@ def zeros(shape, dtype, force_cpu=False): data = fluid.layers.zeros(shape=[1], dtype='int64') """ return fill_constant(value=0.0, **locals()) + + +def save(x, file_path, overwrite=True): + """ + Saves a variable as a file. + + Args: + x(variable): The Tensor/LoDTensor to be saved. + file_path(str): The file path where the variable will be saved. + overwrite(bool): Whether or not cover the given file when it has already + existed. If it's set 'False' and the file is existed, a runtime + error will be thrown. + """ + helper = LayerHelper("save", **locals()) + helper.append_op( + type="save", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, + "overwrite": overwrite}) + + +def save_combine(x, file_path, overwrite=True): + """ + Saves a list of variables into a single file. + + Args: + x(list): A list of Tensor/LoDTensor to be saved together in a single file. + file_path(str): The file path where variables will be saved. + overwrite(bool): Whether or not cover the given file when it has already + existed. If it's set 'False' and the file is existed, a runtime + error will be thrown. + """ + helper = LayerHelper("save_combine", **locals()) + helper.append_op( + type="save_combine", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, + "overwrite": overwrite}) + + +def load(out, file_path): + """ + Loads a variable from a given file. + + Args: + out(variable): The variable to be read from the disk file. + file_path(str): The path of the disk file. + """ + helper = LayerHelper("load", **locals()) + helper.append_op( + type="load", + inputs={}, + output={"Out": out}, + args={"file_path": file_path}) + + +def load_combine(out, file_path): + """ + Loads a list of vairables from a single file. + + Args: + out(list): The list of variables to be read from the disk file. + file_path(str): The path of the disk file. + """ + helper = LayerHelper("load_combine", **locals()) + helper.append_op( + type="load_combine", + inputs={}, + output={"Out": out}, + args={"file_path": file_path}) From ac149bc948046b8baf0155f483193ed7f409b6eb Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Mon, 5 Mar 2018 14:46:20 +0800 Subject: [PATCH 9/9] fix warning: statement is unreachable --- paddle/fluid/framework/dim.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/framework/dim.h b/paddle/fluid/framework/dim.h index 58b75ba4b5..73f92fa389 100644 --- a/paddle/fluid/framework/dim.h +++ b/paddle/fluid/framework/dim.h @@ -157,8 +157,7 @@ HOSTDEVICE int64_t& indexer<0>(Dim<0>& dim, int idx) { throw std::invalid_argument("Invalid index"); #else PADDLE_ASSERT(false); -#endif -#if (defined __CUDA_ARCH__) && (CUDA_VERSION < 8000) +#if CUDA_VERSION < 8000 // On CUDA versions previous to 8.0, only __shared__ variables // could be declared as static in the device code. int64_t head = 0; @@ -166,6 +165,7 @@ HOSTDEVICE int64_t& indexer<0>(Dim<0>& dim, int idx) { static int64_t head = 0; #endif return head; +#endif } template @@ -189,8 +189,7 @@ HOSTDEVICE int64_t indexer<0>(const Dim<0>& dim, int idx) { throw std::invalid_argument("Invalid index"); #else PADDLE_ASSERT(false); -#endif -#if (defined __CUDA_ARCH__) && (CUDA_VERSION < 8000) +#if CUDA_VERSION < 8000 // On CUDA versions previous to 8.0, only __shared__ variables // could be declared as static in the device code. int64_t head = 0; @@ -198,6 +197,7 @@ HOSTDEVICE int64_t indexer<0>(const Dim<0>& dim, int idx) { static int64_t head = 0; #endif return head; +#endif } } // namespace