Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into cpp_parallel_executor
Yang Yu
7 years ago
commit
b0775588c0
@ -0,0 +1,231 @@
|
||||
# go_op Design
|
||||
|
||||
## Introduction
|
||||
|
||||
The **go_op** allows user's of PaddlePaddle to run program blocks on a detached
|
||||
thread. It works in conjuction with CSP operators (channel_send,
|
||||
channel_receive, channel_open, channel_close, and select) to allow users to
|
||||
concurrently process data and communicate easily between different threads.
|
||||
|
||||
## How to use it
|
||||
|
||||
```
|
||||
channel = fluid.make_channel(dtype=core.VarDesc.VarType.LOD_TENSOR)
|
||||
|
||||
with fluid.Go():
|
||||
# Send a tensor of value 99 to "channel" on a detached thread
|
||||
tensor = fill_constant(shape=[1], dtype='int', value=99)
|
||||
tensor.stop_gradient = True
|
||||
fluid.channel_send(channel, tensor)
|
||||
|
||||
# Receive sent tensor from "channel" on the main thread
|
||||
result = fill_constant(shape=[1], dtype='int', value=-1)
|
||||
fluid.channel_recv(ch, result)
|
||||
```
|
||||
|
||||
The go operator can be accessed by using the fluid.Go() control flow. This
|
||||
will create a new sub block, where the user can add additional operators
|
||||
to be ran on the thread.
|
||||
|
||||
**Note:** Since back propegation is currently not support in the go_op, users
|
||||
should ensure that operators in the go block does not require gradient
|
||||
calculations.
|
||||
|
||||
## How it Works
|
||||
|
||||
Similar to other control blocks, go_op will create a sub block and add it
|
||||
as a child to the current block. Operators and variables defined in this
|
||||
block will be added to the go sub_block.
|
||||
|
||||
In addition, the go operator will create a new child scope whose parent is
|
||||
the global scope. Please refer to [block captures](#block-captures) for more
|
||||
information.
|
||||
|
||||
When Paddle executor runs go_op, go_op will take the sub_block and pass it to
|
||||
the executor.run method (along with a newly created local scope) on a detached
|
||||
thread.
|
||||
|
||||
An example of the generated program description is shown below. Take note of
|
||||
the **go_op** in particular. It is added as an operator in the current
|
||||
block (in this example, block0). The **go_op** contains a `sub_block`
|
||||
attribute, which points to the id of the block that will be executed in a
|
||||
detached thread.
|
||||
|
||||
```
|
||||
blocks {
|
||||
idx: 0
|
||||
parent_idx: -1
|
||||
vars {
|
||||
name: "return_value"
|
||||
type {
|
||||
type: LOD_TENSOR
|
||||
lod_tensor {
|
||||
tensor {
|
||||
data_type: INT64
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
vars {
|
||||
name: "status_recv"
|
||||
type {
|
||||
type: LOD_TENSOR
|
||||
lod_tensor {
|
||||
tensor {
|
||||
data_type: BOOL
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
...
|
||||
ops {
|
||||
outputs {
|
||||
parameter: "Out"
|
||||
arguments: "channel"
|
||||
}
|
||||
type: "channel_create"
|
||||
attrs {
|
||||
name: "data_type"
|
||||
type: INT
|
||||
i: 7
|
||||
}
|
||||
attrs {
|
||||
name: "capacity"
|
||||
type: INT
|
||||
i: 0
|
||||
}
|
||||
}
|
||||
ops {
|
||||
inputs {
|
||||
parameter: "X"
|
||||
arguments: "channel"
|
||||
}
|
||||
type: "go"
|
||||
attrs {
|
||||
name: "sub_block"
|
||||
type: BLOCK
|
||||
block_idx: 1
|
||||
}
|
||||
}
|
||||
ops {
|
||||
inputs {
|
||||
parameter: "Channel"
|
||||
arguments: "channel"
|
||||
}
|
||||
outputs {
|
||||
parameter: "Out"
|
||||
arguments: "return_value"
|
||||
}
|
||||
outputs {
|
||||
parameter: "Status"
|
||||
arguments: "status_recv"
|
||||
}
|
||||
type: "channel_recv"
|
||||
}
|
||||
...
|
||||
}
|
||||
|
||||
blocks {
|
||||
idx: 1
|
||||
parent_idx: 0
|
||||
vars {
|
||||
name: "status"
|
||||
type {
|
||||
type: LOD_TENSOR
|
||||
lod_tensor {
|
||||
tensor {
|
||||
data_type: BOOL
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
...
|
||||
|
||||
ops {
|
||||
outputs {
|
||||
parameter: "Out"
|
||||
arguments: "fill_constant_1.tmp_0"
|
||||
}
|
||||
type: "fill_constant"
|
||||
attrs {
|
||||
name: "force_cpu"
|
||||
type: BOOLEAN
|
||||
b: false
|
||||
}
|
||||
attrs {
|
||||
name: "value"
|
||||
type: FLOAT
|
||||
f: 99.0
|
||||
}
|
||||
attrs {
|
||||
name: "shape"
|
||||
type: INTS
|
||||
ints: 1
|
||||
}
|
||||
attrs {
|
||||
name: "dtype"
|
||||
type: INT
|
||||
i: 3
|
||||
}
|
||||
}
|
||||
ops {
|
||||
inputs {
|
||||
parameter: "Channel"
|
||||
arguments: "channel"
|
||||
}
|
||||
inputs {
|
||||
parameter: "X"
|
||||
arguments: "fill_constant_1.tmp_0"
|
||||
}
|
||||
outputs {
|
||||
parameter: "Status"
|
||||
arguments: "status"
|
||||
}
|
||||
type: "channel_send"
|
||||
attrs {
|
||||
name: "copy"
|
||||
type: BOOLEAN
|
||||
b: false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Current Limitations
|
||||
|
||||
#### <a name="block-captures"></a>Scopes and block captures:
|
||||
|
||||
Paddle utilizes [scopes](./../concepts/scope.md) to store variables used in a
|
||||
block. When a block is executed, a new local scope is created from the parent
|
||||
scope (ie: scope derived from the parent block) and associated with the new
|
||||
child block. After the block finishes executing, then the local scope and
|
||||
all associated variables in the scope is deleted.
|
||||
|
||||
This works well in a single threaded scenario, however with introduction of
|
||||
go_op, a child block may continue to execute even after the parent block has
|
||||
exited. If the go_op tries to access variables located in the parent block's
|
||||
scope, it may receive a segmentation fault because the parent scope may have
|
||||
been deleted.
|
||||
|
||||
We need to implement block closures in order to prevent access to parent
|
||||
scope variables from causing a segmentation fault. As a temporary workaround,
|
||||
please ensure that all variables accessed in the go block is not destructed
|
||||
before it is being accessed. Currently, the go_op will explicitly enforce
|
||||
this requirement and raise an exception if a variable could not be found in
|
||||
the scope.
|
||||
|
||||
Please refer to [Closure issue](https://github.com/PaddlePaddle/Paddle/issues/8502)
|
||||
for more details.
|
||||
|
||||
#### Green Threads
|
||||
|
||||
Golang utilizes `green threads`, which is a mechnism for the runtime library to
|
||||
manage multiple threads (instead of natively by the OS). Green threads usually
|
||||
allows for faster thread creation and switching, as there is less overhead
|
||||
when spawning these threads. For the first version of CSP, we only support
|
||||
OS threads.
|
||||
|
||||
|
||||
#### Backward Propegation:
|
||||
|
||||
go_op currently does not support backwards propagation. Please use go_op with
|
||||
non training operators.
|
||||
@ -0,0 +1,96 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <unistd.h>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/framework/operator.h"
|
||||
#include "paddle/fluid/framework/program_desc.h"
|
||||
#include "paddle/fluid/operators/dropout_op.h"
|
||||
#include "paddle/fluid/operators/math/math_function.h"
|
||||
#include "paddle/fluid/string/printf.h"
|
||||
|
||||
namespace f = paddle::framework;
|
||||
namespace p = paddle::platform;
|
||||
namespace m = paddle::operators::math;
|
||||
|
||||
USE_OP(dropout);
|
||||
|
||||
void Compare(f::Scope& scope, p::DeviceContext& ctx) {
|
||||
// init
|
||||
auto var = scope.Var("X");
|
||||
auto tensor = var->GetMutable<f::LoDTensor>();
|
||||
tensor->Resize({10, 10});
|
||||
|
||||
std::vector<float> init;
|
||||
for (int64_t i = 0; i < 10 * 10; ++i) {
|
||||
init.push_back(1.0);
|
||||
}
|
||||
|
||||
TensorFromVector(init, ctx, tensor);
|
||||
|
||||
auto place = ctx.GetPlace();
|
||||
auto out_var = scope.Var("Out");
|
||||
auto out_tensor = out_var->GetMutable<f::LoDTensor>();
|
||||
out_tensor->Resize({10, 10});
|
||||
out_tensor->mutable_data<float>(place); // allocate
|
||||
|
||||
auto mask_var = scope.Var("Mask");
|
||||
auto mask_tensor = mask_var->GetMutable<f::LoDTensor>();
|
||||
mask_tensor->Resize({10, 10});
|
||||
mask_tensor->mutable_data<float>(place); // allocate
|
||||
|
||||
// run
|
||||
f::AttributeMap attrs;
|
||||
float dropout_prob = 0.5;
|
||||
attrs.insert({"fix_seed", 1});
|
||||
attrs.insert({"seed", 3});
|
||||
attrs.insert({"dropout_prob", dropout_prob});
|
||||
auto dropout_op = f::OpRegistry::CreateOp(
|
||||
"dropout", {{"X", {"X"}}}, {{"Out", {"Out"}}, {"Mask", {"Mask"}}}, attrs);
|
||||
|
||||
dropout_op->Run(scope, place);
|
||||
|
||||
std::vector<float> out_vec;
|
||||
TensorToVector(*out_tensor, ctx, &out_vec);
|
||||
|
||||
std::vector<float> std_out = {
|
||||
0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1,
|
||||
1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0,
|
||||
1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1,
|
||||
1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0,
|
||||
1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1};
|
||||
|
||||
EXPECT_EQ(out_vec.size(), std_out.size());
|
||||
for (uint32_t i = 0; i < out_vec.size(); i++) {
|
||||
EXPECT_EQ(out_vec[i], std_out[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Dropout, CPUDense) {
|
||||
f::Scope scope;
|
||||
p::CPUPlace place;
|
||||
p::CPUDeviceContext ctx(place);
|
||||
Compare(scope, ctx);
|
||||
}
|
||||
|
||||
TEST(Dropout, GPUDense) {
|
||||
f::Scope scope;
|
||||
p::CUDAPlace place;
|
||||
p::CUDADeviceContext ctx(place);
|
||||
Compare(scope, ctx);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue