|
|
@ -620,7 +620,23 @@ All parameter, weight, gradient are variables in Paddle.
|
|
|
|
|
|
|
|
|
|
|
|
// -- python binds for parallel executor.
|
|
|
|
// -- python binds for parallel executor.
|
|
|
|
py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
|
|
|
|
py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
|
|
|
|
py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy");
|
|
|
|
py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
|
|
|
|
|
|
|
|
ExecutionStrategy allows the user to more preciously control how to run
|
|
|
|
|
|
|
|
the program in ParallelExecutor by setting the property.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
The available properties include:
|
|
|
|
|
|
|
|
use_cuda (bool): Whether to use CUDA or not. Default True.
|
|
|
|
|
|
|
|
num_threads (int): The number of threads that used to run the
|
|
|
|
|
|
|
|
operators in ParallelExecutor. If it is not set, it will be
|
|
|
|
|
|
|
|
set in ParallelExecutor according to the device count.
|
|
|
|
|
|
|
|
Default 0.
|
|
|
|
|
|
|
|
allow_op_delay (bool): Whether to delay the communication operators
|
|
|
|
|
|
|
|
to run. Default False.
|
|
|
|
|
|
|
|
num_iteration_per_drop_scope (int): how many iterations between
|
|
|
|
|
|
|
|
the two dropping local scopes. Default 100.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
)DOC");
|
|
|
|
|
|
|
|
|
|
|
|
exec_strategy.def(py::init())
|
|
|
|
exec_strategy.def(py::init())
|
|
|
|
.def_property(
|
|
|
|
.def_property(
|
|
|
|
"num_threads",
|
|
|
|
"num_threads",
|
|
|
@ -658,7 +674,25 @@ All parameter, weight, gradient are variables in Paddle.
|
|
|
|
: ExecutionStrategy::kDefault;
|
|
|
|
: ExecutionStrategy::kDefault;
|
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy");
|
|
|
|
py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
|
|
|
|
|
|
|
|
BuildStrategy allows the user to more preciously control how to
|
|
|
|
|
|
|
|
build the SSA Graph in ParallelExecutor by setting the property.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
The available properties include:
|
|
|
|
|
|
|
|
reduce_strategy (str): There are two reduce strategies, 'AllReduce'
|
|
|
|
|
|
|
|
and 'Reduce'. If you want that all parameters will be optimized
|
|
|
|
|
|
|
|
on all devices, you can choose 'AllReduce'; if you choose
|
|
|
|
|
|
|
|
'Reduce', all parameters will be evenly allocated to different
|
|
|
|
|
|
|
|
devices for optimization, and then broadcast the optimized
|
|
|
|
|
|
|
|
parameter to other devices. Default 'AllReduce'.
|
|
|
|
|
|
|
|
gradient_scale_strategy (str): There are two ways of defining loss@grad,
|
|
|
|
|
|
|
|
'CoeffNumDevice' and 'Customized'. By default, ParallelExecutor
|
|
|
|
|
|
|
|
sets the loss@grad according to the number of devices. If you want
|
|
|
|
|
|
|
|
to customize loss@grad, you can choose 'Customized'.
|
|
|
|
|
|
|
|
Default 'CoeffNumDevice'.
|
|
|
|
|
|
|
|
debug_graphviz_path (str): Whether to write the SSA Graph to file in the
|
|
|
|
|
|
|
|
form of graphviz. It is useful for debugging. Default "".
|
|
|
|
|
|
|
|
)DOC");
|
|
|
|
|
|
|
|
|
|
|
|
py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
|
|
|
|
py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
|
|
|
|
.value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
|
|
|
|
.value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
|
|
|
|