Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into refine-prefetch
qiaolongfei
7 years ago
commit
fe65064827
@ -0,0 +1,126 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
|
||||
__all__ = ['parse_args', ]
|
||||
|
||||
BENCHMARK_MODELS = [
|
||||
"machine_translation", "resnet", "vgg", "mnist", "stacked_dynamic_lstm"
|
||||
]
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser('Fluid model benchmarks.')
|
||||
parser.add_argument(
|
||||
'--model',
|
||||
type=str,
|
||||
choices=BENCHMARK_MODELS,
|
||||
default='resnet',
|
||||
help='The model to run benchmark with.')
|
||||
parser.add_argument(
|
||||
'--batch_size', type=int, default=32, help='The minibatch size.')
|
||||
# args related to learning rate
|
||||
parser.add_argument(
|
||||
'--learning_rate', type=float, default=0.001, help='The learning rate.')
|
||||
# TODO(wuyi): add "--use_fake_data" option back.
|
||||
parser.add_argument(
|
||||
'--skip_batch_num',
|
||||
type=int,
|
||||
default=5,
|
||||
help='The first num of minibatch num to skip, for better performance test'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--iterations', type=int, default=80, help='The number of minibatches.')
|
||||
parser.add_argument(
|
||||
'--pass_num', type=int, default=100, help='The number of passes.')
|
||||
parser.add_argument(
|
||||
'--data_format',
|
||||
type=str,
|
||||
default='NCHW',
|
||||
choices=['NCHW', 'NHWC'],
|
||||
help='The data data_format, now only support NCHW.')
|
||||
parser.add_argument(
|
||||
'--device',
|
||||
type=str,
|
||||
default='GPU',
|
||||
choices=['CPU', 'GPU'],
|
||||
help='The device type.')
|
||||
parser.add_argument(
|
||||
'--gpus',
|
||||
type=int,
|
||||
default=1,
|
||||
help='If gpus > 1, will use ParallelExecutor to run, else use Executor.')
|
||||
# this option is available only for vgg and resnet.
|
||||
parser.add_argument(
|
||||
'--cpus',
|
||||
type=int,
|
||||
default=1,
|
||||
help='If cpus > 1, will use ParallelDo to run, else use Executor.')
|
||||
parser.add_argument(
|
||||
'--data_set',
|
||||
type=str,
|
||||
default='flowers',
|
||||
choices=['cifar10', 'flowers'],
|
||||
help='Optional dataset for benchmark.')
|
||||
parser.add_argument(
|
||||
'--infer_only', action='store_true', help='If set, run forward only.')
|
||||
parser.add_argument(
|
||||
'--use_cprof', action='store_true', help='If set, use cProfile.')
|
||||
parser.add_argument(
|
||||
'--use_nvprof',
|
||||
action='store_true',
|
||||
help='If set, use nvprof for CUDA.')
|
||||
parser.add_argument(
|
||||
'--no_test',
|
||||
action='store_true',
|
||||
help='If set, do not test the testset during training.')
|
||||
parser.add_argument(
|
||||
'--memory_optimize',
|
||||
action='store_true',
|
||||
help='If set, optimize runtime memory before start.')
|
||||
parser.add_argument(
|
||||
'--use_fake_data',
|
||||
action='store_true',
|
||||
help='If set ommit the actual read data operators.')
|
||||
parser.add_argument(
|
||||
'--profile', action='store_true', help='If set, profile a few steps.')
|
||||
parser.add_argument(
|
||||
'--update_method',
|
||||
type=str,
|
||||
default='local',
|
||||
choices=['local', 'pserver', 'nccl2'],
|
||||
help='Choose parameter update method, can be local, pserver, nccl2.')
|
||||
parser.add_argument(
|
||||
'--no_split_var',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether split variables into blocks when update_method is pserver')
|
||||
parser.add_argument(
|
||||
'--async_mode',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether start pserver in async mode to support ASGD')
|
||||
parser.add_argument(
|
||||
'--use_reader_op',
|
||||
action='store_true',
|
||||
help='Whether to use reader op, and must specify the data path if set this to true.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--data_path',
|
||||
type=str,
|
||||
default="",
|
||||
help='Directory that contains all the training recordio files.')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(BRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/brpc)
|
||||
SET(BRPC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/brpc)
|
||||
SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include directory." FORCE)
|
||||
SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc library." FORCE)
|
||||
|
||||
INCLUDE_DIRECTORIES(${BRPC_INCLUDE_DIR})
|
||||
|
||||
# Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
|
||||
set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf")
|
||||
|
||||
# If minimal .a is need, you can set WITH_DEBUG_SYMBOLS=OFF
|
||||
ExternalProject_Add(
|
||||
extern_brpc
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
GIT_REPOSITORY "https://github.com/brpc/brpc"
|
||||
GIT_TAG "6d153dd7ff00f960ae6895c9c5fff0ce9f07aff2"
|
||||
PREFIX ${BRPC_SOURCES_DIR}
|
||||
UPDATE_COMMAND ""
|
||||
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
|
||||
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
|
||||
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
|
||||
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
|
||||
-DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR}
|
||||
-DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
||||
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
|
||||
-DCMAKE_PREFIX_PATH=${prefix_path}
|
||||
-DBRPC_WITH_GLOG=ON
|
||||
${EXTERNAL_OPTIONAL_ARGS}
|
||||
LIST_SEPARATOR |
|
||||
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRPC_INSTALL_DIR}
|
||||
-DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
|
||||
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
|
||||
)
|
||||
ADD_DEPENDENCIES(extern_brpc protobuf leveldb gflags glog gtest snappy)
|
||||
ADD_LIBRARY(brpc STATIC IMPORTED GLOBAL)
|
||||
SET_PROPERTY(TARGET brpc PROPERTY IMPORTED_LOCATION ${BRPC_LIBRARIES})
|
||||
ADD_DEPENDENCIES(brpc extern_brpc)
|
||||
|
||||
|
||||
LIST(APPEND external_project_dependencies brpc)
|
||||
@ -0,0 +1,44 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(LEVELDB_SOURCES_DIR ${THIRD_PARTY_PATH}/leveldb)
|
||||
SET(LEVELDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/leveldb)
|
||||
SET(LEVELDB_INCLUDE_DIR "${LEVELDB_INSTALL_DIR}/include" CACHE PATH "leveldb include directory." FORCE)
|
||||
SET(LEVELDB_LIBRARIES "${LEVELDB_INSTALL_DIR}/lib/libleveldb.a" CACHE FILEPATH "leveldb library." FORCE)
|
||||
INCLUDE_DIRECTORIES(${LEVELDB_INCLUDE_DIR})
|
||||
|
||||
ExternalProject_Add(
|
||||
extern_leveldb
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
PREFIX ${LEVELDB_SOURCES_DIR}
|
||||
URL "https://github.com/google/leveldb/archive/v1.18.tar.gz"
|
||||
URL_MD5 "73770de34a2a5ab34498d2e05b2b7fa0"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND CXXFLAGS=-fPIC make -j ${NUM_OF_PROCESSOR} libleveldb.a
|
||||
INSTALL_COMMAND mkdir -p ${LEVELDB_INSTALL_DIR}/lib/
|
||||
&& cp ${LEVELDB_SOURCES_DIR}/src/extern_leveldb/libleveldb.a ${LEVELDB_LIBRARIES}
|
||||
&& cp -r ${LEVELDB_SOURCES_DIR}/src/extern_leveldb/include ${LEVELDB_INSTALL_DIR}/
|
||||
BUILD_IN_SOURCE 1
|
||||
)
|
||||
|
||||
ADD_DEPENDENCIES(extern_leveldb snappy)
|
||||
|
||||
ADD_LIBRARY(leveldb STATIC IMPORTED GLOBAL)
|
||||
SET_PROPERTY(TARGET leveldb PROPERTY IMPORTED_LOCATION ${LEVELDB_LIBRARIES})
|
||||
ADD_DEPENDENCIES(leveldb extern_leveldb)
|
||||
|
||||
LIST(APPEND external_project_dependencies leveldb)
|
||||
|
||||
@ -0,0 +1,87 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/framework/details/ssa_graph.h"
|
||||
#include <string>
|
||||
#include "paddle/fluid/framework/details/ssa_graph_checker.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
namespace details {
|
||||
|
||||
bool SSAGraghBuilderWithChecker::IsValidGraph(const SSAGraph *graph) const {
|
||||
std::unordered_map<OpHandleBase *, size_t> pending_ops;
|
||||
std::unordered_set<VarHandleBase *> pending_vars;
|
||||
std::unordered_set<VarHandleBase *> ready_vars;
|
||||
std::unordered_set<OpHandleBase *> ready_ops;
|
||||
|
||||
auto insert_pending_var = [&](VarHandleBase *var) {
|
||||
pending_vars.insert(var);
|
||||
if (var->generated_op_ == nullptr) {
|
||||
ready_vars.emplace(var);
|
||||
}
|
||||
};
|
||||
|
||||
for (auto &var_map : graph->vars_) {
|
||||
for (auto &name_pair : var_map) {
|
||||
for (auto &version_pair : name_pair.second) {
|
||||
insert_pending_var(version_pair.get());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &var : graph->dep_vars_) {
|
||||
insert_pending_var(var.get());
|
||||
}
|
||||
|
||||
for (auto &op : graph->ops_) {
|
||||
if (op->Inputs().empty()) {
|
||||
ready_ops.insert(op.get());
|
||||
} else {
|
||||
pending_ops.insert({op.get(), op.get()->NoDupInputSize()});
|
||||
}
|
||||
}
|
||||
|
||||
auto run_all_ops = [&](std::unordered_set<OpHandleBase *> &set) {
|
||||
for (auto *op : set) {
|
||||
for (auto out : op->Outputs()) {
|
||||
ready_vars.emplace(out);
|
||||
}
|
||||
}
|
||||
set.clear();
|
||||
};
|
||||
|
||||
while (!pending_vars.empty()) {
|
||||
run_all_ops(ready_ops);
|
||||
|
||||
if (ready_vars.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto ready_var : ready_vars) {
|
||||
pending_vars.erase(ready_var);
|
||||
for (auto *op : ready_var->pending_ops_) {
|
||||
auto &deps = --pending_ops[op];
|
||||
if (deps == 0) {
|
||||
ready_ops.insert(op);
|
||||
}
|
||||
}
|
||||
}
|
||||
ready_vars.clear();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace details
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,44 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/fluid/framework/details/ssa_graph_builder.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
namespace details {
|
||||
class SSAGraph;
|
||||
|
||||
class SSAGraghBuilderWithChecker : public SSAGraphBuilder {
|
||||
public:
|
||||
explicit SSAGraghBuilderWithChecker(
|
||||
std::unique_ptr<SSAGraphBuilder>&& builder)
|
||||
: builder_(std::move(builder)) {}
|
||||
|
||||
std::unique_ptr<SSAGraph> Build(const ProgramDesc& program) const override {
|
||||
auto graph = builder_->Build(program);
|
||||
PADDLE_ENFORCE(IsValidGraph(graph.get()));
|
||||
return graph;
|
||||
}
|
||||
|
||||
bool IsValidGraph(const SSAGraph* graph) const;
|
||||
|
||||
private:
|
||||
std::unique_ptr<SSAGraphBuilder> builder_;
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue