revert-11610-move_hooks
Yancey1989
7 years ago
commit
4444e79e46
@ -0,0 +1,126 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
|
||||
__all__ = ['parse_args', ]
|
||||
|
||||
BENCHMARK_MODELS = [
|
||||
"machine_translation", "resnet", "vgg", "mnist", "stacked_dynamic_lstm"
|
||||
]
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser('Fluid model benchmarks.')
|
||||
parser.add_argument(
|
||||
'--model',
|
||||
type=str,
|
||||
choices=BENCHMARK_MODELS,
|
||||
default='resnet',
|
||||
help='The model to run benchmark with.')
|
||||
parser.add_argument(
|
||||
'--batch_size', type=int, default=32, help='The minibatch size.')
|
||||
# args related to learning rate
|
||||
parser.add_argument(
|
||||
'--learning_rate', type=float, default=0.001, help='The learning rate.')
|
||||
# TODO(wuyi): add "--use_fake_data" option back.
|
||||
parser.add_argument(
|
||||
'--skip_batch_num',
|
||||
type=int,
|
||||
default=5,
|
||||
help='The first num of minibatch num to skip, for better performance test'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--iterations', type=int, default=80, help='The number of minibatches.')
|
||||
parser.add_argument(
|
||||
'--pass_num', type=int, default=100, help='The number of passes.')
|
||||
parser.add_argument(
|
||||
'--data_format',
|
||||
type=str,
|
||||
default='NCHW',
|
||||
choices=['NCHW', 'NHWC'],
|
||||
help='The data data_format, now only support NCHW.')
|
||||
parser.add_argument(
|
||||
'--device',
|
||||
type=str,
|
||||
default='GPU',
|
||||
choices=['CPU', 'GPU'],
|
||||
help='The device type.')
|
||||
parser.add_argument(
|
||||
'--gpus',
|
||||
type=int,
|
||||
default=1,
|
||||
help='If gpus > 1, will use ParallelExecutor to run, else use Executor.')
|
||||
# this option is available only for vgg and resnet.
|
||||
parser.add_argument(
|
||||
'--cpus',
|
||||
type=int,
|
||||
default=1,
|
||||
help='If cpus > 1, will use ParallelDo to run, else use Executor.')
|
||||
parser.add_argument(
|
||||
'--data_set',
|
||||
type=str,
|
||||
default='flowers',
|
||||
choices=['cifar10', 'flowers'],
|
||||
help='Optional dataset for benchmark.')
|
||||
parser.add_argument(
|
||||
'--infer_only', action='store_true', help='If set, run forward only.')
|
||||
parser.add_argument(
|
||||
'--use_cprof', action='store_true', help='If set, use cProfile.')
|
||||
parser.add_argument(
|
||||
'--use_nvprof',
|
||||
action='store_true',
|
||||
help='If set, use nvprof for CUDA.')
|
||||
parser.add_argument(
|
||||
'--no_test',
|
||||
action='store_true',
|
||||
help='If set, do not test the testset during training.')
|
||||
parser.add_argument(
|
||||
'--memory_optimize',
|
||||
action='store_true',
|
||||
help='If set, optimize runtime memory before start.')
|
||||
parser.add_argument(
|
||||
'--use_fake_data',
|
||||
action='store_true',
|
||||
help='If set ommit the actual read data operators.')
|
||||
parser.add_argument(
|
||||
'--profile', action='store_true', help='If set, profile a few steps.')
|
||||
parser.add_argument(
|
||||
'--update_method',
|
||||
type=str,
|
||||
default='local',
|
||||
choices=['local', 'pserver', 'nccl2'],
|
||||
help='Choose parameter update method, can be local, pserver, nccl2.')
|
||||
parser.add_argument(
|
||||
'--no_split_var',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether split variables into blocks when update_method is pserver')
|
||||
parser.add_argument(
|
||||
'--async_mode',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether start pserver in async mode to support ASGD')
|
||||
parser.add_argument(
|
||||
'--use_reader_op',
|
||||
action='store_true',
|
||||
help='Whether to use reader op, and must specify the data path if set this to true.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--data_path',
|
||||
type=str,
|
||||
default="",
|
||||
help='Directory that contains all the training recordio files.')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(BRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/brpc)
|
||||
SET(BRPC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/brpc)
|
||||
SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include directory." FORCE)
|
||||
SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc library." FORCE)
|
||||
|
||||
INCLUDE_DIRECTORIES(${BRPC_INCLUDE_DIR})
|
||||
|
||||
# Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
|
||||
set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf")
|
||||
|
||||
# If minimal .a is need, you can set WITH_DEBUG_SYMBOLS=OFF
|
||||
ExternalProject_Add(
|
||||
extern_brpc
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
GIT_REPOSITORY "https://github.com/brpc/brpc"
|
||||
GIT_TAG "6d153dd7ff00f960ae6895c9c5fff0ce9f07aff2"
|
||||
PREFIX ${BRPC_SOURCES_DIR}
|
||||
UPDATE_COMMAND ""
|
||||
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
|
||||
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
|
||||
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
|
||||
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
|
||||
-DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR}
|
||||
-DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
||||
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
|
||||
-DCMAKE_PREFIX_PATH=${prefix_path}
|
||||
-DBRPC_WITH_GLOG=ON
|
||||
${EXTERNAL_OPTIONAL_ARGS}
|
||||
LIST_SEPARATOR |
|
||||
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRPC_INSTALL_DIR}
|
||||
-DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
|
||||
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
|
||||
)
|
||||
ADD_DEPENDENCIES(extern_brpc protobuf leveldb gflags glog gtest snappy)
|
||||
ADD_LIBRARY(brpc STATIC IMPORTED GLOBAL)
|
||||
SET_PROPERTY(TARGET brpc PROPERTY IMPORTED_LOCATION ${BRPC_LIBRARIES})
|
||||
ADD_DEPENDENCIES(brpc extern_brpc)
|
||||
|
||||
|
||||
LIST(APPEND external_project_dependencies brpc)
|
||||
@ -0,0 +1,44 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(LEVELDB_SOURCES_DIR ${THIRD_PARTY_PATH}/leveldb)
|
||||
SET(LEVELDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/leveldb)
|
||||
SET(LEVELDB_INCLUDE_DIR "${LEVELDB_INSTALL_DIR}/include" CACHE PATH "leveldb include directory." FORCE)
|
||||
SET(LEVELDB_LIBRARIES "${LEVELDB_INSTALL_DIR}/lib/libleveldb.a" CACHE FILEPATH "leveldb library." FORCE)
|
||||
INCLUDE_DIRECTORIES(${LEVELDB_INCLUDE_DIR})
|
||||
|
||||
ExternalProject_Add(
|
||||
extern_leveldb
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
PREFIX ${LEVELDB_SOURCES_DIR}
|
||||
URL "https://github.com/google/leveldb/archive/v1.18.tar.gz"
|
||||
URL_MD5 "73770de34a2a5ab34498d2e05b2b7fa0"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND CXXFLAGS=-fPIC make -j ${NUM_OF_PROCESSOR} libleveldb.a
|
||||
INSTALL_COMMAND mkdir -p ${LEVELDB_INSTALL_DIR}/lib/
|
||||
&& cp ${LEVELDB_SOURCES_DIR}/src/extern_leveldb/libleveldb.a ${LEVELDB_LIBRARIES}
|
||||
&& cp -r ${LEVELDB_SOURCES_DIR}/src/extern_leveldb/include ${LEVELDB_INSTALL_DIR}/
|
||||
BUILD_IN_SOURCE 1
|
||||
)
|
||||
|
||||
ADD_DEPENDENCIES(extern_leveldb snappy)
|
||||
|
||||
ADD_LIBRARY(leveldb STATIC IMPORTED GLOBAL)
|
||||
SET_PROPERTY(TARGET leveldb PROPERTY IMPORTED_LOCATION ${LEVELDB_LIBRARIES})
|
||||
ADD_DEPENDENCIES(leveldb extern_leveldb)
|
||||
|
||||
LIST(APPEND external_project_dependencies leveldb)
|
||||
|
||||
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue