Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into shufflechannel
shippingwang
6 years ago
commit
76c6f11584
@ -0,0 +1,78 @@
|
||||
# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
IF(NOT ${WITH_LIBMCT})
|
||||
return()
|
||||
ENDIF(NOT ${WITH_LIBMCT})
|
||||
|
||||
IF(WIN32 OR APPLE)
|
||||
MESSAGE(WARNING
|
||||
"Windows or Mac is not supported with LIBMCT in Paddle yet."
|
||||
"Force WITH_LIBMCT=OFF")
|
||||
SET(WITH_LIBMCT OFF CACHE STRING "Disable LIBMCT package in Windows and MacOS" FORCE)
|
||||
return()
|
||||
ENDIF()
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(LIBMCT_PROJECT "extern_libmct")
|
||||
IF((NOT DEFINED LIBMCT_VER) OR (NOT DEFINED LIBMCT_URL))
|
||||
MESSAGE(STATUS "use pre defined download url")
|
||||
SET(LIBMCT_VER "0.1.0" CACHE STRING "" FORCE)
|
||||
SET(LIBMCT_NAME "libmct" CACHE STRING "" FORCE)
|
||||
SET(LIBMCT_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${LIBMCT_VER}/${LIBMCT_NAME}.tar.gz" CACHE STRING "" FORCE)
|
||||
ENDIF()
|
||||
MESSAGE(STATUS "LIBMCT_NAME: ${LIBMCT_NAME}, LIBMCT_URL: ${LIBMCT_URL}")
|
||||
SET(LIBMCT_SOURCE_DIR "${THIRD_PARTY_PATH}/libmct")
|
||||
SET(LIBMCT_DOWNLOAD_DIR "${LIBMCT_SOURCE_DIR}/src/${LIBMCT_PROJECT}")
|
||||
SET(LIBMCT_DST_DIR "libmct")
|
||||
SET(LIBMCT_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
|
||||
SET(LIBMCT_INSTALL_DIR ${LIBMCT_INSTALL_ROOT}/${LIBMCT_DST_DIR})
|
||||
SET(LIBMCT_ROOT ${LIBMCT_INSTALL_DIR})
|
||||
SET(LIBMCT_INC_DIR ${LIBMCT_ROOT}/include)
|
||||
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${LIBMCT_ROOT}/lib")
|
||||
|
||||
INCLUDE_DIRECTORIES(${LIBMCT_INC_DIR})
|
||||
|
||||
FILE(WRITE ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
|
||||
"PROJECT(LIBMCT)\n"
|
||||
"cmake_minimum_required(VERSION 3.0)\n"
|
||||
"install(DIRECTORY ${LIBMCT_NAME}/include ${LIBMCT_NAME}/lib \n"
|
||||
" DESTINATION ${LIBMCT_DST_DIR})\n")
|
||||
|
||||
ExternalProject_Add(
|
||||
${LIBMCT_PROJECT}
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
PREFIX ${LIBMCT_SOURCE_DIR}
|
||||
DOWNLOAD_DIR ${LIBMCT_DOWNLOAD_DIR}
|
||||
DOWNLOAD_COMMAND wget --no-check-certificate ${LIBMCT_URL} -c -q -O ${LIBMCT_NAME}.tar.gz
|
||||
&& tar zxvf ${LIBMCT_NAME}.tar.gz
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
UPDATE_COMMAND ""
|
||||
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBMCT_INSTALL_ROOT}
|
||||
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LIBMCT_INSTALL_ROOT}
|
||||
)
|
||||
|
||||
if (${CMAKE_VERSION} VERSION_LESS "3.3.0" OR NOT WIN32)
|
||||
set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/boost_dummy.c)
|
||||
file(WRITE ${dummyfile} "const char *dummy = \"${dummyfile}\";")
|
||||
add_library(libmct STATIC ${dummyfile})
|
||||
else()
|
||||
add_library(libmct INTERFACE)
|
||||
endif()
|
||||
|
||||
#ADD_LIBRARY(libmct SHARED IMPORTED GLOBAL)
|
||||
ADD_DEPENDENCIES(libmct ${LIBMCT_PROJECT})
|
||||
LIST(APPEND external_project_dependencies libmct)
|
||||
|
||||
@ -0,0 +1,77 @@
|
||||
# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
IF(NOT ${WITH_PSLIB})
|
||||
return()
|
||||
ENDIF(NOT ${WITH_PSLIB})
|
||||
|
||||
IF(WIN32 OR APPLE)
|
||||
MESSAGE(WARNING
|
||||
"Windows or Mac is not supported with PSLIB in Paddle yet."
|
||||
"Force WITH_PSLIB=OFF")
|
||||
SET(WITH_PSLIB OFF CACHE STRING "Disable PSLIB package in Windows and MacOS" FORCE)
|
||||
return()
|
||||
ENDIF()
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(PSLIB_PROJECT "extern_pslib")
|
||||
IF((NOT DEFINED PSLIB_VER) OR (NOT DEFINED PSLIB_URL))
|
||||
MESSAGE(STATUS "use pre defined download url")
|
||||
SET(PSLIB_VER "0.1.0" CACHE STRING "" FORCE)
|
||||
SET(PSLIB_NAME "pslib" CACHE STRING "" FORCE)
|
||||
SET(PSLIB_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${PSLIB_VER}/${PSLIB_NAME}.tar.gz" CACHE STRING "" FORCE)
|
||||
ENDIF()
|
||||
MESSAGE(STATUS "PSLIB_NAME: ${PSLIB_NAME}, PSLIB_URL: ${PSLIB_URL}")
|
||||
SET(PSLIB_SOURCE_DIR "${THIRD_PARTY_PATH}/pslib")
|
||||
SET(PSLIB_DOWNLOAD_DIR "${PSLIB_SOURCE_DIR}/src/${PSLIB_PROJECT}")
|
||||
SET(PSLIB_DST_DIR "pslib")
|
||||
SET(PSLIB_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
|
||||
SET(PSLIB_INSTALL_DIR ${PSLIB_INSTALL_ROOT}/${PSLIB_DST_DIR})
|
||||
SET(PSLIB_ROOT ${PSLIB_INSTALL_DIR})
|
||||
SET(PSLIB_INC_DIR ${PSLIB_ROOT}/include)
|
||||
SET(PSLIB_LIB_DIR ${PSLIB_ROOT}/lib)
|
||||
SET(PSLIB_LIB ${PSLIB_LIB_DIR}/libps.so)
|
||||
SET(PSLIB_IOMP_LIB ${PSLIB_LIB_DIR}/libiomp5.so) #todo what is this
|
||||
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${PSLIB_ROOT}/lib")
|
||||
|
||||
INCLUDE_DIRECTORIES(${PSLIB_INC_DIR})
|
||||
|
||||
FILE(WRITE ${PSLIB_DOWNLOAD_DIR}/CMakeLists.txt
|
||||
"PROJECT(PSLIB)\n"
|
||||
"cmake_minimum_required(VERSION 3.0)\n"
|
||||
"install(DIRECTORY ${PSLIB_NAME}/include ${PSLIB_NAME}/lib \n"
|
||||
" DESTINATION ${PSLIB_DST_DIR})\n")
|
||||
|
||||
ExternalProject_Add(
|
||||
${PSLIB_PROJECT}
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
PREFIX ${PSLIB_SOURCE_DIR}
|
||||
DOWNLOAD_DIR ${PSLIB_DOWNLOAD_DIR}
|
||||
DOWNLOAD_COMMAND wget --no-check-certificate ${PSLIB_URL} -c -q -O ${PSLIB_NAME}.tar.gz
|
||||
&& tar zxvf ${PSLIB_NAME}.tar.gz
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
UPDATE_COMMAND ""
|
||||
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PSLIB_INSTALL_ROOT}
|
||||
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PSLIB_INSTALL_ROOT}
|
||||
)
|
||||
|
||||
ADD_LIBRARY(pslib SHARED IMPORTED GLOBAL)
|
||||
SET_PROPERTY(TARGET pslib PROPERTY IMPORTED_LOCATION ${PSLIB_LIB})
|
||||
ADD_DEPENDENCIES(pslib ${PSLIB_PROJECT})
|
||||
LIST(APPEND external_project_dependencies pslib)
|
||||
|
||||
IF(WITH_C_API)
|
||||
INSTALL(FILES ${PSLIB_LIB} ${PSLIB_IOMP_LIB} DESTINATION lib)
|
||||
ENDIF()
|
||||
@ -0,0 +1,77 @@
|
||||
# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
IF(NOT ${WITH_PSLIB_BRPC})
|
||||
return()
|
||||
ENDIF(NOT ${WITH_PSLIB_BRPC})
|
||||
|
||||
IF(WIN32 OR APPLE)
|
||||
MESSAGE(WARNING
|
||||
"Windows or Mac is not supported with PSLIB_BRPC in Paddle yet."
|
||||
"Force WITH_PSLIB_BRPC=OFF")
|
||||
SET(WITH_PSLIB_BRPC OFF CACHE STRING "Disable PSLIB_BRPC package in Windows and MacOS" FORCE)
|
||||
return()
|
||||
ENDIF()
|
||||
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(PSLIB_BRPC_PROJECT "extern_pslib_brpc")
|
||||
IF((NOT DEFINED PSLIB_BRPC_NAME) OR (NOT DEFINED PSLIB_BRPC_URL))
|
||||
MESSAGE(STATUS "use pre defined download url")
|
||||
SET(PSLIB_BRPC_VER "0.1.0" CACHE STRING "" FORCE)
|
||||
SET(PSLIB_BRPC_NAME "pslib_brpc" CACHE STRING "" FORCE)
|
||||
SET(PSLIB_BRPC_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${PSLIB_BRPC_VER}/${PSLIB_BRPC_NAME}.tar.gz" CACHE STRING "" FORCE)
|
||||
ENDIF()
|
||||
MESSAGE(STATUS "PSLIB_BRPC_NAME: ${PSLIB_BRPC_NAME}, PSLIB_BRPC_URL: ${PSLIB_BRPC_URL}")
|
||||
SET(PSLIB_BRPC_SOURCE_DIR "${THIRD_PARTY_PATH}/pslib_brpc")
|
||||
SET(PSLIB_BRPC_DOWNLOAD_DIR "${PSLIB_BRPC_SOURCE_DIR}/src/${PSLIB_BRPC_PROJECT}")
|
||||
SET(PSLIB_BRPC_DST_DIR "pslib_brpc")
|
||||
SET(PSLIB_BRPC_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
|
||||
SET(PSLIB_BRPC_INSTALL_DIR ${PSLIB_BRPC_INSTALL_ROOT}/${PSLIB_BRPC_DST_DIR})
|
||||
SET(PSLIB_BRPC_ROOT ${PSLIB_BRPC_INSTALL_DIR})
|
||||
SET(PSLIB_BRPC_INC_DIR ${PSLIB_BRPC_ROOT}/include)
|
||||
SET(PSLIB_BRPC_LIB_DIR ${PSLIB_BRPC_ROOT}/lib)
|
||||
SET(PSLIB_BRPC_LIB ${PSLIB_BRPC_LIB_DIR}/libbrpc.a)
|
||||
SET(PSLIB_BRPC_IOMP_LIB ${PSLIB_BRPC_LIB_DIR}/libiomp5.so) #todo what is this
|
||||
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${PSLIB_BRPC_ROOT}/lib")
|
||||
|
||||
INCLUDE_DIRECTORIES(${PSLIB_BRPC_INC_DIR})
|
||||
|
||||
FILE(WRITE ${PSLIB_BRPC_DOWNLOAD_DIR}/CMakeLists.txt
|
||||
"PROJECT(PSLIB_BRPC)\n"
|
||||
"cmake_minimum_required(VERSION 3.0)\n"
|
||||
"install(DIRECTORY ${PSLIB_BRPC_NAME}/include ${PSLIB_BRPC_NAME}/lib \n"
|
||||
" DESTINATION ${PSLIB_BRPC_DST_DIR})\n")
|
||||
|
||||
ExternalProject_Add(
|
||||
${PSLIB_BRPC_PROJECT}
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
PREFIX ${PSLIB_BRPC_SOURCE_DIR}
|
||||
DOWNLOAD_DIR ${PSLIB_BRPC_DOWNLOAD_DIR}
|
||||
DOWNLOAD_COMMAND wget --no-check-certificate ${PSLIB_BRPC_URL} -c -q -O ${PSLIB_BRPC_NAME}.tar.gz
|
||||
&& tar zxvf ${PSLIB_BRPC_NAME}.tar.gz
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
UPDATE_COMMAND ""
|
||||
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PSLIB_BRPC_INSTALL_ROOT}
|
||||
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PSLIB_BRPC_INSTALL_ROOT}
|
||||
)
|
||||
|
||||
ADD_LIBRARY(pslib_brpc SHARED IMPORTED GLOBAL)
|
||||
SET_PROPERTY(TARGET pslib_brpc PROPERTY IMPORTED_LOCATION ${PSLIB_BRPC_LIB})
|
||||
ADD_DEPENDENCIES(pslib_brpc ${PSLIB_BRPC_PROJECT})
|
||||
LIST(APPEND external_project_dependencies pslib_brpc)
|
||||
|
||||
IF(WITH_C_API)
|
||||
INSTALL(FILES ${PSLIB_BRPC_LIB} ${PSLIB_BRPC_IOMP_LIB} DESTINATION lib)
|
||||
ENDIF()
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,120 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/fluid/framework/data_type.h"
|
||||
#include "paddle/fluid/framework/details/memory_reuse_types.h"
|
||||
#include "paddle/fluid/framework/ir/graph.h"
|
||||
#include "paddle/fluid/framework/ir/pass.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
namespace details {
|
||||
constexpr char kAllOpDescs[] = "all_op_descs";
|
||||
|
||||
std::vector<ir::Node*> SortOpLikeDescOrder(const ir::Graph& graph);
|
||||
// sort op in bfs order
|
||||
std::vector<ir::Node*> BFSSortGraphOps(const ir::Graph& graph);
|
||||
|
||||
class ControlFlowGraph;
|
||||
|
||||
class AnalysisVarPass : public ir::Pass {
|
||||
protected:
|
||||
std::unique_ptr<ir::Graph> ApplyImpl(
|
||||
std::unique_ptr<ir::Graph> graph) const override;
|
||||
|
||||
private:
|
||||
// fill the variable map(var_nodes) by version.
|
||||
void InitSSAGraphNodes() const;
|
||||
// update program descs
|
||||
void RenameVarInGraphDesc(const std::string& var,
|
||||
const std::string& cache_var, size_t idx) const;
|
||||
// update ir nodes
|
||||
void RenameVarInGraphNode(const std::string& var,
|
||||
const std::string& cache_var, size_t idx,
|
||||
ir::Graph* graph) const;
|
||||
|
||||
void SubGraphOptimize(OpDesc* op_desc) const;
|
||||
// valid a tensor can be reuse or not
|
||||
bool NodeCanReused(ir::Node* node) const;
|
||||
// scan subblock and collect the output/input variables.
|
||||
std::unordered_set<std::string> GetSubBlockVars(
|
||||
const std::unordered_set<ir::Node*>&) const;
|
||||
// check op has subblock or not
|
||||
bool OpHasSubBlock(OpDesc* desc) const;
|
||||
|
||||
private:
|
||||
// Reuse Node Pool, Owned.
|
||||
mutable OrderedNodePairPool pool_;
|
||||
// controlflow Graph
|
||||
mutable std::unique_ptr<ControlFlowGraph> cfg_;
|
||||
// skip set
|
||||
mutable std::unordered_set<std::string> skip_set_;
|
||||
// var nodes
|
||||
mutable std::map<std::string, std::vector<ir::Node*>> var_nodes_;
|
||||
};
|
||||
|
||||
class ControlFlowGraph {
|
||||
public:
|
||||
ControlFlowGraph() = default;
|
||||
// For IR Graph in parallelexecutor
|
||||
explicit ControlFlowGraph(const ir::Graph& graph);
|
||||
|
||||
void LiveVariableAnalysis();
|
||||
|
||||
void RenameVarInCFGGraph(const std::string& old_node,
|
||||
const std::string& new_node, int begin_idx);
|
||||
|
||||
const std::set<std::string> LiveIn(ir::Node* op) const;
|
||||
const std::set<std::string> LiveOut(ir::Node* op) const;
|
||||
const std::set<std::string> Use(ir::Node* op) const;
|
||||
const std::vector<ir::Node*> Ops() const;
|
||||
std::vector<ir::Node*>& Ops();
|
||||
|
||||
// for ssa-graph nodes
|
||||
ir::Node* GetNodeFromVarName(const std::string& name, ir::Node* op) const;
|
||||
|
||||
private:
|
||||
void BuildCFGGraph();
|
||||
void ConnectNodes();
|
||||
using NodeListMap = std::unordered_map<ir::Node*, std::set<ir::Node*>>;
|
||||
using VarSetMap = std::map<ir::Node*, std::set<std::string>>;
|
||||
// successors ops use the output variables.
|
||||
NodeListMap successors_;
|
||||
// predecessors ops generated input variables.
|
||||
NodeListMap predecessors_;
|
||||
// variables lived before run current op.
|
||||
VarSetMap live_in_;
|
||||
// variables lived after run current op.
|
||||
VarSetMap live_out_;
|
||||
VarSetMap uses_; // op inputs
|
||||
VarSetMap defs_; // op outputs
|
||||
|
||||
std::vector<ir::Node*> ops_; // op sequence by topology sort
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue