From e371ae9c4990ba08fb2aa4ac1338afa0ba29a8cb Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Mon, 15 May 2017 11:03:34 +0800 Subject: [PATCH 01/30] Remove opencv-python in python/setup.py. --- python/setup.py.in | 1 - 1 file changed, 1 deletion(-) diff --git a/python/setup.py.in b/python/setup.py.in index 7d9438e3f8..5dfb46192a 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -18,7 +18,6 @@ setup(name='paddle', "numpy", "protobuf==${PROTOBUF_VERSION}", "matplotlib", - "opencv-python", ], packages=packages, package_dir={ From 41780aef177175c201a5d81a95e33959227fa51c Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 14:00:49 -0400 Subject: [PATCH 02/30] implment pserver client lib cgo part, and temporary cmake solution --- paddle/go/pserver/client.go | 61 +++++ paddle/go/pserver/lib/CMakeLists.txt | 33 +++ paddle/go/pserver/lib/client/.gitignore | 2 + paddle/go/pserver/lib/client/CMakeLists.txt | 5 + paddle/go/pserver/lib/client/main.go | 212 ++++++++++++++++++ .../go/pserver/lib/client/test/CMakeLists.txt | 8 + paddle/go/pserver/lib/client/test/main.c | 8 + .../lib/cmake/CMakeDetermineGoCompiler.cmake | 44 ++++ .../lib/cmake/CMakeGoCompiler.cmake.in | 8 + .../lib/cmake/CMakeGoInformation.cmake | 7 + .../lib/cmake/CMakeTestGoCompiler.cmake | 1 + paddle/go/pserver/lib/cmake/flags.cmake | 45 ++++ paddle/go/pserver/lib/cmake/golang.cmake | 46 ++++ 13 files changed, 480 insertions(+) create mode 100644 paddle/go/pserver/client.go create mode 100644 paddle/go/pserver/lib/CMakeLists.txt create mode 100644 paddle/go/pserver/lib/client/.gitignore create mode 100644 paddle/go/pserver/lib/client/CMakeLists.txt create mode 100644 paddle/go/pserver/lib/client/main.go create mode 100644 paddle/go/pserver/lib/client/test/CMakeLists.txt create mode 100644 paddle/go/pserver/lib/client/test/main.c create mode 100644 paddle/go/pserver/lib/cmake/CMakeDetermineGoCompiler.cmake create mode 100644 paddle/go/pserver/lib/cmake/CMakeGoCompiler.cmake.in create mode 100644 paddle/go/pserver/lib/cmake/CMakeGoInformation.cmake create mode 100644 paddle/go/pserver/lib/cmake/CMakeTestGoCompiler.cmake create mode 100644 paddle/go/pserver/lib/cmake/flags.cmake create mode 100644 paddle/go/pserver/lib/cmake/golang.cmake diff --git a/paddle/go/pserver/client.go b/paddle/go/pserver/client.go new file mode 100644 index 0000000000..04bf58cd3e --- /dev/null +++ b/paddle/go/pserver/client.go @@ -0,0 +1,61 @@ +package pserver + +// ElementType is the type of elements of a Parameter. +type ElementType int + +// Supported element types +const ( + Int32 ElementType = iota + UInt32 + Int64 + UInt64 + Float32 + Float64 +) + +type Parameter struct { + Name string + ElementType ElementType + Content []byte +} + +type ParameterWithConfig struct { + Param Parameter + Config []byte +} + +type Gradient Parameter + +type Client struct { +} + +func NewClient(addr string) *Client { + return &Client{} +} + +func (c *Client) BeginInitParams(pserverConfigProto []byte) (bool, error) { + return true, nil +} + +func (c *Client) InitParam(paramWithConfigs ParameterWithConfig) error { + return nil +} + +func (c *Client) FinishInitParams() error { + return nil +} + +func (c *Client) SendGrads(grads []Gradient) error { + return nil +} + +func (c *Client) GetParams(names []string) ([]Parameter, error) { + return nil, nil +} + +func (c *Client) SaveModel(path string) error { + return nil +} + +func (c *Client) Cleanup() { +} diff --git a/paddle/go/pserver/lib/CMakeLists.txt b/paddle/go/pserver/lib/CMakeLists.txt new file mode 100644 index 0000000000..eb98f4b0b6 --- /dev/null +++ b/paddle/go/pserver/lib/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.0) + +if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) + message("-- Found gtest (include: ${GTEST_INCLUDE_DIR}, library: ${GTEST_LIBRARIES})") +else() + # find #include + get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) + include_directories(${PARENT_DIR}) + + # find cmake directory modules + get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) + get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) + get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) + + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") + + # enable c++11 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + + # enable gtest + set(THIRD_PARTY_PATH ./third_party) + set(WITH_TESTING ON) + include(external/gtest) +endif() + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +project(cxx_go CXX C Go) + +include(cmake/golang.cmake) +include(cmake/flags.cmake) + +add_subdirectory(client) diff --git a/paddle/go/pserver/lib/client/.gitignore b/paddle/go/pserver/lib/client/.gitignore new file mode 100644 index 0000000000..946d2d10ca --- /dev/null +++ b/paddle/go/pserver/lib/client/.gitignore @@ -0,0 +1,2 @@ +client.h +client.a diff --git a/paddle/go/pserver/lib/client/CMakeLists.txt b/paddle/go/pserver/lib/client/CMakeLists.txt new file mode 100644 index 0000000000..0f6d47f27c --- /dev/null +++ b/paddle/go/pserver/lib/client/CMakeLists.txt @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.0) + +ExternalGoProject_Add(pserver github.com/PaddlePaddle/Paddle/paddle/go/pserver) +add_go_library(client STATIC pserver) +add_subdirectory(test) diff --git a/paddle/go/pserver/lib/client/main.go b/paddle/go/pserver/lib/client/main.go new file mode 100644 index 0000000000..575f536ba2 --- /dev/null +++ b/paddle/go/pserver/lib/client/main.go @@ -0,0 +1,212 @@ +package main + +/* +#include +#include +typedef enum { + PADDLE_ELEMENT_TYPE_INT32 = 0, + PADDLE_ELEMENT_TYPE_UINT32 = 1, + PADDLE_ELEMENT_TYPE_INT64 = 2, + PADDLE_ELEMENT_TYPE_UINT64 = 3, + PADDLE_ELEMENT_TYPE_FLOAT32 = 4, + PADDLE_ELEMENT_TYPE_FLOAT64 = 5, +} paddle_element_type; + +typedef struct { + char* name; + paddle_element_type element_type; + void* content; + int content_len; +} paddle_parameter, paddle_gradient; + +typedef int client; +*/ +import "C" + +import ( + "log" + "sync" + "unsafe" + + "github.com/PaddlePaddle/Paddle/paddle/go/pserver" +) + +const ( + ptrSize = unsafe.Sizeof(uintptr(0)) +) + +var mu sync.Mutex +var handleMap = make(map[C.client]*pserver.Client) +var curHandle C.client + +func add(c *pserver.Client) C.client { + mu.Lock() + defer mu.Unlock() + client := curHandle + curHandle++ + handleMap[client] = c + return client +} + +func get(client C.client) *pserver.Client { + mu.Lock() + defer mu.Unlock() + return handleMap[client] +} + +func remove(client C.client) *pserver.Client { + mu.Lock() + defer mu.Unlock() + h := handleMap[client] + delete(handleMap, client) + return h +} + +func cArrayToSlice(p unsafe.Pointer, len int) []byte { + // create a Go clice backed by a C array, + // reference: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices + return (*[1 << 30]byte)(p)[:len:len] +} + +//export paddle_new_pserver_client +func paddle_new_pserver_client(addr *C.char) C.client { + c := pserver.NewClient(C.GoString(addr)) + return add(c) +} + +//export paddle_pserver_client_release +func paddle_pserver_client_release(client C.client) { + c := remove(client) + c.Cleanup() +} + +//export paddle_begin_init_params +func paddle_begin_init_params(client C.client, pserver_config unsafe.Pointer, config_len C.int) C.int { + c := get(client) + b := cArrayToSlice(pserver_config, int(config_len)) + selected, err := c.BeginInitParams(b) + if err != nil { + log.Println(err) + return -1 + } + + if selected { + return 1 + } + return 0 +} + +//export paddle_init_param +func paddle_init_param(client C.client, param C.paddle_parameter, param_config unsafe.Pointer, config_len C.int) C.int { + et := pserver.ElementType(param.element_type) + name := C.GoString(param.name) + content := cArrayToSlice(unsafe.Pointer(param.content), int(param.content_len)) + pc := pserver.ParameterWithConfig{ + Param: pserver.Parameter{Name: name, ElementType: et, Content: content}, + Config: cArrayToSlice(param_config, int(config_len)), + } + c := get(client) + err := c.InitParam(pc) + if err != nil { + log.Println(err) + return -1 + } + + return 0 +} + +//export paddle_finish_init_params +func paddle_finish_init_params(client C.client) C.int { + c := get(client) + err := c.FinishInitParams() + if err != nil { + log.Println(err) + return -1 + } + + return 0 +} + +//export paddle_send_grads +func paddle_send_grads(client C.client, grads *C.paddle_gradient, total C.int) C.int { + var gs []pserver.Gradient + for i := 0; i < int(total); i++ { + grad := (*C.paddle_gradient)(unsafe.Pointer((uintptr(unsafe.Pointer(grads)) + uintptr(i)*ptrSize))) + et := pserver.ElementType(grad.element_type) + name := C.GoString(grad.name) + content := cArrayToSlice(unsafe.Pointer(grad.content), int(grad.content_len)) + gs = append(gs, pserver.Gradient{Name: name, ElementType: et, Content: content}) + } + + c := get(client) + err := c.SendGrads(gs) + if err != nil { + log.Println(err) + return -1 + } + + return 0 +} + +//export paddle_get_params +func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, total C.int) C.int { + var ns []string + for i := 0; i < int(total); i++ { + name := *(**C.char)(unsafe.Pointer((uintptr(unsafe.Pointer(names)) + uintptr(i)*ptrSize))) + ns = append(ns, C.GoString(name)) + } + c := get(client) + ps, err := c.GetParams(ns) + if err != nil { + log.Println(err) + return -1 + } + + for i := 0; i < int(total); i++ { + if i >= len(ps) { + break + } + + p := ps[i] + name := C.CString(p.Name) + param := (*C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*ptrSize))) + + if unsafe.Pointer(param.name) != unsafe.Pointer(uintptr(0)) { + C.free(unsafe.Pointer(param.name)) + } + param.name = name + + memReady := false + if param.content != unsafe.Pointer(uintptr(0)) { + if int(param.content_len) == len(p.Content) { + memReady = true + } else { + C.free(param.content) + } + } + + if !memReady { + param.content = C.malloc(C.size_t(len(p.Content))) + } + C.memcpy(param.content, unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content))) + param.content_len = C.int(len(p.Content)) + param.element_type = C.paddle_element_type(p.ElementType) + } + + return 0 +} + +//export paddle_save_model +func paddle_save_model(client C.client, path *C.char) C.int { + p := C.GoString(path) + c := get(client) + err := c.SaveModel(p) + if err != nil { + log.Println(err) + return -1 + } + + return 0 +} + +func main() {} // Required but ignored diff --git a/paddle/go/pserver/lib/client/test/CMakeLists.txt b/paddle/go/pserver/lib/client/test/CMakeLists.txt new file mode 100644 index 0000000000..d3bb8ee50e --- /dev/null +++ b/paddle/go/pserver/lib/client/test/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.0) + +include_directories(/env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/pserver/lib/build/client) + +add_executable(main main.c) +add_dependencies(main client) +set (CMAKE_EXE_LINKER_FLAGS "-pthread") +target_link_libraries(main /env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/pserver/lib/build/client/libclient.a ${GTEST_LIBRARIES}) diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c new file mode 100644 index 0000000000..a36509b0d9 --- /dev/null +++ b/paddle/go/pserver/lib/client/test/main.c @@ -0,0 +1,8 @@ +#include "libclient.h" + +#include "gtest/gtest.h" + +int main() { + client c = paddle_new_pserver_client(NULL); + return 0; +} diff --git a/paddle/go/pserver/lib/cmake/CMakeDetermineGoCompiler.cmake b/paddle/go/pserver/lib/cmake/CMakeDetermineGoCompiler.cmake new file mode 100644 index 0000000000..b3f8fbe271 --- /dev/null +++ b/paddle/go/pserver/lib/cmake/CMakeDetermineGoCompiler.cmake @@ -0,0 +1,44 @@ +if(NOT CMAKE_Go_COMPILER) + if(NOT $ENV{GO_COMPILER} STREQUAL "") + get_filename_component(CMAKE_Go_COMPILER_INIT $ENV{GO_COMPILER} PROGRAM PROGRAM_ARGS CMAKE_Go_FLAGS_ENV_INIT) + + if(CMAKE_Go_FLAGS_ENV_INIT) + set(CMAKE_Go_COMPILER_ARG1 "${CMAKE_Go_FLAGS_ENV_INIT}" CACHE STRING "First argument to Go compiler") + endif() + + if(NOT EXISTS ${CMAKE_Go_COMPILER_INIT}) + message(SEND_ERROR "Could not find compiler set in environment variable GO_COMPILER:\n$ENV{GO_COMPILER}.") + endif() + + endif() + + set(Go_BIN_PATH + $ENV{GOPATH} + $ENV{GOROOT} + $ENV{GOROOT}/../bin + $ENV{GO_COMPILER} + /usr/bin + /usr/local/bin + ) + + if(CMAKE_Go_COMPILER_INIT) + set(CMAKE_Go_COMPILER ${CMAKE_Go_COMPILER_INIT} CACHE PATH "Go Compiler") + else() + find_program(CMAKE_Go_COMPILER + NAMES go + PATHS ${Go_BIN_PATH} + ) + EXEC_PROGRAM(${CMAKE_Go_COMPILER} ARGS version OUTPUT_VARIABLE GOLANG_VERSION) + STRING(REGEX MATCH "go[0-9]+.[0-9]+.[0-9]+[ /A-Za-z0-9]*" VERSION "${GOLANG_VERSION}") + message("-- The Golang compiler identification is ${VERSION}") + message("-- Check for working Golang compiler: ${CMAKE_Go_COMPILER}") + endif() + +endif() + +mark_as_advanced(CMAKE_Go_COMPILER) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CMakeGoCompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeGoCompiler.cmake @ONLY) + +set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER") diff --git a/paddle/go/pserver/lib/cmake/CMakeGoCompiler.cmake.in b/paddle/go/pserver/lib/cmake/CMakeGoCompiler.cmake.in new file mode 100644 index 0000000000..a71f08e064 --- /dev/null +++ b/paddle/go/pserver/lib/cmake/CMakeGoCompiler.cmake.in @@ -0,0 +1,8 @@ +set(CMAKE_Go_COMPILER "@CMAKE_Go_COMPILER@") +set(CMAKE_Go_COMPILER_LOADED 1) + +set(CMAKE_Go_SOURCE_FILE_EXTENSIONS go) +set(CMAKE_Go_LINKER_PREFERENCE 40) +set(CMAKE_Go_OUTPUT_EXTENSION .o) +set(CMAKE_Go_OUTPUT_EXTENSION_REPLACE 1) +set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER") diff --git a/paddle/go/pserver/lib/cmake/CMakeGoInformation.cmake b/paddle/go/pserver/lib/cmake/CMakeGoInformation.cmake new file mode 100644 index 0000000000..ba51ac93fc --- /dev/null +++ b/paddle/go/pserver/lib/cmake/CMakeGoInformation.cmake @@ -0,0 +1,7 @@ +if(NOT CMAKE_Go_COMPILE_OBJECT) + set(CMAKE_Go_COMPILE_OBJECT "go tool compile -l -N -o ") +endif() + +if(NOT CMAKE_Go_LINK_EXECUTABLE) + set(CMAKE_Go_LINK_EXECUTABLE "go tool link -o ") +endif() diff --git a/paddle/go/pserver/lib/cmake/CMakeTestGoCompiler.cmake b/paddle/go/pserver/lib/cmake/CMakeTestGoCompiler.cmake new file mode 100644 index 0000000000..b9891b015b --- /dev/null +++ b/paddle/go/pserver/lib/cmake/CMakeTestGoCompiler.cmake @@ -0,0 +1 @@ +set(CMAKE_Go_COMPILER_WORKS 1 CACHE INTERNAL "") diff --git a/paddle/go/pserver/lib/cmake/flags.cmake b/paddle/go/pserver/lib/cmake/flags.cmake new file mode 100644 index 0000000000..062d5ab660 --- /dev/null +++ b/paddle/go/pserver/lib/cmake/flags.cmake @@ -0,0 +1,45 @@ +# Setting Paddle Compile Flags +include(CheckCXXCompilerFlag) +include(CheckCCompilerFlag) +include(CheckCXXSymbolExists) +include(CheckTypeSize) + +function(CheckCompilerCXX11Flag) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 4.8) + message(FATAL_ERROR "Unsupported GCC version. GCC >= 4.8 required.") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + # cmake >= 3.0 compiler id "AppleClang" on Mac OS X, otherwise "Clang" + # Apple Clang is a different compiler than upstream Clang which havs different version numbers. + # https://gist.github.com/yamaya/2924292 + if(APPLE) # cmake < 3.0 compiler id "Clang" on Mac OS X + if(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 5.1) + message(FATAL_ERROR "Unsupported AppleClang version. AppleClang >= 5.1 required.") + endif() + else() + if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.3) + message(FATAL_ERROR "Unsupported Clang version. Clang >= 3.3 required.") + endif() + endif() + endif() +endfunction() + +CheckCompilerCXX11Flag() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + +# Common gpu architectures: Kepler, Maxwell +foreach(capability 30 35 50) + list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") +endforeach() + +if (CUDA_VERSION VERSION_GREATER "7.0" OR CUDA_VERSION VERSION_EQUAL "7.0") + list(APPEND __arch_flags " -gencode arch=compute_52,code=sm_52") +endif() + +# Modern gpu architectures: Pascal +if (CUDA_VERSION VERSION_GREATER "8.0" OR CUDA_VERSION VERSION_EQUAL "8.0") + list(APPEND __arch_flags " -gencode arch=compute_60,code=sm_60") +endif() + +set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS}) \ No newline at end of file diff --git a/paddle/go/pserver/lib/cmake/golang.cmake b/paddle/go/pserver/lib/cmake/golang.cmake new file mode 100644 index 0000000000..5d39868bfd --- /dev/null +++ b/paddle/go/pserver/lib/cmake/golang.cmake @@ -0,0 +1,46 @@ +set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go") +file(MAKE_DIRECTORY ${GOPATH}) + +function(ExternalGoProject_Add TARG) + add_custom_target(${TARG} env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get ${ARGN}) +endfunction(ExternalGoProject_Add) + +function(add_go_executable NAME) + file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go") + add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp + COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build + -o "${CMAKE_CURRENT_BINARY_DIR}/${NAME}" + ${CMAKE_GO_FLAGS} ${GO_SOURCE} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) + + add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN}) + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${NAME} DESTINATION bin) +endfunction(add_go_executable) + + +function(ADD_GO_LIBRARY NAME BUILD_TYPE) + if(BUILD_TYPE STREQUAL "STATIC") + set(BUILD_MODE -buildmode=c-archive) + set(LIB_NAME "lib${NAME}.a") + else() + set(BUILD_MODE -buildmode=c-shared) + if(APPLE) + set(LIB_NAME "lib${NAME}.dylib") + else() + set(LIB_NAME "lib${NAME}.so") + endif() + endif() + + file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go") + add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp + COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE} + -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}" + ${CMAKE_GO_FLAGS} ${GO_SOURCE} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) + + add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN}) + + if(NOT BUILD_TYPE STREQUAL "STATIC") + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} DESTINATION bin) + endif() +endfunction(ADD_GO_LIBRARY) From 6011b2e74575da9b5306da114ccb30b5fb2f89b4 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 14:02:55 -0400 Subject: [PATCH 03/30] fix cmake --- paddle/go/pserver/lib/client/test/CMakeLists.txt | 2 +- paddle/go/pserver/lib/client/test/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/go/pserver/lib/client/test/CMakeLists.txt b/paddle/go/pserver/lib/client/test/CMakeLists.txt index d3bb8ee50e..895f0f66c9 100644 --- a/paddle/go/pserver/lib/client/test/CMakeLists.txt +++ b/paddle/go/pserver/lib/client/test/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(/env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/pse add_executable(main main.c) add_dependencies(main client) set (CMAKE_EXE_LINKER_FLAGS "-pthread") -target_link_libraries(main /env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/pserver/lib/build/client/libclient.a ${GTEST_LIBRARIES}) +target_link_libraries(main /env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/pserver/lib/build/client/libclient.a) # ${GTEST_LIBRARIES}) diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c index a36509b0d9..7bf89eddf9 100644 --- a/paddle/go/pserver/lib/client/test/main.c +++ b/paddle/go/pserver/lib/client/test/main.c @@ -1,6 +1,6 @@ #include "libclient.h" -#include "gtest/gtest.h" +//#include "gtest/gtest.h" int main() { client c = paddle_new_pserver_client(NULL); From a5091ef2cec535bcd46bfcffd0c0c0afdfcada2f Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 15:42:09 -0400 Subject: [PATCH 04/30] fix bug, add functional test --- paddle/go/pserver/lib/client/main.go | 31 ++++++++------ paddle/go/pserver/lib/client/test/main.c | 51 +++++++++++++++++++++++- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/paddle/go/pserver/lib/client/main.go b/paddle/go/pserver/lib/client/main.go index 575f536ba2..9040d24611 100644 --- a/paddle/go/pserver/lib/client/main.go +++ b/paddle/go/pserver/lib/client/main.go @@ -31,10 +31,7 @@ import ( "github.com/PaddlePaddle/Paddle/paddle/go/pserver" ) -const ( - ptrSize = unsafe.Sizeof(uintptr(0)) -) - +var nullPtr = unsafe.Pointer(uintptr(0)) var mu sync.Mutex var handleMap = make(map[C.client]*pserver.Client) var curHandle C.client @@ -63,6 +60,10 @@ func remove(client C.client) *pserver.Client { } func cArrayToSlice(p unsafe.Pointer, len int) []byte { + if p == nullPtr { + return nil + } + // create a Go clice backed by a C array, // reference: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices return (*[1 << 30]byte)(p)[:len:len] @@ -131,7 +132,7 @@ func paddle_finish_init_params(client C.client) C.int { func paddle_send_grads(client C.client, grads *C.paddle_gradient, total C.int) C.int { var gs []pserver.Gradient for i := 0; i < int(total); i++ { - grad := (*C.paddle_gradient)(unsafe.Pointer((uintptr(unsafe.Pointer(grads)) + uintptr(i)*ptrSize))) + grad := (*C.paddle_gradient)(unsafe.Pointer((uintptr(unsafe.Pointer(grads)) + uintptr(i)*unsafe.Sizeof(*grads)))) et := pserver.ElementType(grad.element_type) name := C.GoString(grad.name) content := cArrayToSlice(unsafe.Pointer(grad.content), int(grad.content_len)) @@ -152,7 +153,7 @@ func paddle_send_grads(client C.client, grads *C.paddle_gradient, total C.int) C func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, total C.int) C.int { var ns []string for i := 0; i < int(total); i++ { - name := *(**C.char)(unsafe.Pointer((uintptr(unsafe.Pointer(names)) + uintptr(i)*ptrSize))) + name := *(**C.char)(unsafe.Pointer((uintptr(unsafe.Pointer(names)) + uintptr(i)*unsafe.Sizeof(*names)))) ns = append(ns, C.GoString(name)) } c := get(client) @@ -169,18 +170,24 @@ func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, p := ps[i] name := C.CString(p.Name) - param := (*C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*ptrSize))) + param := (*C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst)))) - if unsafe.Pointer(param.name) != unsafe.Pointer(uintptr(0)) { - C.free(unsafe.Pointer(param.name)) + if unsafe.Pointer(param.name) != nullPtr { + if n := C.GoString(param.name); n != p.Name { + log.Println("warning: pre-allocated parameter name not match parameter name, pre-allocated parameter name will be freed.", n, p.Name) + C.free(unsafe.Pointer(param.name)) + param.name = name + } + } else { + param.name = name } - param.name = name memReady := false - if param.content != unsafe.Pointer(uintptr(0)) { - if int(param.content_len) == len(p.Content) { + if param.content != nullPtr { + if int(param.content_len) < len(p.Content) { memReady = true } else { + log.Println("warning: pre-allocated content len is smaller than parameter content len, pre-allocated content will be freed.", param.content_len, len(p.Content)) C.free(param.content) } } diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c index 7bf89eddf9..f27e57508c 100644 --- a/paddle/go/pserver/lib/client/test/main.c +++ b/paddle/go/pserver/lib/client/test/main.c @@ -2,7 +2,56 @@ //#include "gtest/gtest.h" +void panic() { + // TODO(helin): fix: gtest using cmake is not working, using this + // hacky way for now. + *(void*)0; +} + int main() { - client c = paddle_new_pserver_client(NULL); + char addr[] = "localhost:3000"; + client c = paddle_new_pserver_client(addr); + retry: + if (paddle_begin_init_params(c, NULL, 0)) { + paddle_parameter param; + char name_a[] = "param_a"; + char name_b[] = "param_b"; + char content[] = {0x00, 0x11, 0x22}; + param.element_type = PADDLE_ELEMENT_TYPE_FLOAT32; + param.name = name_a; + param.content = content; + param.content_len = 3; + if (paddle_init_param(c, param, NULL, 0) != 0) { + goto retry; + } + param.element_type = PADDLE_ELEMENT_TYPE_INT32; + param.name = name_b; + param.content = content; + param.content_len = 3; + if (paddle_init_param(c, param, NULL, 0) != 0) { + goto retry; + } + if (paddle_finish_init_params(c) != 0) { + goto retry; + } + } else { + panic(); + } + + char content[] = {0x00, 0x11, 0x22}; + paddle_gradient params[] = {{"param_a", PADDLE_ELEMENT_TYPE_INT32, content, 3}, {"param_b", PADDLE_ELEMENT_TYPE_FLOAT32, content, 3}}; + if (!paddle_send_grads(c, params, 2)) { + panic(); + } + + char* names[]={"param_a", "param_b"}; + if (!paddle_get_params(c, names, params, 2)) { + panic(); + } + + if (!paddle_save_model(c, "/tmp/")) { + panic(); + } + return 0; } From e21d56ee3fea86fc9728b9419ee031a5a89faa8c Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 16:18:49 -0400 Subject: [PATCH 05/30] add reuse variable --- paddle/go/pserver/lib/client/main.go | 6 +++--- paddle/go/pserver/lib/client/test/main.c | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/paddle/go/pserver/lib/client/main.go b/paddle/go/pserver/lib/client/main.go index 9040d24611..ec61398ef4 100644 --- a/paddle/go/pserver/lib/client/main.go +++ b/paddle/go/pserver/lib/client/main.go @@ -150,7 +150,7 @@ func paddle_send_grads(client C.client, grads *C.paddle_gradient, total C.int) C } //export paddle_get_params -func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, total C.int) C.int { +func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, reuse C.int, total C.int) C.int { var ns []string for i := 0; i < int(total); i++ { name := *(**C.char)(unsafe.Pointer((uintptr(unsafe.Pointer(names)) + uintptr(i)*unsafe.Sizeof(*names)))) @@ -172,7 +172,7 @@ func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, name := C.CString(p.Name) param := (*C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst)))) - if unsafe.Pointer(param.name) != nullPtr { + if reuse != 0 && unsafe.Pointer(param.name) != nullPtr { if n := C.GoString(param.name); n != p.Name { log.Println("warning: pre-allocated parameter name not match parameter name, pre-allocated parameter name will be freed.", n, p.Name) C.free(unsafe.Pointer(param.name)) @@ -183,7 +183,7 @@ func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, } memReady := false - if param.content != nullPtr { + if reuse != 0 && param.content != nullPtr { if int(param.content_len) < len(p.Content) { memReady = true } else { diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c index f27e57508c..d9d9e99701 100644 --- a/paddle/go/pserver/lib/client/test/main.c +++ b/paddle/go/pserver/lib/client/test/main.c @@ -39,13 +39,15 @@ int main() { } char content[] = {0x00, 0x11, 0x22}; - paddle_gradient params[] = {{"param_a", PADDLE_ELEMENT_TYPE_INT32, content, 3}, {"param_b", PADDLE_ELEMENT_TYPE_FLOAT32, content, 3}}; - if (!paddle_send_grads(c, params, 2)) { + paddle_gradient grads[2] = {{"param_a", PADDLE_ELEMENT_TYPE_INT32, content, 3}, {"param_b", PADDLE_ELEMENT_TYPE_FLOAT32, content, 3}}; + + if (!paddle_send_grads(c, grads, 2)) { panic(); } + paddle_parameter params[2]; char* names[]={"param_a", "param_b"}; - if (!paddle_get_params(c, names, params, 2)) { + if (!paddle_get_params(c, names, params, 0, 2)) { panic(); } From d08c8ea7a15400f6a0a6318f2f8e9ff771c87ce4 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 16:30:50 -0400 Subject: [PATCH 06/30] change interface for clearity --- paddle/go/pserver/lib/client/main.go | 46 +++++++++++++----------- paddle/go/pserver/lib/client/test/main.c | 4 +-- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/paddle/go/pserver/lib/client/main.go b/paddle/go/pserver/lib/client/main.go index ec61398ef4..92aa54bd10 100644 --- a/paddle/go/pserver/lib/client/main.go +++ b/paddle/go/pserver/lib/client/main.go @@ -150,7 +150,7 @@ func paddle_send_grads(client C.client, grads *C.paddle_gradient, total C.int) C } //export paddle_get_params -func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, reuse C.int, total C.int) C.int { +func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter, total C.int) C.int { var ns []string for i := 0; i < int(total); i++ { name := *(**C.char)(unsafe.Pointer((uintptr(unsafe.Pointer(names)) + uintptr(i)*unsafe.Sizeof(*names)))) @@ -169,30 +169,36 @@ func paddle_get_params(client C.client, names **C.char, dst *C.paddle_parameter, } p := ps[i] - name := C.CString(p.Name) - param := (*C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst)))) - - if reuse != 0 && unsafe.Pointer(param.name) != nullPtr { - if n := C.GoString(param.name); n != p.Name { - log.Println("warning: pre-allocated parameter name not match parameter name, pre-allocated parameter name will be freed.", n, p.Name) - C.free(unsafe.Pointer(param.name)) - param.name = name - } + param := *(**C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst)))) + nameReady := false + contentAllocated := false + + if unsafe.Pointer(param) == nullPtr { + param = (*C.paddle_parameter)(C.calloc(1, C.size_t(unsafe.Sizeof(*param)))) } else { - param.name = name - } + if unsafe.Pointer(param.name) != nullPtr { + if n := C.GoString(param.name); n != p.Name { + log.Println("warning: pre-allocated parameter name not match parameter name, pre-allocated parameter name will be freed.", n, p.Name) + C.free(unsafe.Pointer(param.name)) + } else { + nameReady = true + } + } - memReady := false - if reuse != 0 && param.content != nullPtr { - if int(param.content_len) < len(p.Content) { - memReady = true - } else { - log.Println("warning: pre-allocated content len is smaller than parameter content len, pre-allocated content will be freed.", param.content_len, len(p.Content)) - C.free(param.content) + if param.content != nullPtr { + if int(param.content_len) == len(p.Content) { + contentAllocated = true + } else { + log.Println("warning: pre-allocated content len does not match parameter content len, pre-allocated content will be freed.", param.content_len, len(p.Content)) + C.free(param.content) + } } } - if !memReady { + if !nameReady { + param.name = C.CString(p.Name) + } + if !contentAllocated { param.content = C.malloc(C.size_t(len(p.Content))) } C.memcpy(param.content, unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content))) diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c index d9d9e99701..6fec3a7a83 100644 --- a/paddle/go/pserver/lib/client/test/main.c +++ b/paddle/go/pserver/lib/client/test/main.c @@ -45,9 +45,9 @@ int main() { panic(); } - paddle_parameter params[2]; + paddle_parameter* params[2] = {NULL, NULL}; char* names[]={"param_a", "param_b"}; - if (!paddle_get_params(c, names, params, 0, 2)) { + if (!paddle_get_params(c, names, params, 2)) { panic(); } From 1c908df681943623fdded59d52a54b376fb016ec Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 16:36:33 -0400 Subject: [PATCH 07/30] fix test memory leak --- paddle/go/pserver/lib/client/test/main.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c index 6fec3a7a83..14d4522eac 100644 --- a/paddle/go/pserver/lib/client/test/main.c +++ b/paddle/go/pserver/lib/client/test/main.c @@ -8,6 +8,20 @@ void panic() { *(void*)0; } +void releaseParam(paddle_parameter* param) { + if (param != NULL) { + if (param->name != NULL) { + free(param->name); + } + + if (param->content != NULL) { + free(param->content); + } + + free(param); + } +} + int main() { char addr[] = "localhost:3000"; client c = paddle_new_pserver_client(addr); @@ -51,6 +65,9 @@ int main() { panic(); } + releaseParam(params[0]); + releaseParam(params[1]); + if (!paddle_save_model(c, "/tmp/")) { panic(); } From 61a49e49f0e8a12ad6ccfe6c4b102a3b86732991 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 16:54:45 -0400 Subject: [PATCH 08/30] add paddle_release_param --- paddle/go/pserver/lib/client/main.go | 24 +++++++++++++++++++----- paddle/go/pserver/lib/client/test/main.c | 18 ++---------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/paddle/go/pserver/lib/client/main.go b/paddle/go/pserver/lib/client/main.go index 92aa54bd10..8aa757edcd 100644 --- a/paddle/go/pserver/lib/client/main.go +++ b/paddle/go/pserver/lib/client/main.go @@ -15,10 +15,24 @@ typedef enum { typedef struct { char* name; paddle_element_type element_type; - void* content; + char* content; int content_len; } paddle_parameter, paddle_gradient; +static inline void paddle_release_param(paddle_parameter* param) { + if (param != NULL) { + if (param->name != NULL) { + free(param->name); + } + + if (param->content != NULL) { + free(param->content); + } + + free(param); + } +} + typedef int client; */ import "C" @@ -185,12 +199,12 @@ func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter } } - if param.content != nullPtr { + if unsafe.Pointer(param.content) != nullPtr { if int(param.content_len) == len(p.Content) { contentAllocated = true } else { log.Println("warning: pre-allocated content len does not match parameter content len, pre-allocated content will be freed.", param.content_len, len(p.Content)) - C.free(param.content) + C.free(unsafe.Pointer(param.content)) } } } @@ -199,9 +213,9 @@ func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter param.name = C.CString(p.Name) } if !contentAllocated { - param.content = C.malloc(C.size_t(len(p.Content))) + param.content = (*C.char)(C.malloc(C.size_t(len(p.Content)))) } - C.memcpy(param.content, unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content))) + C.memcpy(unsafe.Pointer(param.content), unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content))) param.content_len = C.int(len(p.Content)) param.element_type = C.paddle_element_type(p.ElementType) } diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c index 14d4522eac..619661d4ac 100644 --- a/paddle/go/pserver/lib/client/test/main.c +++ b/paddle/go/pserver/lib/client/test/main.c @@ -8,20 +8,6 @@ void panic() { *(void*)0; } -void releaseParam(paddle_parameter* param) { - if (param != NULL) { - if (param->name != NULL) { - free(param->name); - } - - if (param->content != NULL) { - free(param->content); - } - - free(param); - } -} - int main() { char addr[] = "localhost:3000"; client c = paddle_new_pserver_client(addr); @@ -65,8 +51,8 @@ int main() { panic(); } - releaseParam(params[0]); - releaseParam(params[1]); + paddle_release_param(params[0]); + paddle_release_param(params[1]); if (!paddle_save_model(c, "/tmp/")) { panic(); From 4db7c546f4dcdec614ef83de6686272041ef603d Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 16:55:02 -0400 Subject: [PATCH 09/30] add documentation for client --- paddle/go/pserver/client.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/paddle/go/pserver/client.go b/paddle/go/pserver/client.go index 04bf58cd3e..5b110af648 100644 --- a/paddle/go/pserver/client.go +++ b/paddle/go/pserver/client.go @@ -13,49 +13,71 @@ const ( Float64 ) +// Parameter is a piece of data to sync with the parameter server. type Parameter struct { Name string ElementType ElementType Content []byte } +// ParameterWithConfig contains the parameter and the configuration. type ParameterWithConfig struct { Param Parameter - Config []byte + Config []byte // parameter configuration in Proto Buffer format } +// Gradient is the gradient of the parameter. type Gradient Parameter +// Client is the client to parameter servers. type Client struct { } +// NewClient creates a new client. func NewClient(addr string) *Client { return &Client{} } -func (c *Client) BeginInitParams(pserverConfigProto []byte) (bool, error) { +// BeginInitParams begins to initialize parameters on parameter +// servers. +// +// BeginInitParams will be called from multiple trainers, only one +// trainer will be selected to initialize the parameters on parameter +// servers. Other trainers will be blocked until the initialization is +// done, and they need to get the initialized parameters from +// parameter servers using GetParams. +func (c *Client) BeginInitParams(pserverConfigProto []byte) (selected bool, err error) { return true, nil } +// InitParam initializes the parameter on parameter servers. func (c *Client) InitParam(paramWithConfigs ParameterWithConfig) error { return nil } +// FinishInitParams tells parameter servers client has sent all +// parameters to parameter servers as initialization. func (c *Client) FinishInitParams() error { return nil } +// SendGrads sends gradients to parameter servers for updating +// parameters. func (c *Client) SendGrads(grads []Gradient) error { return nil } +// GetParams gets parameters from parameter servers. func (c *Client) GetParams(names []string) ([]Parameter, error) { return nil, nil } +// SaveModel indicates parameters to save the parameter to the given +// path. func (c *Client) SaveModel(path string) error { return nil } +// Cleanup cleans up the client states. func (c *Client) Cleanup() { } From c17cef98099a11a6a07da77682428129aaf5ef5b Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 17:07:42 -0400 Subject: [PATCH 10/30] better warning message --- paddle/go/pserver/lib/client/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/go/pserver/lib/client/main.go b/paddle/go/pserver/lib/client/main.go index 8aa757edcd..2bb76dcb7d 100644 --- a/paddle/go/pserver/lib/client/main.go +++ b/paddle/go/pserver/lib/client/main.go @@ -192,7 +192,7 @@ func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter } else { if unsafe.Pointer(param.name) != nullPtr { if n := C.GoString(param.name); n != p.Name { - log.Println("warning: pre-allocated parameter name not match parameter name, pre-allocated parameter name will be freed.", n, p.Name) + log.Println("Warning: the pre-allocated parameter name does not match the parameter name, it will be freed.", n, p.Name) C.free(unsafe.Pointer(param.name)) } else { nameReady = true @@ -203,7 +203,7 @@ func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter if int(param.content_len) == len(p.Content) { contentAllocated = true } else { - log.Println("warning: pre-allocated content len does not match parameter content len, pre-allocated content will be freed.", param.content_len, len(p.Content)) + log.Println("Warning: the pre-allocated content len does not match parameter content len, the pre-allocated content will be freed.", param.content_len, len(p.Content)) C.free(unsafe.Pointer(param.content)) } } From 4759b373aa15881f52b78298ebcc62a3d9f67915 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 16 May 2017 17:23:45 -0400 Subject: [PATCH 11/30] add reuse buffer in functional test --- paddle/go/pserver/lib/client/test/main.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/pserver/lib/client/test/main.c index 619661d4ac..d71a023a6a 100644 --- a/paddle/go/pserver/lib/client/test/main.c +++ b/paddle/go/pserver/lib/client/test/main.c @@ -51,6 +51,11 @@ int main() { panic(); } + // get parameters again by reusing the allocated parameter buffers. + if (!paddle_get_params(c, names, params, 2)) { + panic(); + } + paddle_release_param(params[0]); paddle_release_param(params[1]); From 418917e813fd5d617e5e8cffca683551e6b5b9e0 Mon Sep 17 00:00:00 2001 From: qijun Date: Wed, 17 May 2017 13:32:01 +0800 Subject: [PATCH 12/30] add details about implementing tensor in paddle --- paddle/majel/README.md | 82 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/paddle/majel/README.md b/paddle/majel/README.md index 5539853056..ed9fa7f02a 100644 --- a/paddle/majel/README.md +++ b/paddle/majel/README.md @@ -93,34 +93,86 @@ typedef boost::variant< Because `variant` may be thought of as "multi-type, single value", we can utilize it to implement unified interfaces for PaddlePaddle. -## implement Tensor in Paddle +## Implement Tensor in Paddle + +We want to create a Tensor class to replace Vector and Matrix, and to support high-dimensional data. The operations on Tensor are implemented in both CPU and GPU. We also want to make sure that the Tensor interface is friendly to its callers. + +Tensor is only responsible for describing computing. It will not take charge of memory allocation policy, handles of some CUDA library context(e.g. cublasHandle, cudnnHandle), and dispatching CUDA kernels. Paddle has realize the initialization and resources management of hardware. Before writing code, please make sure you already look through Majel Source Code and grabbed the design philosophy of `DArray` in Majel. -To assign subtasks to our colleagues, we have to discuss how to divide it to independent subtasks. -- [ ] 1. First, we need to consider the third-party dependencies in Majel. +### Memory Management +`Allocation` manages a block of memory in device(CPU/GPU). We use `Place` to decribe memory location. The details of memory allocation and deallocation are implememted in `Allocator` and `DeAllocator`. Related low-level API such as `hl_malloc_device()` and `hl_malloc_host()` are provided by Paddle. + +### Dim and Array +#### Dim + +`Dim` decribes the dimension information of an array. + +`DDimVar` is an alias of a specializd class of boost.variant class template. + +`DDim` is introduced to represent a dynamically sized dimension. + +For example: + +``` +Dim<2> d1 = make_dim(3, 3); +DDim d2 = make_ddim({1, 2, 3}); +``` + +You must appoint a concrete sized dimension to Dim, whereas DDim can represent a dynamically sized dimension. +#### Array + +`Array` represents for a tensor with specific type and size. - Majel heavily use `boost.variant`, but we don't want to integrate `boost` into PaddlePaddle. It's better to replace boost using the lightweight implementation. https://github.com/mapbox/variant Mapbox variant has the same speedy performance of `boost::variant `but is faster to compile, results in smaller binaries, and has no dependencies. +`DArrarVar` is an alias of a specialized class of boost.variant class template. -> @gangliao +`DArray` is introduced to represent a dynamically typed array. + +For example: + +``` +Array a1(Dim<2>(2, 2)); +DArray a2 = make_darray(make_ddim({3, 4}), 0.0, CpuPlace()); +``` -- [ ] 2. Re-implement `Place` and `Allocation/Memory` +You must appoint the type and dimension of a Array, whereas DArray can represent a dynanmically typed array. - I found @wangkuiyi submitted a pull request includes `Place`. @gangliao and @qijun could re-implement `Allocation`, because we have the GPU development experience before joining Paddle team. -> @wangkuiyi @gangliao @qijun +Please reference the section of `Learn from Majel` for more details. -- [ ] 3. Re-implement `Dim`. - `Dim` is an excellent implementation in Majel. -> ??? +### ArrayView -- [ ] 4. Re-implement `Array/Tensor`. +`ViewIterator` is a class template which implements basic iterator operation, including increment(++), decrement(--), dereference(*), equalit comparisons(==) and so on. + +`ArrayView` is an encapsulation of `Array`， which introduces extra iterator menthods, such as `begin()` and `end()`. The `begin()` method returns an iterator pointing to the first element in the ArrayView. And the `end()` method returns an iterator pointing to the pass-the-end element in the ArrayView. + +`ArrayView` make the visting and manipulating an array more efficently, flexibly and safely. + + +A global function `make_view` is provided to transform an array to corresponding arrayview. + +``` +template +ArrayView make_view(const Array& in) { + return in; +} +``` + +A global function `make_iterator` is provided to make iterator of an array. + +``` +template +ViewIterator> make_iterator(const Array& in, Dim idx) { + return make_iterator(make_view(in), idx); +} +``` -> Prerequisites: 1 - 3 +### Basic Operations -- [ ] 5. Re-implement fundamental operators for `Array/Tensor`. +The operations that manipulate DArray are defined as global functions, such as `ones`, `zeros`, `reshape`, `gemm` and so on. -> Prerequisites: 1 - 4 +An array will be trasformed into an arrayview and then passed to the operation launching on a specific deviec(CPU/GPU). From 7ee280eb7f6b125ba9587f6259b14eac7fc714cc Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Wed, 17 May 2017 15:00:53 -0400 Subject: [PATCH 13/30] change folder structure --- paddle/go/{pserver/lib/client => cclient}/.gitignore | 0 paddle/go/{pserver/lib => cclient}/CMakeLists.txt | 5 +++-- paddle/go/{pserver/lib/client/main.go => cclient/cclient.go} | 0 .../lib => cclient}/cmake/CMakeDetermineGoCompiler.cmake | 0 .../{pserver/lib => cclient}/cmake/CMakeGoCompiler.cmake.in | 0 .../{pserver/lib => cclient}/cmake/CMakeGoInformation.cmake | 0 .../{pserver/lib => cclient}/cmake/CMakeTestGoCompiler.cmake | 0 paddle/go/{pserver/lib => cclient}/cmake/flags.cmake | 0 paddle/go/{pserver/lib => cclient}/cmake/golang.cmake | 0 .../go/{pserver/lib/client => cclient}/test/CMakeLists.txt | 4 ++-- paddle/go/{pserver/lib/client => cclient}/test/main.c | 0 paddle/go/pserver/lib/client/CMakeLists.txt | 5 ----- 12 files changed, 5 insertions(+), 9 deletions(-) rename paddle/go/{pserver/lib/client => cclient}/.gitignore (100%) rename paddle/go/{pserver/lib => cclient}/CMakeLists.txt (86%) rename paddle/go/{pserver/lib/client/main.go => cclient/cclient.go} (100%) rename paddle/go/{pserver/lib => cclient}/cmake/CMakeDetermineGoCompiler.cmake (100%) rename paddle/go/{pserver/lib => cclient}/cmake/CMakeGoCompiler.cmake.in (100%) rename paddle/go/{pserver/lib => cclient}/cmake/CMakeGoInformation.cmake (100%) rename paddle/go/{pserver/lib => cclient}/cmake/CMakeTestGoCompiler.cmake (100%) rename paddle/go/{pserver/lib => cclient}/cmake/flags.cmake (100%) rename paddle/go/{pserver/lib => cclient}/cmake/golang.cmake (100%) rename paddle/go/{pserver/lib/client => cclient}/test/CMakeLists.txt (68%) rename paddle/go/{pserver/lib/client => cclient}/test/main.c (100%) delete mode 100644 paddle/go/pserver/lib/client/CMakeLists.txt diff --git a/paddle/go/pserver/lib/client/.gitignore b/paddle/go/cclient/.gitignore similarity index 100% rename from paddle/go/pserver/lib/client/.gitignore rename to paddle/go/cclient/.gitignore diff --git a/paddle/go/pserver/lib/CMakeLists.txt b/paddle/go/cclient/CMakeLists.txt similarity index 86% rename from paddle/go/pserver/lib/CMakeLists.txt rename to paddle/go/cclient/CMakeLists.txt index eb98f4b0b6..29a2089fb1 100644 --- a/paddle/go/pserver/lib/CMakeLists.txt +++ b/paddle/go/cclient/CMakeLists.txt @@ -10,7 +10,6 @@ else() # find cmake directory modules get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) - get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") @@ -30,4 +29,6 @@ project(cxx_go CXX C Go) include(cmake/golang.cmake) include(cmake/flags.cmake) -add_subdirectory(client) +ExternalGoProject_Add(pserver github.com/PaddlePaddle/Paddle/paddle/go/pserver) +add_go_library(client STATIC pserver) +add_subdirectory(test) diff --git a/paddle/go/pserver/lib/client/main.go b/paddle/go/cclient/cclient.go similarity index 100% rename from paddle/go/pserver/lib/client/main.go rename to paddle/go/cclient/cclient.go diff --git a/paddle/go/pserver/lib/cmake/CMakeDetermineGoCompiler.cmake b/paddle/go/cclient/cmake/CMakeDetermineGoCompiler.cmake similarity index 100% rename from paddle/go/pserver/lib/cmake/CMakeDetermineGoCompiler.cmake rename to paddle/go/cclient/cmake/CMakeDetermineGoCompiler.cmake diff --git a/paddle/go/pserver/lib/cmake/CMakeGoCompiler.cmake.in b/paddle/go/cclient/cmake/CMakeGoCompiler.cmake.in similarity index 100% rename from paddle/go/pserver/lib/cmake/CMakeGoCompiler.cmake.in rename to paddle/go/cclient/cmake/CMakeGoCompiler.cmake.in diff --git a/paddle/go/pserver/lib/cmake/CMakeGoInformation.cmake b/paddle/go/cclient/cmake/CMakeGoInformation.cmake similarity index 100% rename from paddle/go/pserver/lib/cmake/CMakeGoInformation.cmake rename to paddle/go/cclient/cmake/CMakeGoInformation.cmake diff --git a/paddle/go/pserver/lib/cmake/CMakeTestGoCompiler.cmake b/paddle/go/cclient/cmake/CMakeTestGoCompiler.cmake similarity index 100% rename from paddle/go/pserver/lib/cmake/CMakeTestGoCompiler.cmake rename to paddle/go/cclient/cmake/CMakeTestGoCompiler.cmake diff --git a/paddle/go/pserver/lib/cmake/flags.cmake b/paddle/go/cclient/cmake/flags.cmake similarity index 100% rename from paddle/go/pserver/lib/cmake/flags.cmake rename to paddle/go/cclient/cmake/flags.cmake diff --git a/paddle/go/pserver/lib/cmake/golang.cmake b/paddle/go/cclient/cmake/golang.cmake similarity index 100% rename from paddle/go/pserver/lib/cmake/golang.cmake rename to paddle/go/cclient/cmake/golang.cmake diff --git a/paddle/go/pserver/lib/client/test/CMakeLists.txt b/paddle/go/cclient/test/CMakeLists.txt similarity index 68% rename from paddle/go/pserver/lib/client/test/CMakeLists.txt rename to paddle/go/cclient/test/CMakeLists.txt index 895f0f66c9..c899bd275d 100644 --- a/paddle/go/pserver/lib/client/test/CMakeLists.txt +++ b/paddle/go/cclient/test/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.0) -include_directories(/env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/pserver/lib/build/client) +include_directories(/env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/cclient/build/) add_executable(main main.c) add_dependencies(main client) set (CMAKE_EXE_LINKER_FLAGS "-pthread") -target_link_libraries(main /env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/pserver/lib/build/client/libclient.a) # ${GTEST_LIBRARIES}) +target_link_libraries(main /env/gopath/src/github.com/PaddlePaddle/Paddle/paddle/go/cclient/build/libclient.a) # ${GTEST_LIBRARIES}) diff --git a/paddle/go/pserver/lib/client/test/main.c b/paddle/go/cclient/test/main.c similarity index 100% rename from paddle/go/pserver/lib/client/test/main.c rename to paddle/go/cclient/test/main.c diff --git a/paddle/go/pserver/lib/client/CMakeLists.txt b/paddle/go/pserver/lib/client/CMakeLists.txt deleted file mode 100644 index 0f6d47f27c..0000000000 --- a/paddle/go/pserver/lib/client/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake_minimum_required(VERSION 3.0) - -ExternalGoProject_Add(pserver github.com/PaddlePaddle/Paddle/paddle/go/pserver) -add_go_library(client STATIC pserver) -add_subdirectory(test) From f71453be9e74471724c912c2786502f4515de61e Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Wed, 17 May 2017 15:04:34 -0400 Subject: [PATCH 14/30] remove unnecessary .gitignore --- paddle/go/cclient/.gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 paddle/go/cclient/.gitignore diff --git a/paddle/go/cclient/.gitignore b/paddle/go/cclient/.gitignore deleted file mode 100644 index 946d2d10ca..0000000000 --- a/paddle/go/cclient/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -client.h -client.a From 877447361e3edccb90306ff276342899f115f9d1 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Wed, 17 May 2017 15:45:09 -0400 Subject: [PATCH 15/30] use unsigned char* for parameter.content --- paddle/go/cclient/cclient.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/go/cclient/cclient.go b/paddle/go/cclient/cclient.go index 2bb76dcb7d..dc86d47e8d 100644 --- a/paddle/go/cclient/cclient.go +++ b/paddle/go/cclient/cclient.go @@ -15,7 +15,7 @@ typedef enum { typedef struct { char* name; paddle_element_type element_type; - char* content; + unsigned char* content; int content_len; } paddle_parameter, paddle_gradient; @@ -213,7 +213,7 @@ func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter param.name = C.CString(p.Name) } if !contentAllocated { - param.content = (*C.char)(C.malloc(C.size_t(len(p.Content)))) + param.content = (*C.uchar)(C.malloc(C.size_t(len(p.Content)))) } C.memcpy(unsafe.Pointer(param.content), unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content))) param.content_len = C.int(len(p.Content)) From e6063f6c333d0473f5e460e045c2b8f137dfd0f6 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 17 May 2017 16:02:45 -0700 Subject: [PATCH 16/30] Design Doc: New Building System --- doc/design/build_system/README.md | 110 ++++++++++++++++++++++++++++++ doc/design/build_system/go.md | 98 ++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 doc/design/build_system/README.md create mode 100644 doc/design/build_system/go.md diff --git a/doc/design/build_system/README.md b/doc/design/build_system/README.md new file mode 100644 index 0000000000..4c4315d6b9 --- /dev/null +++ b/doc/design/build_system/README.md @@ -0,0 +1,110 @@ +A few months ago when we were trying to replace CMake with Bazel, @emailweixu suggested that we rewrite those handy Bazel functions using CMake. Now it seems that it's the right time to get this done, as we are facing problems from the porting of Majel and the development of new the parameter server using Go and C++. + +Here are some initial thoughts. Your comments are welcome! + +### Required CMake Function + +I think we need only the following few CMake functions to make a project description mean and clean: + +| C++ | CUDA C++ | Go | +|---|---|---| +| cc_library | nv_library | go_library | +| cc_binary | nv_binary | go_binary | +| cc_test | nv_test | go_test | + +- The `_library` functions generate .a files from source code. +- The `_binary` functions generate executable binary files. +- The `_test` functions generate executable unit test files. They work like `_binary` but links `-lgtest` and `-lgtest_main`. + +The difference between `nv_` functions and `cc_` functions is that the former use `nvcc` instead of the system-default C++ compiler. + +Both `nv_` and `cc_` functions enables C++11 (-std=c++11). + +In addition, + +- to describe external dependencies, we need `external_library`. +- to build shared libraries, we need `shared_library`. + +### An Example Project + +Suppose that we have aforementioned functions defined in our `/cmake` directory. The following example `CMakeLists.txt` describes a project including the following source files: + +- tensor.h +- tensor.cc +- tensor_test.cc +- ops.h +- ops.cu +- ops_test.cu +- api.go +- api_test.go + +Suppose that ops.cu depends on CUDNN. + +```cmake +# cc_binary parses tensor.cc and figures out that target also depend +# on tensor.h. +cc_binary(tensor + SRCS + tensor.cc) + +# The dependency to target tensor implies that if any of +# tensor{.h,.cc,_test.cc} is changed, tensor_test need to be re-built. +cc_test(tensor_test + SRCS + tensor_test.cc + DEPS + tensor) + +# I don't have a clear idea what parameters external_library need to +# have. @gangliao as a CMake expert would have better ideas. +external_library(cudnn + ....) + +# Suppose that ops.cu depends on external target CUDNN. Also, ops.cu +# include global functions that take Tensor as their parameters, so +# ops depend on tensor. This implies that if any of tensor.{h.cc}, +# ops.{h,cu} is changed, ops need to be re-built. +nv_library(ops + SRCS + ops.cu + DEPS + tensor + cudnn) # cudnn is defined later. + +nv_test(ops_test + SRCS + ops_test.cu + DEPS + ops) + +# Because api.go defines a GO wrapper to ops and tensor, it depends on +# both. This implies that if any of tensor.{h,cc}, ops.{h,cu}, or +# api.go is changed, api need to be re-built. +go_library(api + SRCS + api.go + DEPS + tensor # Because ops depend on tensor, this line is optional. + ops) + +go_test(api_test + SRCS + api_test.go + DEPS + api) + + +# This builds libapi.so. shared_library might use CMake target +# api_shared so to distinguish it from above target api. +shared_library(api + DEPS + api) + +``` + +### Implementation + +As above example CMakeLists.txt executes, each function invocation +adds "nodes" to a dependency graph. It also use this graph to +generate CMake commands including `add_executable`, +`add_dependencies`, `target_link_libraries`, and `add_test`. diff --git a/doc/design/build_system/go.md b/doc/design/build_system/go.md new file mode 100644 index 0000000000..90399ff701 --- /dev/null +++ b/doc/design/build_system/go.md @@ -0,0 +1,98 @@ +# Design Doc: `go_{library,binary,test}` + +## Concerns + +1. Need to build Go libraries callable from Go and from C. + + For usual Go libraries, the bulding command line is as + + ```bash + go build foo.go bar.go -o libfoobar.a + ``` + + For Go libraries callable from C/C++, the command line is + + ```bash + go build -buildmode=c-archive foo.go bar.go -o libstatic.a + ``` + + or for shared libraries: + + ```bash + go build -buildmode=c-shared foo.go bar.go -o libdynamic.so + ``` + + and `foo.go`, `bar.go`, etc must start with a line `package main`, + which defines all symbols in special pacakge `main`. There also + must be a `func main`, which could have empty body. + +1. Need to support building-in-Docker. + + We are going to support two ways to building -- (1) in Ubuntu, and + (2) in Docker container whose base image is Ubuntu. + + The challenge is (2), because to build in Docker, we run the + development image as: + + ```bash + git clone https://github.com/PaddlePaddle/Paddle -o paddle + cd paddle + docker run -v $PWD:/paddle paddle:dev + ``` + + which maps the local repo to `/paddle` in the container. + + This assumes that all Go code, including third-party packages, must + be in the local repo. Actually, it assumes that `$GOPATH` must be + in the local repo. This would affect how we write `import` + statemetns, and how we maintain third-party packages. + + +## A Solution + +This might not be the optimal solution. Comments are welcome. + +### Directory structure + +We propose the following directory structure: + +``` +https://github.com/PaddlePaddle/Paddle + ↓ git clone + ~/work/paddle/go/pkg1/foo.go + /pkg2/bar.go + /cmd/cmd1/wee.go + /cmd/cmd2/qux.go + /github.com/someone/a_3rd_party_pkg (Git submodule to a 3rd-party pkg) + /golang.org/another_3rd_party_pkg (Git submodule to another one) + /build/go ($GOPATH, required by Go toolchain) + /src (symlink to ~/work/paddle/go/) + /pkg (libraries built by Go toolchain) + /bin (binaries bulit by Go toolchain) +``` + +Above figure explains how we organize Paddle's Go code: + +1. Go source code in Paddle is in `/go` of the repo. +1. Each library package is a sub-directory under `/go`. +1. Each (executable) binary package is a sub-directory under + `/go/cmd`. This is the source tree convention of Go itself. +1. Each 3rd-party Go package is a Git submodule under `/go`. + +These rules make sure that all Go source code are in `/go`. + +At build-time, Go toolchain requires a directory structure rooted at +`$GOPATH` and having three sub-directories: `$GOPATH/src`, +`$GOPATH/pkg`, and `$GOPATH/bin`, where `$GOPATH/src` is the source +tree and the root of Go's `import` paths. + +For example, if `/go/pkg1/foo.go` contains `import +"github.com/someone/a_3rd_party_pkg"`, the Go toolchain will find the +package at `$GOPATH/src/github.com/someone/a_3rd_party_pkg`. + +In order to create such a `$GOPATH`, our build system creates +`/build/go`. Remeber that we want to make sure that all output files +generated at build-time are place in `/build`. + +Under `/build/go`, our build system creates a symoblic link `src` +pointing to `/go`, where all Go source code resides. From 75674710b5e795c009ab570708602e26091c9d55 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Thu, 18 May 2017 11:52:53 +0800 Subject: [PATCH 17/30] Remove opencv-python in setup --- python/paddle/v2/image.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py index 13f53919aa..85ad6984ba 100644 --- a/python/paddle/v2/image.py +++ b/python/paddle/v2/image.py @@ -1,10 +1,10 @@ import numpy as np try: import cv2 -except ImportError: - cv2 = None - -from cv2 import resize +except: + print( + "import cv2 error, please install opencv-python: pip install opencv-python" + ) __all__ = [ "load_image", "resize_short", "to_chw", "center_crop", "random_crop", @@ -76,7 +76,7 @@ def resize_short(im, size): h_new = size * h / w else: w_new = size * w / h - im = resize(im, (h_new, w_new), interpolation=cv2.INTER_CUBIC) + im = cv2.resize(im, (h_new, w_new), interpolation=cv2.INTER_CUBIC) return im From b67291aedb30f0d00ed86de3bc9d15b89d14a7b6 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 18 May 2017 14:42:36 +0800 Subject: [PATCH 18/30] fix typo error --- paddle/majel/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/majel/README.md b/paddle/majel/README.md index 670c1792c9..7a80816d8e 100644 --- a/paddle/majel/README.md +++ b/paddle/majel/README.md @@ -157,11 +157,11 @@ Please reference the section of `Learn from Majel` for more details. ### ArrayView -`ViewIterator` is a class template which implements basic iterator operation, including increment(++), decrement(--), dereference(*), equalit comparisons(==) and so on. +`ViewIterator` is a class template which implements basic iterator operation, including increment(++), decrement(--), dereference(*), equality comparisons(==) and so on. -`ArrayView` is an encapsulation of `Array`， which introduces extra iterator menthods, such as `begin()` and `end()`. The `begin()` method returns an iterator pointing to the first element in the ArrayView. And the `end()` method returns an iterator pointing to the pass-the-end element in the ArrayView. +`ArrayView` is an encapsulation of `Array`， which introduces extra iterator methods, such as `begin()` and `end()`. The `begin()` method returns an iterator pointing to the first element in the ArrayView. And the `end()` method returns an iterator pointing to the pass-the-end element in the ArrayView. -`ArrayView` make the visting and manipulating an array more efficently, flexibly and safely. +`ArrayView` make the visting and manipulating an array more efficiently, flexibly and safely. A global function `make_view` is provided to transform an array to corresponding arrayview. @@ -186,4 +186,4 @@ ViewIterator> make_iterator(const Array& in, Dim idx) { The operations that manipulate DArray are defined as global functions, such as `ones`, `zeros`, `reshape`, `gemm` and so on. -An array will be trasformed into an arrayview and then passed to the operation launching on a specific deviec(CPU/GPU). +An array will be trasformed into an arrayview and then passed to the operation launching on a specific device(CPU/GPU). From 646334b558d617b393bb6583162fb27baa7371cc Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 18 May 2017 15:34:18 +0800 Subject: [PATCH 19/30] import DDim of Majel into Paddle --- paddle/majel/CMakeLists.txt | 2 +- paddle/majel/ddim.cc | 222 +++++++++++++++ paddle/majel/ddim.h | 109 ++++++++ paddle/majel/dim.h | 456 +++++++++++++++++++++++++++++++ paddle/majel/hostdevice.h | 9 + paddle/majel/test/CMakeLists.txt | 5 + paddle/majel/test/ddim_test.cc | 65 +++++ paddle/majel/test/dim_test.cu | 128 +++++++++ 8 files changed, 995 insertions(+), 1 deletion(-) create mode 100644 paddle/majel/ddim.cc create mode 100644 paddle/majel/ddim.h create mode 100644 paddle/majel/dim.h create mode 100644 paddle/majel/hostdevice.h create mode 100644 paddle/majel/test/ddim_test.cc create mode 100644 paddle/majel/test/dim_test.cu diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt index d4bce38906..47f33dd832 100644 --- a/paddle/majel/CMakeLists.txt +++ b/paddle/majel/CMakeLists.txt @@ -1,4 +1,4 @@ -cc_library(majel SRCS place.cc) +cc_library(majel SRCS place.cc ddim.cc) if(WITH_TESTING) add_subdirectory(test) diff --git a/paddle/majel/ddim.cc b/paddle/majel/ddim.cc new file mode 100644 index 0000000000..e92d9afa0b --- /dev/null +++ b/paddle/majel/ddim.cc @@ -0,0 +1,222 @@ +#include + +namespace majel { + +///@cond HIDDEN + +template +Dim make_dim(const int* d) { + return Dim(*d, make_dim(d + 1)); +} + +template <> +Dim<1> make_dim<1>(const int* d) { + return Dim<1>(*d); +} + +void make_ddim(DDim& ddim, const int* dims, int n) { + switch (n) { + case 1: + ddim = make_dim<1>(dims); + break; + case 2: + ddim = make_dim<2>(dims); + break; + case 3: + ddim = make_dim<3>(dims); + break; + case 4: + ddim = make_dim<4>(dims); + break; + case 5: + ddim = make_dim<5>(dims); + break; + case 6: + ddim = make_dim<6>(dims); + break; + case 7: + ddim = make_dim<7>(dims); + break; + case 8: + ddim = make_dim<8>(dims); + break; + case 9: + ddim = make_dim<9>(dims); + break; + default: + throw std::invalid_argument( + "Dynamic dimensions must have between [1, 9] dimensions."); + } +} + +///@endcond + +DDim make_ddim(std::initializer_list dims) { + DDim result(make_dim(0)); + make_ddim(result, dims.begin(), dims.size()); + return result; +} + +DDim make_ddim(const std::vector& dims) { + DDim result(make_dim(0)); + make_ddim(result, &dims[0], dims.size()); + return result; +} + +///@cond HIDDEN +// XXX For some reason, putting this in an anonymous namespace causes errors +class DynamicMutableIndexer : public boost::static_visitor { +public: + DynamicMutableIndexer(int idx) : idx_(idx) {} + + template + int& operator()(Dim& dim) const { + return dim[idx_]; + } + +private: + int idx_; +}; + +class DynamicConstIndexer : public boost::static_visitor { +public: + DynamicConstIndexer(int idx) : idx_(idx) {} + + template + int operator()(const Dim& dim) const { + return dim[idx_]; + } + +private: + int idx_; +}; + +///@endcond + +int& DDim::operator[](int idx) { + return boost::apply_visitor(DynamicMutableIndexer(idx), var); +} + +int DDim::operator[](int idx) const { + return boost::apply_visitor(DynamicConstIndexer(idx), var); +} + +bool DDim::operator==(DDim d) const { + if (var.which() != d.getVar().which()) { + return false; + } else { + std::vector v1 = vectorize(*this); + std::vector v2 = vectorize(d); + + for (unsigned int i = 0; i < v1.size(); i++) { + if (v1[i] != v2[i]) { + return false; + } + } + + return true; + } +} + +bool DDim::operator!=(DDim d) const { return !(*this == d); } + +DDim DDim::operator+(DDim d) const { + std::vector v1 = vectorize(*this); + std::vector v2 = vectorize(d); + + std::vector v3; + + assert(v1.size() == v2.size()); + + for (unsigned int i = 0; i < v1.size(); i++) { + v3.push_back(v1[i] + v2[i]); + } + + return make_ddim(v3); +} + +DDim DDim::operator*(DDim d) const { + std::vector v1 = vectorize(*this); + std::vector v2 = vectorize(d); + + std::vector v3; + + assert(v1.size() == v2.size()); + + for (unsigned int i = 0; i < v1.size(); i++) { + v3.push_back(v1[i] * v2[i]); + } + + return make_ddim(v3); +} + +int get(const DDim& ddim, int idx) { return ddim[idx]; } + +void set(DDim& ddim, int idx, int value) { ddim[idx] = value; } + +///@cond HIDDEN +struct VectorizeVisitor : public boost::static_visitor<> { + std::vector& vector; + + VectorizeVisitor(std::vector& v) : vector(v) {} + + template + void operator()(const T& t) { + vector.push_back(t.head); + this->operator()(t.tail); + } + + void operator()(const Dim<1>& t) { vector.push_back(t.head); } +}; +///@endcond + +std::vector vectorize(const DDim& ddim) { + std::vector result; + VectorizeVisitor visitor(result); + boost::apply_visitor(visitor, ddim); + return result; +} + +ssize_t product(const DDim& ddim) { + ssize_t result = 1; + std::vector v = vectorize(ddim); + for (auto i : v) { + result *= i; + } + return result; +} + +///\cond HIDDEN + +struct ArityVisitor : boost::static_visitor { + template + int operator()(Dim) const { + return D; + } +}; + +///\endcond + +int arity(const DDim& d) { return boost::apply_visitor(ArityVisitor(), d); } + +///\cond HIDDEN + +struct DDimPrinter : boost::static_visitor { + std::ostream& os; + DDimPrinter(std::ostream& os_) : os(os_) {} + + template + void operator()(const T& t) { + os << t; + } +}; + +///\endcond + +std::ostream& operator<<(std::ostream& os, const majel::DDim& ddim) { + DDimPrinter printer(os); + boost::apply_visitor(printer, ddim); + return os; +} + +} // namespace majel diff --git a/paddle/majel/ddim.h b/paddle/majel/ddim.h new file mode 100644 index 0000000000..64cebf8958 --- /dev/null +++ b/paddle/majel/ddim.h @@ -0,0 +1,109 @@ +#pragma once + +#include +#include +#include +#include + +#include "majel/dim.h" + +namespace majel { + +namespace { +typedef boost::variant, + Dim<2>, + Dim<3>, + Dim<4>, + Dim<5>, + Dim<6>, + Dim<7>, + Dim<8>, + Dim<9>> + DDimVar; +} + +/** + * \brief A dynamically sized dimension. + * + * The number of dimensions must be between [1, 9]. + */ +struct DDim { + DDimVar var; + + DDim() : var(Dim<1>()) {} + + template + DDim(const Dim& in) : var(in) {} + + template + DDim& operator=(const Dim& in) { + var = in; + return *this; + } + + int& operator[](int idx); + int operator[](int idx) const; + + template + typename Visitor::result_type apply_visitor(Visitor& visitor) { + return var.apply_visitor(visitor); + } + + template + typename Visitor::result_type apply_visitor(Visitor& visitor) const { + return var.apply_visitor(visitor); + } + + DDimVar getVar() { return var; } + + bool operator==(DDim d) const; + + bool operator!=(DDim d) const; + + DDim operator+(DDim d) const; + + DDim operator*(DDim d) const; +}; + +/** + * \brief Make a DDim from std::vector + * + * \param dims An vector of ints. Must be sized between [1, 9] + */ +DDim make_ddim(const std::vector& dims); + +/** + * \brief Make a DDim from an initializer list + * + * \param dims An initializer list of ints. Must be sized between [1, 9] + * + */ +DDim make_ddim(std::initializer_list dims); + +int get(const DDim& dim, int idx); +void set(DDim& dim, int idx, int val); + +std::vector vectorize(const DDim& ddim); + +ssize_t product(const DDim& ddim); + +/** + * \brief What is the length of this dimension? + * + * \param Dynamic dimension to inspect + */ + +int arity(const DDim& ddim); + +std::ostream& operator<<(std::ostream&, const majel::DDim&); + +} // namespace majel + +namespace boost { + +template +T get(const majel::DDim& in) { + return boost::get(in.var); +} + +} // namespace boost diff --git a/paddle/majel/dim.h b/paddle/majel/dim.h new file mode 100644 index 0000000000..cf7682b686 --- /dev/null +++ b/paddle/majel/dim.h @@ -0,0 +1,456 @@ +#pragma once + +#include +#include +#include +#include +/* +#ifdef __CUDACC__ + #include +#endif +*/ + +#include "hostdevice.h" +#include "paddle/utils/Logging.h" + +namespace majel { + +// Statically sized, statically indexed dimension +template +struct Dim { + static constexpr int dimensions = i; + + template + HOSTDEVICE Dim(int _head, Args... _tail) : head(_head), tail(_tail...) { + static_assert(sizeof...(_tail) == i - 1, + "Dim initialized with the wrong number of parameters"); + } + + HOSTDEVICE + Dim(int _head, const Dim& _tail) : head(_head), tail(_tail) {} + + HOSTDEVICE + Dim() : head(0), tail() {} + + /** Construct a Dim from a linear index and size. Uses Fortran order + * indexing. */ + HOSTDEVICE + Dim(int idx, const Dim& size) + : head(idx % size.head), tail(idx / size.head, size.tail) {} + + /** Construct a Dim with each dimension set to the given index */ + HOSTDEVICE + Dim(int idx) : head(idx), tail(idx) {} + + HOSTDEVICE + bool operator==(const Dim& o) const { + return (head == o.head) && (tail == o.tail); + } + + HOSTDEVICE + bool operator!=(const Dim& o) const { return !(*this == o); } + + HOSTDEVICE + int& operator[](int idx); + HOSTDEVICE + int operator[](int idx) const; + + HOST std::string to_string() const; + + int head; + Dim tail; +}; + +// Base case specialization +template <> +struct Dim<1> { + static constexpr int dimensions = 1; + + HOSTDEVICE + Dim(int _head) : head(_head) {} + + HOSTDEVICE + Dim() : head(0) {} + + HOSTDEVICE + Dim(int idx, const Dim<1>& size) : head(idx) { +#ifndef __CUDA_ARCH__ + if (idx >= size.head) { + throw std::invalid_argument("Index out of range."); + } +#else + CHECK(idx < size.head); +#endif + } + + HOSTDEVICE + bool operator==(const Dim<1>& o) const { return (head == o.head); } + + HOSTDEVICE + bool operator!=(const Dim<1>& o) const { return !(*this == o); } + + HOSTDEVICE + int& operator[](int idx); + HOSTDEVICE + int operator[](int idx) const; + + int head; +}; + +namespace { + +// Helper for accessing Dim classes +template +struct DimGetter { + // Return a copy if Dim is const + template + HOSTDEVICE static int impl(const D& d) { + return DimGetter::impl(d.tail); + } + // Return a reference if Dim is mutable + template + HOSTDEVICE static int& impl(D& d) { + return DimGetter::impl(d.tail); + } +}; + +// Eureka! We found the element! +template <> +struct DimGetter<0> { + // Return a copy if Dim is const + template + HOSTDEVICE static int impl(const D& d) { + return d.head; + } + // Return a reference if Dim is mutable + template + HOSTDEVICE static int& impl(D& d) { + return d.head; + } +}; + +template +HOSTDEVICE int& indexer(Dim& dim, int idx) { +#ifndef __CUDA_ARCH__ + if (idx < 0) { + throw std::invalid_argument("Tried to access a negative dimension"); + } +#else + CHECK(idx >= 0); +#endif + if (idx == 0) { + return dim.head; + } + return indexer(dim.tail, idx - 1); +} + +template <> +HOSTDEVICE int& indexer<1>(Dim<1>& dim, int idx) { +#ifndef __CUDA_ARCH__ + if (idx != 0) { + throw std::invalid_argument("Invalid index"); + } +#else + CHECK(idx == 0); +#endif + return dim.head; +} + +template +HOSTDEVICE int indexer(const Dim& dim, int idx) { +#ifndef __CUDA_ARCH__ + if (idx < 0) { + throw std::invalid_argument("Tried to access a negative dimension"); + } +#else + CHECK(idx >= 0); +#endif + if (idx == 0) { + return dim.head; + } + return indexer(dim.tail, idx - 1); +} + +template <> +HOSTDEVICE int indexer<1>(const Dim<1>& dim, int idx) { +#ifndef __CUDA_ARCH__ + if (idx != 0) { + throw std::invalid_argument("Invalid index"); + } +#else + CHECK(idx == 0); +#endif + return dim.head; +} + +} // namespace +// Static access to constant Dim +template +HOSTDEVICE int get(const Dim& d) { + return DimGetter::impl(d); +} + +// Static access to mutable Dim +template +HOSTDEVICE int& get(Dim& d) { + return DimGetter::impl(d); +} + +// Dynamic access to constant Dim +template +HOSTDEVICE int Dim::operator[](int i) const { + return indexer(*this, i); +} + +// Dynamic access to mutable Dim +template +HOSTDEVICE int& Dim::operator[](int i) { + return indexer(*this, i); +} + +// Dynamic access to constant Dim +inline HOSTDEVICE int Dim<1>::operator[](int i) const { + return indexer(*this, i); +} + +// Dynamic access to mutable Dim +inline HOSTDEVICE int& Dim<1>::operator[](int i) { return indexer(*this, i); } + +// Dynamic access to constant Dim +// without std::enable_if will try to instantiate this on get<0>(d) +template +HOSTDEVICE typename std::enable_if<(l > 0), int>::type get(const Dim& d, + int i) { + return d[i]; +} + +// Dynamic access to mutable Dim +template +HOSTDEVICE typename std::enable_if<(l > 0), int&>::type get(Dim& d, int i) { + return d[i]; +} + +// Dot product of two dims +template +HOSTDEVICE int linearize(const Dim& a, const Dim& b) { + return a.head * b.head + linearize(a.tail, b.tail); +} + +// Base case dot product of two Dims +// Notice it is inline because it is no longer a template +template <> +HOSTDEVICE inline int linearize(const Dim<1>& a, const Dim<1>& b) { + return a.head * b.head; +} + +// Product of a Dim +template +HOSTDEVICE int product(const Dim& a, int prod = 1) { + return prod * a.head * product(a.tail); +} + +// Base case product of a Dim +// Notice it is inline because it is no longer a template +template <> +HOSTDEVICE inline int product(const Dim<1>& a, int prod) { + return prod * a.head; +} + +// Is 0 <= idx_i < size_i for all i? +template +HOSTDEVICE bool contained(const Dim& idx, const Dim& size) { + return ((0 <= idx.head) && (idx.head < size.head) && + contained(idx.tail, size.tail)); +} + +// Base case of is 0 <= idx_i < size_i ? +// Notice it is inline because it is no longer a template +template <> +HOSTDEVICE inline bool contained(const Dim<1>& idx, const Dim<1>& size) { + return ((0 <= idx.head) && (idx.head < size.head)); +} + +/** + * \brief Check if a size and a stride create a Fortran order contiguous + * block of memory. + */ +template +HOST bool contiguous(const Dim& size, const Dim& stride, int mul = 1) { + if (product(size) == 0) return true; + int contiguous_stride = get<0>(size) == 1 ? 0 : mul; + return (get<0>(stride) == contiguous_stride && + contiguous(size.tail, stride.tail, mul * get<0>(size))); +} + +///\cond HIDDEN +// Base case of contiguous, check the nth stride is the size of +// the prefix multiply of n-1 dims. +template <> +inline bool contiguous(const Dim<1>& size, const Dim<1>& stride, int mul) { + if (get<0>(size) == 0) return true; + int contiguous_stride = get<0>(size) == 1 ? 0 : mul; + return get<0>(stride) == contiguous_stride; +} +///\endcond + +/** + * \brief Compute exclusive prefix-multiply of a Dim. + */ +template +HOSTDEVICE Dim ex_prefix_mul(const Dim& src, int mul = 1) { + return Dim(mul, ex_prefix_mul(src.tail, mul * src.head)); +} + +///\cond HIDDEN +// Base case of ex_prefix_mul +// Notice it is inline because it is no longer a template +template <> +HOSTDEVICE inline Dim<1> ex_prefix_mul(const Dim<1>& src, int mul) { + return Dim<1>(mul); +} +///\endcond + +/** + * \brief Calculate strides of a contiguous array of the given size + * + * Sets the stride for any dimension with an extent of 1 to 0. + * \param size Dim object containing the size of the array. + * \param base The base stride to use. + * \return Dim object the same size as \p size with the strides. + */ +template +HOSTDEVICE Dim contiguous_strides(const Dim& size, int base = 1) { + int stride = size.head == 1 ? 0 : base; + return Dim(stride, contiguous_strides(size.tail, base * size.head)); +} + +///\cond HIDDEN + +// Base case of contiguous_strides +template <> +HOSTDEVICE inline Dim<1> contiguous_strides(const Dim<1>& size, int base) { + int stride = size.head == 1 ? 0 : base; + return Dim<1>(stride); +} + +///\endcond + +/** + * Add two dimensions together + */ +template +HOSTDEVICE Dim dim_plus(const Dim& a, const Dim& b) { + return Dim(a.head + b.head, dim_plus(a.tail, b.tail)); +} + +// Base case +template <> +HOSTDEVICE inline Dim<1> dim_plus(const Dim<1>& a, const Dim<1>& b) { + return Dim<1>(a.head + b.head); +} + +template +HOSTDEVICE Dim operator+(const Dim& lhs, const Dim& rhs) { + return dim_plus(lhs, rhs); +} + +/** + * Multiply two dimensions together + */ +template +HOSTDEVICE Dim dim_mult(const Dim& a, const Dim& b) { + return Dim(a.head * b.head, dim_mult(a.tail, b.tail)); +} + +// Base case +template <> +HOSTDEVICE inline Dim<1> dim_mult(const Dim<1>& a, const Dim<1>& b) { + return Dim<1>(a.head * b.head); +} + +template +HOSTDEVICE Dim operator*(const Dim& lhs, const Dim& rhs) { + return dim_mult(lhs, rhs); +} + +/** + * \brief Normalize strides to ensure any dimension with extent 1 + * has stride 0. + * + * \param size Dim object containing the size of an array + * \param stride Dim object containing stride of an array + * \return Dim object the same size as \p size with normalized strides + * + */ + +template +HOSTDEVICE Dim normalize_strides(const Dim& size, const Dim& stride) { + int norm_stride = size.head == 1 ? 0 : stride.head; + return Dim(norm_stride, normalize_strides(size.tail, stride.tail)); +} + +///\cond HIDDEN + +template <> +HOSTDEVICE inline Dim<1> normalize_strides(const Dim<1>& size, + const Dim<1>& stride) { + int norm_stride = size.head == 1 ? 0 : stride.head; + return Dim<1>(norm_stride); +} + +///\endcond + +/** + * Helper function to create a Dim + * + * \param idxes The type of Dim constructed depends on the number of params + * + */ + +template +HOSTDEVICE Dim make_dim(Args... idxes) { + return Dim(idxes...); +} + +// Allows us to output a Dim +// XXX For some reason, overloading fails to resolve this correctly +template +typename std::enable_if<(i > 1), std::ostream&>::type operator<<( + std::ostream& os, const majel::Dim& d) { + os << d.head << ", " << d.tail; + return os; +} + +// Base case that allows us to output a Dim +// XXX I wish this could be an overload instead of a template +template +typename std::enable_if<(i == 1), std::ostream&>::type operator<<( + std::ostream& os, const majel::Dim& d) { + os << d.head; + return os; +} + +template +HOST std::string Dim::to_string() const { + std::stringstream stream; + + stream << *this; + + return stream.str(); +} + +template +HOSTDEVICE Dim linear_to_dimension(int linear_index, Dim extents) { + Dim result; + + for (int i = 0; i < D - 1; ++i) { + result[i] = linear_index % extents[i]; + linear_index /= extents[i]; + } + + result[D - 1] = linear_index; + + return result; +} + +} // namespace majel diff --git a/paddle/majel/hostdevice.h b/paddle/majel/hostdevice.h new file mode 100644 index 0000000000..e7de86b7b2 --- /dev/null +++ b/paddle/majel/hostdevice.h @@ -0,0 +1,9 @@ +#pragma once + +#ifdef __CUDACC__ +#define HOSTDEVICE __host__ __device__ +#define HOST __host__ +#else +#define HOSTDEVICE +#define HOST +#endif diff --git a/paddle/majel/test/CMakeLists.txt b/paddle/majel/test/CMakeLists.txt index 68f9059874..4245379952 100644 --- a/paddle/majel/test/CMakeLists.txt +++ b/paddle/majel/test/CMakeLists.txt @@ -2,6 +2,11 @@ cc_test(place_test SRCS place_test.cc DEPS majel) +cc_test(ddim_test + SRCS ddim_test.cc + DEPS majel) + if(WITH_GPU) nv_test(cuda_test SRCS cuda_test.cu) + nv_test(dim_test SRCS dim_test.cu DEPS majel) endif() diff --git a/paddle/majel/test/ddim_test.cc b/paddle/majel/test/ddim_test.cc new file mode 100644 index 0000000000..5c60451796 --- /dev/null +++ b/paddle/majel/test/ddim_test.cc @@ -0,0 +1,65 @@ +//#include +//#include +#include +#include + +#include "gtest/gtest.h" +#include "majel/ddim.h" + +TEST(DDim, Equality) { + // construct a DDim from an initialization list + majel::DDim ddim = majel::make_ddim({9, 1, 5}); + EXPECT_EQ(ddim[0], 9); + EXPECT_EQ(ddim[1], 1); + EXPECT_EQ(ddim[2], 5); + + // construct a DDim from a vector + std::vector vec({9, 1, 5}); + majel::DDim vddim = majel::make_ddim(vec); + EXPECT_EQ(ddim[0], 9); + EXPECT_EQ(ddim[1], 1); + EXPECT_EQ(ddim[2], 5); + + // mutate a DDim + ddim[1] = 2; + EXPECT_EQ(ddim[1], 2); + majel::set(ddim, 0, 6); + EXPECT_EQ(majel::get(ddim, 0), 6); + + // vectorize a DDim + std::vector res_vec = majel::vectorize(vddim); + EXPECT_EQ(res_vec[0], 9); + EXPECT_EQ(res_vec[1], 1); + EXPECT_EQ(res_vec[2], 5); + majel::Dim<3> d(3, 2, 1); + res_vec = majel::vectorize(majel::DDim(d)); + EXPECT_EQ(res_vec[0], 3); + EXPECT_EQ(res_vec[1], 2); + EXPECT_EQ(res_vec[2], 1); + + // add two DDims + majel::DDim ddim_sum = ddim + vddim; + EXPECT_EQ(ddim_sum[0], 15); + EXPECT_EQ(ddim_sum[1], 3); + EXPECT_EQ(ddim_sum[2], 10); + + // multiply two DDims + majel::DDim ddim_mul = ddim * vddim; + EXPECT_EQ(ddim_mul[0], 54); + EXPECT_EQ(ddim_mul[1], 2); + EXPECT_EQ(ddim_mul[2], 25); + + // arity of a DDim + EXPECT_EQ(majel::arity(ddim), 3); + + // product of a DDim + EXPECT_EQ(majel::product(vddim), 45); +} + +TEST(DDim, Print) { + // print a DDim + std::stringstream ss; + majel::DDim ddim = majel::make_ddim({2, 3, 4}); + ss << ddim; + EXPECT_EQ("2, 3, 4", ss.str()); +} diff --git a/paddle/majel/test/dim_test.cu b/paddle/majel/test/dim_test.cu new file mode 100644 index 0000000000..380204531d --- /dev/null +++ b/paddle/majel/test/dim_test.cu @@ -0,0 +1,128 @@ +#include +#include +#include "majel/dim.h" + +#include "gtest/gtest.h" + +__global__ void test(majel::Dim<2>* o) { + o[0] = majel::make_dim(5, 6); +} + +__global__ void dyn_idx_gpu(int* o) { + auto d = majel::make_dim(5, 6); + o[0] = d[1]; +} + +TEST(Dim, Equality) { + // construct a Dim on the CPU + auto a = majel::make_dim(3, 4); + EXPECT_EQ(get<0>(a), 3); + EXPECT_EQ(get<1>(a), 4); + + // construct a Dim on the GPU + thrust::device_vector> t(2); + test<<<1,1>>>(thrust::raw_pointer_cast(t.data())); + a = t[0]; + EXPECT_EQ(get<0>(a), 5); + EXPECT_EQ(get<1>(a), 6); + + // linearization + auto b = make_dim(7, 8); + EXPECT_EQ(linearize(a, b), 83); + + // product + EXPECT_EQ(product(a), 30); + + // mutate a Dim + majel::get<1>(b) = 10; + EXPECT_EQ(majel::get<0>(b), 7); + EXPECT_EQ(majel::get<1>(b), 10); + + // dynamic access + majel::get(b, 0) = 8; + b[1] = 11; + EXPECT_EQ(majel::get<0>(b), 8); + EXPECT_EQ(majel::get<1>(b), 11); + EXPECT_EQ(majel::get(b, 0), 8); + EXPECT_EQ(b[1], 11); + + // dynamic access on GPU + thrust::device_vector r(1); + dyn_idx_gpu<<<1,1>>>(thrust::raw_pointer_cast(r.data())); + int res = r[0]; + EXPECT_EQ(res, 6); + + // ex_prefix_mul + majel::Dim<3> c = majel::ex_prefix_mul(Dim<3>(3, 4, 5)); + EXPECT_EQ(majel::get<0>(c), 1); + EXPECT_EQ(majel::get<1>(c), 3); + EXPECT_EQ(majel::get<2>(c), 12); + + // contiguous_strides + c = majel::contiguous_strides(majel::Dim<3>(10, 1, 10)); + EXPECT_EQ(majel::get<0>(c), 1); + EXPECT_EQ(majel::get<1>(c), 0); + EXPECT_EQ(majel::get<2>(c), 10); + c = majel::contiguous_strides(majel::Dim<3>(10, 10, 1)); + EXPECT_EQ(majel::get<0>(c), 1); + EXPECT_EQ(majel::get<1>(c), 10); + EXPECT_EQ(majel::get<2>(c), 0); + c = majel::contiguous_strides(majel::Dim<3>(1, 10, 10)); + EXPECT_EQ(majel::get<0>(c), 0); + EXPECT_EQ(majel::get<1>(c), 1); + EXPECT_EQ(majel::get<2>(c), 10); + c = majel::contiguous_strides(majel::Dim<3>(2, 3, 4)); + EXPECT_EQ(majel::get<0>(c), 1); + EXPECT_EQ(majel::get<1>(c), 2); + EXPECT_EQ(majel::get<2>(c), 6); + + // generate from an index + auto size = majel::make_dim(4, 5, 2); + c = majel::Dim<3>(14, size); + EXPECT_EQ(majel::get<0>(c), 2); + EXPECT_EQ(majel::get<1>(c), 3); + EXPECT_EQ(majel::get<2>(c), 0); + c = majel::Dim<3>(25, size); + EXPECT_EQ(majel::get<0>(c), 1); + EXPECT_EQ(majel::get<1>(c), 1); + EXPECT_EQ(majel::get<2>(c), 1); +} + +TEST(Dim, Bool) { + auto a = majel::make_dim(3, 4); + auto b = majel::make_dim(5, 6); + auto c = majel::make_dim(3, 4); + + // in_bounds check + EXPECT_TRUE(majel::contained(a, b)); + EXPECT_FALSE(majel::contained(b, a)); + + // comparison + EXPECT_TRUE(a == a); + EXPECT_FALSE(a == b); + EXPECT_TRUE(a == c); + + // contiguous check + int x = 4, y = 5, z = 2; + majel::Dim<3> sizef(x, y, z); + majel::Dim<3> stridea(1, x, x*y); + majel::Dim<3> strideb(2, 2*x, 2*x*y); + majel::Dim<3> stridec(1, x, 2*x*y); + EXPECT_TRUE(majel::contiguous(sizef, stridea)); + EXPECT_FALSE(majel::contiguous(sizef, strideb)); + EXPECT_FALSE(majel::contiguous(sizef, stridec)); +} + +TEST(Dim, Print) { + { + std::stringstream ss; + auto a = majel::make_dim(2, 3); + ss << a; + EXPECT_EQ(ss.str(), "2, 3"); + } + { + std::stringstream ss; + ss << majel::make_dim(8); + EXPECT_EQ(ss.str(), "8"); + } +} From 835ccb326252846fa54c17298d235751c82fd974 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 18 May 2017 16:31:14 +0800 Subject: [PATCH 20/30] refine cc_xxx in generic --- cmake/generic.cmake | 16 ++++++++++------ paddle/majel/test/place_test.cc | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 22a26d7c5b..06b09a6ef8 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -57,8 +57,9 @@ function(cc_binary TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(cc_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${TARGET_NAME} ${cc_binary_SRCS}) - add_dependencies(${TARGET_NAME} ${cc_binary_DEPS} ${external_project_dependencies}) + link_paddle_exe(${TARGET_NAME}) target_link_libraries(${TARGET_NAME} ${cc_binary_DEPS}) + add_dependencies(${TARGET_NAME} ${cc_binary_DEPS}) endfunction(cc_binary) # The dependency to target tensor implies that if any of @@ -74,8 +75,9 @@ function(cc_test TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(cc_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${TARGET_NAME} ${cc_test_SRCS}) - add_dependencies(${TARGET_NAME} ${cc_test_DEPS} ${external_project_dependencies}) - target_link_libraries(${TARGET_NAME} ${cc_test_DEPS} ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) + link_paddle_test(${TARGET_NAME}) + target_link_libraries(${TARGET_NAME} ${cc_test_DEPS}) + add_dependencies(${TARGET_NAME} ${cc_test_DEPS}) add_test(${TARGET_NAME} ${TARGET_NAME}) endfunction(cc_test) @@ -106,8 +108,9 @@ function(nv_binary TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(nv_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) cuda_add_executable(${TARGET_NAME} ${nv_binary_SRCS}) - add_dependencies(${TARGET_NAME} ${nv_binary_DEPS} ${external_project_dependencies}) + link_paddle_exe(${TARGET_NAME}) target_link_libraries(${TARGET_NAME} ${nv_binary_DEPS}) + add_dependencies(${TARGET_NAME} ${nv_binary_DEPS}) endfunction(nv_binary) # The dependency to target tensor implies that if any of @@ -123,7 +126,8 @@ function(nv_test TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(nv_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) cuda_add_executable(${TARGET_NAME} ${nv_test_SRCS}) - add_dependencies(${TARGET_NAME} ${nv_test_DEPS} ${external_project_dependencies}) - target_link_libraries(${TARGET_NAME} ${nv_test_DEPS} ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) + link_paddle_test(${TARGET_NAME}) + target_link_libraries(${TARGET_NAME} ${nv_test_DEPS}) + add_dependencies(${TARGET_NAME} ${nv_test_DEPS}) add_test(${TARGET_NAME} ${TARGET_NAME}) endfunction(nv_test) diff --git a/paddle/majel/test/place_test.cc b/paddle/majel/test/place_test.cc index c9a53802b2..686ed9c0e8 100644 --- a/paddle/majel/test/place_test.cc +++ b/paddle/majel/test/place_test.cc @@ -1,6 +1,7 @@ #include "majel/place.h" #include #include "gtest/gtest.h" +#include "paddle/utils/Logging.h" TEST(Place, Equality) { majel::CpuPlace cpu; @@ -37,4 +38,5 @@ TEST(Place, Print) { ss << majel::CpuPlace(); EXPECT_EQ("CpuPlace", ss.str()); } + LOG(INFO) << "\n[----------] Done \n"; } From 8ea4548ad3b61c31d1e125baf742bc4d5f8d741f Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 18 May 2017 16:51:09 +0800 Subject: [PATCH 21/30] fix a bug --- cmake/generic.cmake | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 06b09a6ef8..3ca735189d 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -58,8 +58,10 @@ function(cc_binary TARGET_NAME) cmake_parse_arguments(cc_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${TARGET_NAME} ${cc_binary_SRCS}) link_paddle_exe(${TARGET_NAME}) - target_link_libraries(${TARGET_NAME} ${cc_binary_DEPS}) - add_dependencies(${TARGET_NAME} ${cc_binary_DEPS}) + if(cc_binary_DEPS) + target_link_libraries(${TARGET_NAME} ${cc_binary_DEPS}) + add_dependencies(${TARGET_NAME} ${cc_binary_DEPS}) + endif() endfunction(cc_binary) # The dependency to target tensor implies that if any of @@ -76,8 +78,10 @@ function(cc_test TARGET_NAME) cmake_parse_arguments(cc_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${TARGET_NAME} ${cc_test_SRCS}) link_paddle_test(${TARGET_NAME}) - target_link_libraries(${TARGET_NAME} ${cc_test_DEPS}) - add_dependencies(${TARGET_NAME} ${cc_test_DEPS}) + if(cc_test_DEPS) + target_link_libraries(${TARGET_NAME} ${cc_test_DEPS}) + add_dependencies(${TARGET_NAME} ${cc_test_DEPS}) + endif() add_test(${TARGET_NAME} ${TARGET_NAME}) endfunction(cc_test) @@ -109,8 +113,10 @@ function(nv_binary TARGET_NAME) cmake_parse_arguments(nv_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) cuda_add_executable(${TARGET_NAME} ${nv_binary_SRCS}) link_paddle_exe(${TARGET_NAME}) - target_link_libraries(${TARGET_NAME} ${nv_binary_DEPS}) - add_dependencies(${TARGET_NAME} ${nv_binary_DEPS}) + if(nv_binary_DEPS) + target_link_libraries(${TARGET_NAME} ${nv_binary_DEPS}) + add_dependencies(${TARGET_NAME} ${nv_binary_DEPS}) + endif() endfunction(nv_binary) # The dependency to target tensor implies that if any of @@ -127,7 +133,9 @@ function(nv_test TARGET_NAME) cmake_parse_arguments(nv_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) cuda_add_executable(${TARGET_NAME} ${nv_test_SRCS}) link_paddle_test(${TARGET_NAME}) - target_link_libraries(${TARGET_NAME} ${nv_test_DEPS}) - add_dependencies(${TARGET_NAME} ${nv_test_DEPS}) + if(nv_test_DEPS) + target_link_libraries(${TARGET_NAME} ${nv_test_DEPS}) + add_dependencies(${TARGET_NAME} ${nv_test_DEPS}) + endif() add_test(${TARGET_NAME} ${TARGET_NAME}) endfunction(nv_test) From f8a42d5de98f83bc9703c31dc47ca05c6590341e Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 18 May 2017 16:54:32 +0800 Subject: [PATCH 22/30] add doc for AggregateLevel and ExpandLevel --- doc/api/v2/config/layer.rst | 10 ++++++ .../paddle/trainer_config_helpers/layers.py | 31 +++++++++++++++++++ python/paddle/v2/layer.py | 4 +-- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index 154cfe2443..1efa74ecda 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -207,6 +207,11 @@ trans_full_matrix_projection Aggregate Layers ================ +AggregateLevel +-------------- +.. autoclass:: paddle.v2.layer.AggregateLevel + :noindex: + .. _api_v2.layer_pooling: pooling @@ -248,6 +253,11 @@ block_expand .. _api_v2.layer_expand: +ExpandLevel +----------- +.. autoclass:: paddle.v2.layer.ExpandLevel + :noindex: + expand ------ .. autoclass:: paddle.v2.layer.expand diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 3b6f0270de..cf7e54ef50 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -225,6 +225,24 @@ class LayerType(object): class AggregateLevel(object): + """ + As PaddlePaddle supports three sequence types: + + - :code:`SequenceType.NO_SEQUENCE` means the sample is not a sequence. + - :code:`SequenceType.SEQUENCE` means the sample is a sequence. + - :code:`SequenceType.SUB_SEQUENCE` means it is a nested sequence, that + each timestep of the input sequence is also a sequence. + + Thus, AggregateLevel supports two modes: + + - :code:`AggregateLevel.EACH_TIMESTEP` means the aggregation acts on each + timestep of sequence, both :code:`SUB_SEQUENCE` and :code:`SEQUENCE` will + be aggregated to :code:`NO_SEQUENCE`. + + - :code:`AggregateLevel.EACH_SEQUENCE` means the aggregation acts on each + sequence of a nested sequence, :code:`SUB_SEQUENCE` will be aggregated to + :code:`SEQUENCE`. + """ EACH_TIMESTEP = 'non-seq' EACH_SEQUENCE = 'seq' @@ -1454,6 +1472,19 @@ def first_seq(input, class ExpandLevel(object): + """ + Please refer to AggregateLevel first. + + ExpandLevel supports two modes: + + - :code:`ExpandLevel.FROM_TIMESTEP` means the expandation acts on each + timestep of a sequence, :code:`NO_SEQUENCE` will be expanded to + :code:`SEQUENCE` or :code:`SUB_SEQUENCE`. + + - :code:`ExpandLevel.FROM_SEQUENCE` means the expandation acts on each + sequence of a nested sequence, :code:`SEQUENCE` will be expanded to + :code:`SUB_SEQUENCE`. + """ FROM_TIMESTEP = AggregateLevel.EACH_TIMESTEP FROM_SEQUENCE = AggregateLevel.EACH_SEQUENCE diff --git a/python/paddle/v2/layer.py b/python/paddle/v2/layer.py index 89cca7acd3..3d9caeec58 100644 --- a/python/paddle/v2/layer.py +++ b/python/paddle/v2/layer.py @@ -404,8 +404,8 @@ class RecurrentLayerOutput(Layer): LayerV2 = Layer data = DataLayerV2 data.__name__ = 'data' -AggregateLevel = conf_helps.layers.AggregateLevel -ExpandLevel = conf_helps.layers.ExpandLevel +AggregateLevel = conf_helps.AggregateLevel +ExpandLevel = conf_helps.ExpandLevel memory = MemoryV2 memory.__name__ = 'memory' memory.__doc__ = conf_helps.memory.__doc__ From 1cf9b5ae055dab5e9513f67c5c6cc6c14ebfe612 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 18 May 2017 17:11:58 +0800 Subject: [PATCH 23/30] fix swig compile flags --- paddle/api/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index c9a285c90b..53db938b35 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -26,10 +26,7 @@ FILE(GLOB PY_PADDLE_PYTHON_FILES ${PROJ_ROOT}/paddle/py_paddle/*.py) SET_SOURCE_FILES_PROPERTIES(Paddle.i PROPERTIES CPLUSPLUS ON) SET(CMAKE_SWIG_OUTDIR ${CMAKE_CURRENT_BINARY_DIR}) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wall") -IF(WITH_COVERAGE) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") -ENDIF(WITH_COVERAGE) +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality -Wno-self-assign") SET(SWIG_MODULE_swig_paddle_EXTRA_DEPS paddle_parameter From 3a4ab8206a8e1b0d7a59926a2dae701628bba65e Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 18 May 2017 17:19:54 +0800 Subject: [PATCH 24/30] add Wno-missing-field-initializers --- paddle/api/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index 53db938b35..e147659566 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -26,7 +26,7 @@ FILE(GLOB PY_PADDLE_PYTHON_FILES ${PROJ_ROOT}/paddle/py_paddle/*.py) SET_SOURCE_FILES_PROPERTIES(Paddle.i PROPERTIES CPLUSPLUS ON) SET(CMAKE_SWIG_OUTDIR ${CMAKE_CURRENT_BINARY_DIR}) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality -Wno-self-assign") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality -Wno-missing-field-initializers -Wno-self-assign") SET(SWIG_MODULE_swig_paddle_EXTRA_DEPS paddle_parameter From 110a402221d7d719f8c0eb50cea8f1f3bfedee9d Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 18 May 2017 09:28:49 +0000 Subject: [PATCH 25/30] make up missing 'majel::' and import majel/util.h --- paddle/majel/dim.h | 19 +++++++---------- paddle/majel/test/dim_test.cu | 18 ++++++++-------- paddle/majel/util.h | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 paddle/majel/util.h diff --git a/paddle/majel/dim.h b/paddle/majel/dim.h index cf7682b686..ba718ce5b7 100644 --- a/paddle/majel/dim.h +++ b/paddle/majel/dim.h @@ -4,14 +4,9 @@ #include #include #include -/* -#ifdef __CUDACC__ - #include -#endif -*/ -#include "hostdevice.h" -#include "paddle/utils/Logging.h" +#include "majel/hostdevice.h" +#include "majel/util.h" namespace majel { @@ -79,7 +74,7 @@ struct Dim<1> { throw std::invalid_argument("Index out of range."); } #else - CHECK(idx < size.head); + MAJEL_ASSERT(idx < size.head); #endif } @@ -136,7 +131,7 @@ HOSTDEVICE int& indexer(Dim& dim, int idx) { throw std::invalid_argument("Tried to access a negative dimension"); } #else - CHECK(idx >= 0); + MAJEL_ASSERT(idx >= 0); #endif if (idx == 0) { return dim.head; @@ -151,7 +146,7 @@ HOSTDEVICE int& indexer<1>(Dim<1>& dim, int idx) { throw std::invalid_argument("Invalid index"); } #else - CHECK(idx == 0); + MAJEL_ASSERT(idx == 0); #endif return dim.head; } @@ -163,7 +158,7 @@ HOSTDEVICE int indexer(const Dim& dim, int idx) { throw std::invalid_argument("Tried to access a negative dimension"); } #else - CHECK(idx >= 0); + MAJEL_ASSERT(idx >= 0); #endif if (idx == 0) { return dim.head; @@ -178,7 +173,7 @@ HOSTDEVICE int indexer<1>(const Dim<1>& dim, int idx) { throw std::invalid_argument("Invalid index"); } #else - CHECK(idx == 0); + MAJEL_ASSERT(idx == 0); #endif return dim.head; } diff --git a/paddle/majel/test/dim_test.cu b/paddle/majel/test/dim_test.cu index 380204531d..ed7c34ffd1 100644 --- a/paddle/majel/test/dim_test.cu +++ b/paddle/majel/test/dim_test.cu @@ -1,7 +1,7 @@ #include #include -#include "majel/dim.h" +#include "majel/dim.h" #include "gtest/gtest.h" __global__ void test(majel::Dim<2>* o) { @@ -16,22 +16,22 @@ __global__ void dyn_idx_gpu(int* o) { TEST(Dim, Equality) { // construct a Dim on the CPU auto a = majel::make_dim(3, 4); - EXPECT_EQ(get<0>(a), 3); - EXPECT_EQ(get<1>(a), 4); + EXPECT_EQ(majel::get<0>(a), 3); + EXPECT_EQ(majel::get<1>(a), 4); // construct a Dim on the GPU thrust::device_vector> t(2); test<<<1,1>>>(thrust::raw_pointer_cast(t.data())); a = t[0]; - EXPECT_EQ(get<0>(a), 5); - EXPECT_EQ(get<1>(a), 6); + EXPECT_EQ(majel::get<0>(a), 5); + EXPECT_EQ(majel::get<1>(a), 6); // linearization - auto b = make_dim(7, 8); - EXPECT_EQ(linearize(a, b), 83); + auto b = majel::make_dim(7, 8); + EXPECT_EQ(majel::linearize(a, b), 83); // product - EXPECT_EQ(product(a), 30); + EXPECT_EQ(majel::product(a), 30); // mutate a Dim majel::get<1>(b) = 10; @@ -53,7 +53,7 @@ TEST(Dim, Equality) { EXPECT_EQ(res, 6); // ex_prefix_mul - majel::Dim<3> c = majel::ex_prefix_mul(Dim<3>(3, 4, 5)); + majel::Dim<3> c = majel::ex_prefix_mul(majel::Dim<3>(3, 4, 5)); EXPECT_EQ(majel::get<0>(c), 1); EXPECT_EQ(majel::get<1>(c), 3); EXPECT_EQ(majel::get<2>(c), 12); diff --git a/paddle/majel/util.h b/paddle/majel/util.h new file mode 100644 index 0000000000..ab21e1869f --- /dev/null +++ b/paddle/majel/util.h @@ -0,0 +1,39 @@ +#pragma once + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + +#if defined(__APPLE__) && defined(__CUDA_ARCH__) && !defined(NDEBUG) +#include +#define MAJEL_ASSERT(e) \ + do { \ + if (!(e)) { \ + printf( \ + "%s:%d Assertion `%s` failed.\n", __FILE__, __LINE__, TOSTRING(e)); \ + asm("trap;"); \ + } \ + } while (0) + +#define MAJEL_ASSERT_MSG(e, m) \ + do { \ + if (!(e)) { \ + printf("%s:%d Assertion `%s` failed (%s).\n", \ + __FILE__, \ + __LINE__, \ + TOSTRING(e), \ + m); \ + asm("trap;"); \ + } \ + } while (0) +#else +#include +#define MAJEL_ASSERT(e) assert(e) +#define MAJEL_ASSERT_MSG(e, m) assert((e) && (m)) +#endif + +namespace majel { +namespace detail { + +inline int div_up(int x, int y) { return (x + y - 1) / y; } +} +} From c7fb78b0a5245cf6852b3cf7d0e617e75c404b46 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 18 May 2017 19:48:54 +0800 Subject: [PATCH 26/30] follow comments --- python/paddle/trainer_config_helpers/layers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index cf7e54ef50..ec81e1dc3d 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -226,17 +226,17 @@ class LayerType(object): class AggregateLevel(object): """ - As PaddlePaddle supports three sequence types: + PaddlePaddle supports three sequence types: - :code:`SequenceType.NO_SEQUENCE` means the sample is not a sequence. - :code:`SequenceType.SEQUENCE` means the sample is a sequence. - - :code:`SequenceType.SUB_SEQUENCE` means it is a nested sequence, that - each timestep of the input sequence is also a sequence. + - :code:`SequenceType.SUB_SEQUENCE` means the sample is a nested sequence, + each timestep of which is also a sequence. - Thus, AggregateLevel supports two modes: + Accordingly, AggregateLevel supports two modes: - :code:`AggregateLevel.EACH_TIMESTEP` means the aggregation acts on each - timestep of sequence, both :code:`SUB_SEQUENCE` and :code:`SEQUENCE` will + timestep of a sequence, both :code:`SUB_SEQUENCE` and :code:`SEQUENCE` will be aggregated to :code:`NO_SEQUENCE`. - :code:`AggregateLevel.EACH_SEQUENCE` means the aggregation acts on each From a36c6fb965152b8ecf3c0b0ebb7a6d51470d567f Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Thu, 18 May 2017 14:12:25 -0700 Subject: [PATCH 27/30] Remove go.md and Grammerly README.md --- doc/design/build_system/README.md | 7 +-- doc/design/build_system/go.md | 98 ------------------------------- 2 files changed, 2 insertions(+), 103 deletions(-) delete mode 100644 doc/design/build_system/go.md diff --git a/doc/design/build_system/README.md b/doc/design/build_system/README.md index 4c4315d6b9..310739f37a 100644 --- a/doc/design/build_system/README.md +++ b/doc/design/build_system/README.md @@ -20,7 +20,7 @@ The difference between `nv_` functions and `cc_` functions is that the former us Both `nv_` and `cc_` functions enables C++11 (-std=c++11). -In addition, +Also, - to describe external dependencies, we need `external_library`. - to build shared libraries, we need `shared_library`. @@ -104,7 +104,4 @@ shared_library(api ### Implementation -As above example CMakeLists.txt executes, each function invocation -adds "nodes" to a dependency graph. It also use this graph to -generate CMake commands including `add_executable`, -`add_dependencies`, `target_link_libraries`, and `add_test`. +As above example CMakeLists.txt executes, each function invocation adds "nodes" to a dependency graph. It also use this graph to generate CMake commands including `add_executable`, `add_dependencies`, `target_link_libraries`, and `add_test`. diff --git a/doc/design/build_system/go.md b/doc/design/build_system/go.md deleted file mode 100644 index 90399ff701..0000000000 --- a/doc/design/build_system/go.md +++ /dev/null @@ -1,98 +0,0 @@ -# Design Doc: `go_{library,binary,test}` - -## Concerns - -1. Need to build Go libraries callable from Go and from C. - - For usual Go libraries, the bulding command line is as - - ```bash - go build foo.go bar.go -o libfoobar.a - ``` - - For Go libraries callable from C/C++, the command line is - - ```bash - go build -buildmode=c-archive foo.go bar.go -o libstatic.a - ``` - - or for shared libraries: - - ```bash - go build -buildmode=c-shared foo.go bar.go -o libdynamic.so - ``` - - and `foo.go`, `bar.go`, etc must start with a line `package main`, - which defines all symbols in special pacakge `main`. There also - must be a `func main`, which could have empty body. - -1. Need to support building-in-Docker. - - We are going to support two ways to building -- (1) in Ubuntu, and - (2) in Docker container whose base image is Ubuntu. - - The challenge is (2), because to build in Docker, we run the - development image as: - - ```bash - git clone https://github.com/PaddlePaddle/Paddle -o paddle - cd paddle - docker run -v $PWD:/paddle paddle:dev - ``` - - which maps the local repo to `/paddle` in the container. - - This assumes that all Go code, including third-party packages, must - be in the local repo. Actually, it assumes that `$GOPATH` must be - in the local repo. This would affect how we write `import` - statemetns, and how we maintain third-party packages. - - -## A Solution - -This might not be the optimal solution. Comments are welcome. - -### Directory structure - -We propose the following directory structure: - -``` -https://github.com/PaddlePaddle/Paddle - ↓ git clone - ~/work/paddle/go/pkg1/foo.go - /pkg2/bar.go - /cmd/cmd1/wee.go - /cmd/cmd2/qux.go - /github.com/someone/a_3rd_party_pkg (Git submodule to a 3rd-party pkg) - /golang.org/another_3rd_party_pkg (Git submodule to another one) - /build/go ($GOPATH, required by Go toolchain) - /src (symlink to ~/work/paddle/go/) - /pkg (libraries built by Go toolchain) - /bin (binaries bulit by Go toolchain) -``` - -Above figure explains how we organize Paddle's Go code: - -1. Go source code in Paddle is in `/go` of the repo. -1. Each library package is a sub-directory under `/go`. -1. Each (executable) binary package is a sub-directory under - `/go/cmd`. This is the source tree convention of Go itself. -1. Each 3rd-party Go package is a Git submodule under `/go`. - -These rules make sure that all Go source code are in `/go`. - -At build-time, Go toolchain requires a directory structure rooted at -`$GOPATH` and having three sub-directories: `$GOPATH/src`, -`$GOPATH/pkg`, and `$GOPATH/bin`, where `$GOPATH/src` is the source -tree and the root of Go's `import` paths. - -For example, if `/go/pkg1/foo.go` contains `import -"github.com/someone/a_3rd_party_pkg"`, the Go toolchain will find the -package at `$GOPATH/src/github.com/someone/a_3rd_party_pkg`. - -In order to create such a `$GOPATH`, our build system creates -`/build/go`. Remeber that we want to make sure that all output files -generated at build-time are place in `/build`. - -Under `/build/go`, our build system creates a symoblic link `src` -pointing to `/go`, where all Go source code resides. From 478ebc42b73e30e16389bf27693d93ce3a9c0cb0 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Thu, 18 May 2017 18:37:08 -0400 Subject: [PATCH 28/30] apply clang-format --- paddle/go/cclient/test/main.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/paddle/go/cclient/test/main.c b/paddle/go/cclient/test/main.c index d71a023a6a..28e3d03b7a 100644 --- a/paddle/go/cclient/test/main.c +++ b/paddle/go/cclient/test/main.c @@ -11,7 +11,7 @@ void panic() { int main() { char addr[] = "localhost:3000"; client c = paddle_new_pserver_client(addr); - retry: +retry: if (paddle_begin_init_params(c, NULL, 0)) { paddle_parameter param; char name_a[] = "param_a"; @@ -39,14 +39,16 @@ int main() { } char content[] = {0x00, 0x11, 0x22}; - paddle_gradient grads[2] = {{"param_a", PADDLE_ELEMENT_TYPE_INT32, content, 3}, {"param_b", PADDLE_ELEMENT_TYPE_FLOAT32, content, 3}}; + paddle_gradient grads[2] = { + {"param_a", PADDLE_ELEMENT_TYPE_INT32, content, 3}, + {"param_b", PADDLE_ELEMENT_TYPE_FLOAT32, content, 3}}; if (!paddle_send_grads(c, grads, 2)) { panic(); } paddle_parameter* params[2] = {NULL, NULL}; - char* names[]={"param_a", "param_b"}; + char* names[] = {"param_a", "param_b"}; if (!paddle_get_params(c, names, params, 2)) { panic(); } From 82c60bbbb5831ffb5a0a8f65cdc528b0c637e60d Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 19 May 2017 11:49:38 +0800 Subject: [PATCH 29/30] adjust '#include' style and move macro definition files into folder 'detail' --- paddle/majel/ddim.cc | 2 +- paddle/majel/ddim.h | 2 +- paddle/majel/{util.h => detail/cuda_assert.h} | 7 ------- paddle/majel/{ => detail}/hostdevice.h | 0 paddle/majel/dim.h | 4 ++-- paddle/majel/place.cc | 4 ++-- paddle/majel/test/ddim_test.cc | 2 +- paddle/majel/test/dim_test.cu | 2 +- paddle/majel/test/place_test.cc | 2 +- 9 files changed, 9 insertions(+), 16 deletions(-) rename paddle/majel/{util.h => detail/cuda_assert.h} (93%) rename paddle/majel/{ => detail}/hostdevice.h (100%) diff --git a/paddle/majel/ddim.cc b/paddle/majel/ddim.cc index e92d9afa0b..f32408ed53 100644 --- a/paddle/majel/ddim.cc +++ b/paddle/majel/ddim.cc @@ -1,4 +1,4 @@ -#include +#include "paddle/majel/ddim.h" namespace majel { diff --git a/paddle/majel/ddim.h b/paddle/majel/ddim.h index 64cebf8958..7be756f8c0 100644 --- a/paddle/majel/ddim.h +++ b/paddle/majel/ddim.h @@ -5,7 +5,7 @@ #include #include -#include "majel/dim.h" +#include "paddle/majel/dim.h" namespace majel { diff --git a/paddle/majel/util.h b/paddle/majel/detail/cuda_assert.h similarity index 93% rename from paddle/majel/util.h rename to paddle/majel/detail/cuda_assert.h index ab21e1869f..9490d0ae3e 100644 --- a/paddle/majel/util.h +++ b/paddle/majel/detail/cuda_assert.h @@ -30,10 +30,3 @@ #define MAJEL_ASSERT(e) assert(e) #define MAJEL_ASSERT_MSG(e, m) assert((e) && (m)) #endif - -namespace majel { -namespace detail { - -inline int div_up(int x, int y) { return (x + y - 1) / y; } -} -} diff --git a/paddle/majel/hostdevice.h b/paddle/majel/detail/hostdevice.h similarity index 100% rename from paddle/majel/hostdevice.h rename to paddle/majel/detail/hostdevice.h diff --git a/paddle/majel/dim.h b/paddle/majel/dim.h index ba718ce5b7..5eff4c05cc 100644 --- a/paddle/majel/dim.h +++ b/paddle/majel/dim.h @@ -5,8 +5,8 @@ #include #include -#include "majel/hostdevice.h" -#include "majel/util.h" +#include "paddle/majel/detail/hostdevice.h" +#include "paddle/majel/detail/util.h" namespace majel { diff --git a/paddle/majel/place.cc b/paddle/majel/place.cc index eecd8e5b73..ca50b37843 100644 --- a/paddle/majel/place.cc +++ b/paddle/majel/place.cc @@ -1,4 +1,4 @@ -#include +#include "paddle/majel/place.h" namespace majel { @@ -16,7 +16,7 @@ public: void operator()(const GpuPlace& p) { os_ << "GpuPlace(" << p.device << ")"; } }; -} // namespace majel +} // namespace detail static Place the_default_place; diff --git a/paddle/majel/test/ddim_test.cc b/paddle/majel/test/ddim_test.cc index 5c60451796..a5b8a7c4d2 100644 --- a/paddle/majel/test/ddim_test.cc +++ b/paddle/majel/test/ddim_test.cc @@ -4,7 +4,7 @@ #include #include "gtest/gtest.h" -#include "majel/ddim.h" +#include "paddle/majel/ddim.h" TEST(DDim, Equality) { // construct a DDim from an initialization list diff --git a/paddle/majel/test/dim_test.cu b/paddle/majel/test/dim_test.cu index ed7c34ffd1..a7d81e595b 100644 --- a/paddle/majel/test/dim_test.cu +++ b/paddle/majel/test/dim_test.cu @@ -1,7 +1,7 @@ #include #include -#include "majel/dim.h" +#include "paddle/majel/dim.h" #include "gtest/gtest.h" __global__ void test(majel::Dim<2>* o) { diff --git a/paddle/majel/test/place_test.cc b/paddle/majel/test/place_test.cc index c9a53802b2..6a099ae6b6 100644 --- a/paddle/majel/test/place_test.cc +++ b/paddle/majel/test/place_test.cc @@ -1,4 +1,4 @@ -#include "majel/place.h" +#include "paddle/majel/place.h" #include #include "gtest/gtest.h" From fc358816cbb32965c825746c634294ffba5187ba Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 19 May 2017 12:01:13 +0800 Subject: [PATCH 30/30] try adjusting building config --- paddle/majel/CMakeLists.txt | 3 ++- paddle/majel/dim.h | 2 +- paddle/majel/test/CMakeLists.txt | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt index 47f33dd832..cb8bece00e 100644 --- a/paddle/majel/CMakeLists.txt +++ b/paddle/majel/CMakeLists.txt @@ -1,4 +1,5 @@ -cc_library(majel SRCS place.cc ddim.cc) +cc_library(place SRCS place.cc) +cc_library(ddim SRCS ddim.cc) if(WITH_TESTING) add_subdirectory(test) diff --git a/paddle/majel/dim.h b/paddle/majel/dim.h index 5eff4c05cc..c4b0c6aea6 100644 --- a/paddle/majel/dim.h +++ b/paddle/majel/dim.h @@ -5,8 +5,8 @@ #include #include +#include "paddle/majel/detail/cuda_assert.h" #include "paddle/majel/detail/hostdevice.h" -#include "paddle/majel/detail/util.h" namespace majel { diff --git a/paddle/majel/test/CMakeLists.txt b/paddle/majel/test/CMakeLists.txt index 4245379952..9d632d568e 100644 --- a/paddle/majel/test/CMakeLists.txt +++ b/paddle/majel/test/CMakeLists.txt @@ -1,12 +1,12 @@ cc_test(place_test SRCS place_test.cc - DEPS majel) + DEPS place) cc_test(ddim_test SRCS ddim_test.cc - DEPS majel) + DEPS ddim) if(WITH_GPU) nv_test(cuda_test SRCS cuda_test.cu) - nv_test(dim_test SRCS dim_test.cu DEPS majel) + nv_test(dim_test SRCS dim_test.cu DEPS ddim) endif()