Merge branch 'develop' of https://github.com/baidu/Paddle into ImageExpandFunction
hedaoyuan
8 years ago
commit
69271c92d5
@ -0,0 +1,21 @@
|
|||||||
|
# Design Doc: Remote Parameter Updater for Cluster Train
|
||||||
|
|
||||||
|
For an overview of distribute training, please refer to [distributed training design doc](README.md). In this design doc, we will discuss the parameter updater that will use parameter server cclient [The Client Library of Parameter Server Design Doc](pserver_client.md) to manage and update parameters.
|
||||||
|
|
||||||
|
## Parameter Updater
|
||||||
|
|
||||||
|
Parameter Updater is used by trainer to manage and update parameter, there are mainly two kind of parameter updater: local and remote, since this design is for cluster train, we will only discuss remote parameter updater here.
|
||||||
|
|
||||||
|
### Remote Parameter Updater
|
||||||
|
|
||||||
|
Remote Parameter Updater manage parameters through remote parameter server with the client that communicate with pserver([The Client Library of Parameter Server Design Doc](pserver_client.md))
|
||||||
|
|
||||||
|
In PaddlePaddle Python V2 API, trainer is implemented in python, and the trainer will hold a instance of parameter updater and call it's functions directly. In this design, we will also expose the api of RemoteParameterUpdater to python with swig.
|
||||||
|
|
||||||
|
#### Sparse Remote Parameter Updater
|
||||||
|
|
||||||
|
Since we will only implement dense parameter management new, the mechanism for sparse parameter will be discussed in next stage.
|
||||||
|
|
||||||
|
### Interface Design
|
||||||
|
|
||||||
|
TBD
|
||||||
@ -1,2 +1,7 @@
|
|||||||
RNN Models
|
RNN Models
|
||||||
==========
|
==========
|
||||||
|
|
||||||
|
.. toctree::
|
||||||
|
:maxdepth: 1
|
||||||
|
|
||||||
|
rnn_config_en.rst
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,21 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.0)
|
||||||
|
|
||||||
|
get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
|
||||||
|
get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
|
||||||
|
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
|
||||||
|
|
||||||
|
project(cxx_go C Go)
|
||||||
|
|
||||||
|
include(golang)
|
||||||
|
include(flags)
|
||||||
|
|
||||||
|
set(MASTER_LIB_NAME "paddle_master")
|
||||||
|
go_library(${MASTER_LIB_NAME} SHARED)
|
||||||
|
|
||||||
|
if(PROJ_ROOT)
|
||||||
|
add_custom_command(OUTPUT ${PROJ_ROOT}/python/paddle/v2/master/lib${MASTER_LIB_NAME}.so
|
||||||
|
COMMAND rm ${CMAKE_CURRENT_BINARY_DIR}/lib${MASTER_LIB_NAME}.h
|
||||||
|
COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/lib${MASTER_LIB_NAME}.so ${PROJ_ROOT}/python/paddle/v2/master/
|
||||||
|
DEPENDS ${MASTER_LIB_NAME})
|
||||||
|
add_custom_target(paddle_master_shared ALL DEPENDS ${PROJ_ROOT}/python/paddle/v2/master/lib${MASTER_LIB_NAME}.so)
|
||||||
|
endif(PROJ_ROOT)
|
||||||
@ -0,0 +1,110 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
/*
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
|
#define PADDLE_MASTER_OK 0
|
||||||
|
#define PADDLE_MASTER_ERROR -1
|
||||||
|
|
||||||
|
typedef int paddle_master_client;
|
||||||
|
*/
|
||||||
|
import "C"
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"github.com/PaddlePaddle/Paddle/go/master"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
var nullPtr = unsafe.Pointer(uintptr(0))
|
||||||
|
var mu sync.Mutex
|
||||||
|
var handleMap = make(map[C.paddle_master_client]*master.Client)
|
||||||
|
var curHandle C.paddle_master_client
|
||||||
|
|
||||||
|
func add(c *master.Client) C.paddle_master_client {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
client := curHandle
|
||||||
|
curHandle++
|
||||||
|
handleMap[client] = c
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
func get(client C.paddle_master_client) *master.Client {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return handleMap[client]
|
||||||
|
}
|
||||||
|
|
||||||
|
func remove(client C.paddle_master_client) *master.Client {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
h := handleMap[client]
|
||||||
|
delete(handleMap, client)
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
type addresser string
|
||||||
|
|
||||||
|
func (a addresser) Address() string {
|
||||||
|
return string(a)
|
||||||
|
}
|
||||||
|
|
||||||
|
//export paddle_new_master_client
|
||||||
|
func paddle_new_master_client(addr *C.char, bufSize int) C.paddle_master_client {
|
||||||
|
a := C.GoString(addr)
|
||||||
|
c := master.NewClient(addresser(a), bufSize)
|
||||||
|
return add(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
//export paddle_release_master_client
|
||||||
|
func paddle_release_master_client(client C.paddle_master_client) {
|
||||||
|
remove(client)
|
||||||
|
}
|
||||||
|
|
||||||
|
//export paddle_set_dataset
|
||||||
|
func paddle_set_dataset(client C.paddle_master_client, path **C.char, size C.int) C.int {
|
||||||
|
c := get(client)
|
||||||
|
var paths []string
|
||||||
|
for i := 0; i < int(size); i++ {
|
||||||
|
ptr := (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(path)) + uintptr(i)*unsafe.Sizeof(*path)))
|
||||||
|
str := C.GoString(*ptr)
|
||||||
|
paths = append(paths, str)
|
||||||
|
}
|
||||||
|
err := c.SetDataset(paths)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorln(err)
|
||||||
|
return C.PADDLE_MASTER_ERROR
|
||||||
|
}
|
||||||
|
|
||||||
|
return C.PADDLE_MASTER_OK
|
||||||
|
}
|
||||||
|
|
||||||
|
//export paddle_next_record
|
||||||
|
func paddle_next_record(client C.paddle_master_client, record **C.uchar) C.int {
|
||||||
|
c := get(client)
|
||||||
|
r := c.NextRecord()
|
||||||
|
if len(r) == 0 {
|
||||||
|
*record = (*C.uchar)(nullPtr)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
size := C.size_t(len(r))
|
||||||
|
*record = (*C.uchar)(C.malloc(size))
|
||||||
|
C.memcpy(unsafe.Pointer(*record), unsafe.Pointer(&r[0]), size)
|
||||||
|
return C.int(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
//export mem_free
|
||||||
|
func mem_free(p unsafe.Pointer) {
|
||||||
|
// "free" may be a better name for this function, but doing so
|
||||||
|
// will cause calling any function of this library from Python
|
||||||
|
// ctypes hanging.
|
||||||
|
C.free(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {}
|
||||||
@ -0,0 +1,137 @@
|
|||||||
|
package master
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/PaddlePaddle/Paddle/go/connection"
|
||||||
|
"github.com/PaddlePaddle/recordio"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Addresser provide the address of the master server.
|
||||||
|
type Addresser interface {
|
||||||
|
Address() string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Client is the client of the master server.
|
||||||
|
type Client struct {
|
||||||
|
conn *connection.Conn
|
||||||
|
ch chan []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewClient creates a new Client.
|
||||||
|
//
|
||||||
|
// bufSize is the record buffer size. NextRecord will read from this
|
||||||
|
// buffer.
|
||||||
|
func NewClient(addr Addresser, bufSize int) *Client {
|
||||||
|
c := &Client{}
|
||||||
|
c.conn = connection.New()
|
||||||
|
c.ch = make(chan []byte, bufSize)
|
||||||
|
go c.monitorMaster(addr)
|
||||||
|
go c.getRecords()
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) getRecords() {
|
||||||
|
for {
|
||||||
|
t, err := c.getTask()
|
||||||
|
if err != nil {
|
||||||
|
// TODO(helin): wait before move on with next
|
||||||
|
// getTask call.
|
||||||
|
log.Errorln(err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, chunk := range t.Chunks {
|
||||||
|
f, err := os.Open(chunk.Path)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorln(err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
s := recordio.NewRangeScanner(f, &chunk.Index, -1, -1)
|
||||||
|
for s.Scan() {
|
||||||
|
c.ch <- s.Record()
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.Err() != nil {
|
||||||
|
log.Errorln(err, chunk.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = f.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorln(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// We treat a task as finished whenever the last data
|
||||||
|
// instance of the task is read. This is not exactly
|
||||||
|
// correct, but a reasonable approximation.
|
||||||
|
c.taskFinished(t.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) monitorMaster(addr Addresser) {
|
||||||
|
lastMaster := ""
|
||||||
|
monitor := func() {
|
||||||
|
// get the lastest address of the master server,
|
||||||
|
// connect to the new address once address changed.
|
||||||
|
curMaster := addr.Address()
|
||||||
|
if curMaster != lastMaster {
|
||||||
|
if curMaster == "" {
|
||||||
|
err := c.conn.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorln(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
err := c.conn.Connect(curMaster)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorln(err)
|
||||||
|
|
||||||
|
// connect to addr failed, set
|
||||||
|
// to last known addr in order
|
||||||
|
// to retry next time.
|
||||||
|
curMaster = lastMaster
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lastMaster = curMaster
|
||||||
|
}
|
||||||
|
|
||||||
|
monitor()
|
||||||
|
ticker := time.NewTicker(10 * time.Second)
|
||||||
|
for _ = range ticker.C {
|
||||||
|
monitor()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDataset set dataset for the master server to dispatch.
|
||||||
|
//
|
||||||
|
// SetDataset can be call multiple times from different nodes. But
|
||||||
|
// only the first call will be honored.
|
||||||
|
func (c *Client) SetDataset(globPaths []string) error {
|
||||||
|
return c.conn.Call("Service.SetDataset", globPaths, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getTask gets a new task from the master server.
|
||||||
|
func (c *Client) getTask() (Task, error) {
|
||||||
|
var t Task
|
||||||
|
err := c.conn.Call("Service.GetTask", 0, &t)
|
||||||
|
return t, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// TaskFinished tells the master server a task is finished.
|
||||||
|
func (c *Client) taskFinished(taskID int) error {
|
||||||
|
return c.conn.Call("Service.TaskFinished", taskID, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NextRecord returns next record in the dataset.
|
||||||
|
//
|
||||||
|
// NextRecord will block until the next record is available. It is
|
||||||
|
// thread-safe.
|
||||||
|
func (c *Client) NextRecord() []byte {
|
||||||
|
return <-c.ch
|
||||||
|
}
|
||||||
@ -0,0 +1,121 @@
|
|||||||
|
package master
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/rpc"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/PaddlePaddle/Paddle/go/connection"
|
||||||
|
"github.com/PaddlePaddle/recordio"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
totalTask = 20
|
||||||
|
chunkPerTask = 10
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
log.SetLevel(log.ErrorLevel)
|
||||||
|
}
|
||||||
|
|
||||||
|
type TestAddresser string
|
||||||
|
|
||||||
|
func (a TestAddresser) Address() string {
|
||||||
|
return string(a)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetFinishTask(t *testing.T) {
|
||||||
|
const path = "/tmp/master_client_test_0"
|
||||||
|
|
||||||
|
l, err := net.Listen("tcp", ":0")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ss := strings.Split(l.Addr().String(), ":")
|
||||||
|
p, err := strconv.Atoi(ss[len(ss)-1])
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
go func(l net.Listener) {
|
||||||
|
s := NewService(chunkPerTask, time.Second, 1)
|
||||||
|
server := rpc.NewServer()
|
||||||
|
err := server.Register(s)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.Handle(rpc.DefaultRPCPath, server)
|
||||||
|
err = http.Serve(l, mux)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}(l)
|
||||||
|
|
||||||
|
f, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < totalTask*chunkPerTask; i++ {
|
||||||
|
w := recordio.NewWriter(f, -1, -1)
|
||||||
|
w.Write(nil)
|
||||||
|
// call Close to force RecordIO writing a chunk.
|
||||||
|
w.Close()
|
||||||
|
}
|
||||||
|
f.Close()
|
||||||
|
|
||||||
|
// Manually intialize client to avoid calling c.getRecords()
|
||||||
|
c := &Client{}
|
||||||
|
c.conn = connection.New()
|
||||||
|
go c.monitorMaster(TestAddresser(fmt.Sprintf(":%d", p)))
|
||||||
|
c.SetDataset([]string{path})
|
||||||
|
|
||||||
|
checkOnePass := func(i int) {
|
||||||
|
var tasks []Task
|
||||||
|
for idx := 0; idx < totalTask; idx++ {
|
||||||
|
task, err := c.getTask()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Error: %v, pass: %d\n", err, i)
|
||||||
|
}
|
||||||
|
tasks = append(tasks, task)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = c.getTask()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Should get error, pass: %d\n", i)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.taskFinished(tasks[0].ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Error: %v, pass: %d\n", err, i)
|
||||||
|
}
|
||||||
|
tasks = tasks[1:]
|
||||||
|
task, err := c.getTask()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tasks = append(tasks, task)
|
||||||
|
|
||||||
|
for _, task := range tasks {
|
||||||
|
err = c.taskFinished(task.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Error: %v, pass: %d\n", err, i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
checkOnePass(i)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
package master_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/rpc"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/PaddlePaddle/Paddle/go/master"
|
||||||
|
"github.com/PaddlePaddle/recordio"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNextRecord(t *testing.T) {
|
||||||
|
const (
|
||||||
|
path = "/tmp/master_client_TestFull"
|
||||||
|
total = 50
|
||||||
|
)
|
||||||
|
|
||||||
|
l, err := net.Listen("tcp", ":0")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ss := strings.Split(l.Addr().String(), ":")
|
||||||
|
p, err := strconv.Atoi(ss[len(ss)-1])
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
go func(l net.Listener) {
|
||||||
|
s := master.NewService(10, time.Second, 1)
|
||||||
|
server := rpc.NewServer()
|
||||||
|
err := server.Register(s)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.Handle(rpc.DefaultRPCPath, server)
|
||||||
|
err = http.Serve(l, mux)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}(l)
|
||||||
|
|
||||||
|
f, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := recordio.NewWriter(f, -1, -1)
|
||||||
|
for i := 0; i < total; i++ {
|
||||||
|
w.Write([]byte{byte(i)})
|
||||||
|
}
|
||||||
|
w.Close()
|
||||||
|
f.Close()
|
||||||
|
|
||||||
|
c := master.NewClient(master.TestAddresser(fmt.Sprintf(":%d", p)), 10)
|
||||||
|
c.SetDataset([]string{path})
|
||||||
|
|
||||||
|
for pass := 0; pass < 50; pass++ {
|
||||||
|
received := make(map[byte]bool)
|
||||||
|
for i := 0; i < total; i++ {
|
||||||
|
r := c.NextRecord()
|
||||||
|
if len(r) != 1 {
|
||||||
|
t.Fatal("Length should be 1.", r)
|
||||||
|
}
|
||||||
|
if received[r[0]] {
|
||||||
|
t.Fatal("Received duplicate.", received, r)
|
||||||
|
}
|
||||||
|
received[r[0]] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,11 +1,22 @@
|
|||||||
cmake_minimum_required(VERSION 3.0)
|
cmake_minimum_required(VERSION 3.0)
|
||||||
|
|
||||||
include_directories(${CMAKE_BINARY_DIR})
|
|
||||||
|
|
||||||
add_executable(main main.c)
|
add_executable(main main.c)
|
||||||
add_dependencies(main client)
|
add_dependencies(main paddle_pserver_cclient)
|
||||||
|
add_executable(test_cclient test_cclient.c)
|
||||||
|
add_dependencies(test_cclient paddle_pserver_cclient)
|
||||||
|
|
||||||
if(APPLE)
|
if(APPLE)
|
||||||
set(CMAKE_EXE_LINKER_FLAGS "-framework CoreFoundation -framework Security")
|
set(CMAKE_EXE_LINKER_FLAGS "-framework CoreFoundation -framework Security")
|
||||||
|
else()
|
||||||
|
set(CMAKE_EXE_LINKER_FLAGS "-pthread")
|
||||||
endif()
|
endif()
|
||||||
target_link_libraries(main ${CMAKE_BINARY_DIR}/libclient.a)
|
|
||||||
|
if(PROJ_ROOT)
|
||||||
|
include_directories(${CMAKE_CURRENT_BINARY_DIR}/..)
|
||||||
|
target_link_libraries(main ${CMAKE_CURRENT_BINARY_DIR}/../libpaddle_pserver_cclient.a pthread)
|
||||||
|
target_link_libraries(test_cclient ${CMAKE_CURRENT_BINARY_DIR}/../libpaddle_pserver_cclient.a pthread)
|
||||||
|
else(PROJ_ROOT)
|
||||||
|
include_directories(${CMAKE_BINARY_DIR})
|
||||||
|
target_link_libraries(main ${CMAKE_BINARY_DIR}/libpaddle_pserver_cclient.a pthread)
|
||||||
|
target_link_libraries(test_cclient ${CMAKE_BINARY_DIR}/libpaddle_pserver_cclient.a pthread)
|
||||||
|
endif(PROJ_ROOT)
|
||||||
|
|||||||
@ -0,0 +1,117 @@
|
|||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
#include "libpaddle_pserver_cclient.h"
|
||||||
|
|
||||||
|
typedef float real;
|
||||||
|
|
||||||
|
void fail() {
|
||||||
|
// TODO(helin): fix: gtest using cmake is not working, using this
|
||||||
|
// hacky way for now.
|
||||||
|
printf("test failed.\n");
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
void print_parameter(paddle_gradient* param) {
|
||||||
|
if (param == NULL) {
|
||||||
|
printf("param is NULL!!\n");
|
||||||
|
} else {
|
||||||
|
printf("==== parameter ====\n");
|
||||||
|
printf("name: %s\n", param->name);
|
||||||
|
printf("content_len: %d\n", param->content_len);
|
||||||
|
printf("content_type: %d\n", param->element_type);
|
||||||
|
int i;
|
||||||
|
for (i = 0; i < param->content_len / (int)sizeof(real); ++i) {
|
||||||
|
printf("%f ", ((float*)param->content)[i]);
|
||||||
|
}
|
||||||
|
printf("\n\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int main() {
|
||||||
|
char addr[] = "localhost:3000";
|
||||||
|
paddle_pserver_client c = paddle_new_pserver_client(addr, 1);
|
||||||
|
|
||||||
|
char* names[] = {"param_a", "param_b"};
|
||||||
|
|
||||||
|
retry:
|
||||||
|
printf("init parameter to pserver:\n");
|
||||||
|
|
||||||
|
real param_content1[] = {0.1, 0.2, 0.3};
|
||||||
|
real param_content2[] = {0.4, 0.5, 0.6};
|
||||||
|
paddle_parameter** params =
|
||||||
|
(paddle_parameter**)malloc(sizeof(paddle_parameter*) * 2);
|
||||||
|
params[0] = (paddle_parameter*)malloc(sizeof(paddle_parameter));
|
||||||
|
params[0]->name = names[0];
|
||||||
|
params[0]->content = (unsigned char*)param_content1;
|
||||||
|
params[0]->content_len = 3 * sizeof(real);
|
||||||
|
params[0]->element_type = PADDLE_ELEMENT_TYPE_FLOAT32;
|
||||||
|
|
||||||
|
params[1] = (paddle_parameter*)malloc(sizeof(paddle_parameter));
|
||||||
|
params[1]->name = names[1];
|
||||||
|
params[1]->content = (unsigned char*)param_content2;
|
||||||
|
params[1]->content_len = 3 * sizeof(real);
|
||||||
|
params[1]->element_type = PADDLE_ELEMENT_TYPE_INT32;
|
||||||
|
|
||||||
|
if (paddle_begin_init_params(c)) {
|
||||||
|
if (paddle_init_param(c, *params[0], NULL, 0) != 0) {
|
||||||
|
goto retry;
|
||||||
|
}
|
||||||
|
if (paddle_init_param(c, *params[1], NULL, 0) != 0) {
|
||||||
|
goto retry;
|
||||||
|
}
|
||||||
|
if (paddle_finish_init_params(c) != 0) {
|
||||||
|
goto retry;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("get inited parameters from pserver:\n");
|
||||||
|
// get parameters again by reusing the allocated parameter buffers.
|
||||||
|
if (paddle_get_params(c, params, 2) != 0) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
print_parameter(params[0]);
|
||||||
|
print_parameter(params[1]);
|
||||||
|
|
||||||
|
printf("send gradient to pserver:\n");
|
||||||
|
real gradient_content1[] = {0.01, 0.02, 0.03};
|
||||||
|
real gradinet_content2[] = {0.04, 0.05, 0.06};
|
||||||
|
|
||||||
|
paddle_gradient** grads =
|
||||||
|
(paddle_gradient**)malloc(sizeof(paddle_gradient*) * 2);
|
||||||
|
grads[0] = (paddle_gradient*)malloc(sizeof(paddle_gradient));
|
||||||
|
grads[0]->name = names[0];
|
||||||
|
grads[0]->content = (unsigned char*)gradient_content1;
|
||||||
|
grads[0]->content_len = 3 * sizeof(real);
|
||||||
|
grads[0]->element_type = PADDLE_ELEMENT_TYPE_FLOAT32;
|
||||||
|
|
||||||
|
grads[1] = (paddle_gradient*)malloc(sizeof(paddle_gradient));
|
||||||
|
grads[1]->name = names[1];
|
||||||
|
grads[1]->content = (unsigned char*)gradinet_content2;
|
||||||
|
grads[1]->content_len = 3 * sizeof(real);
|
||||||
|
grads[1]->element_type = PADDLE_ELEMENT_TYPE_INT32;
|
||||||
|
|
||||||
|
printf("print gradient sent to pserver:\n");
|
||||||
|
print_parameter(grads[0]);
|
||||||
|
print_parameter(grads[1]);
|
||||||
|
|
||||||
|
if (paddle_send_grads(c, grads, 2) != 0) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("get updated parameters from pserver:\n");
|
||||||
|
// get parameters again by reusing the allocated parameter buffers.
|
||||||
|
if (paddle_get_params(c, params, 2) != 0) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
print_parameter(params[0]);
|
||||||
|
print_parameter(params[1]);
|
||||||
|
|
||||||
|
if (paddle_save_model(c, "/tmp/") != 0) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@ -0,0 +1,131 @@
|
|||||||
|
import paddle.v2 as paddle
|
||||||
|
import gzip
|
||||||
|
|
||||||
|
|
||||||
|
def softmax_regression(img):
|
||||||
|
predict = paddle.layer.fc(input=img,
|
||||||
|
size=10,
|
||||||
|
act=paddle.activation.Softmax())
|
||||||
|
return predict
|
||||||
|
|
||||||
|
|
||||||
|
def multilayer_perceptron(img):
|
||||||
|
# The first fully-connected layer
|
||||||
|
hidden1 = paddle.layer.fc(input=img, size=128, act=paddle.activation.Relu())
|
||||||
|
# The second fully-connected layer and the according activation function
|
||||||
|
hidden2 = paddle.layer.fc(input=hidden1,
|
||||||
|
size=64,
|
||||||
|
act=paddle.activation.Relu())
|
||||||
|
# The thrid fully-connected layer, note that the hidden size should be 10,
|
||||||
|
# which is the number of unique digits
|
||||||
|
predict = paddle.layer.fc(input=hidden2,
|
||||||
|
size=10,
|
||||||
|
act=paddle.activation.Softmax())
|
||||||
|
return predict
|
||||||
|
|
||||||
|
|
||||||
|
def convolutional_neural_network(img):
|
||||||
|
# first conv layer
|
||||||
|
conv_pool_1 = paddle.networks.simple_img_conv_pool(
|
||||||
|
input=img,
|
||||||
|
filter_size=5,
|
||||||
|
num_filters=20,
|
||||||
|
num_channel=1,
|
||||||
|
pool_size=2,
|
||||||
|
pool_stride=2,
|
||||||
|
act=paddle.activation.Tanh())
|
||||||
|
# second conv layer
|
||||||
|
conv_pool_2 = paddle.networks.simple_img_conv_pool(
|
||||||
|
input=conv_pool_1,
|
||||||
|
filter_size=5,
|
||||||
|
num_filters=50,
|
||||||
|
num_channel=20,
|
||||||
|
pool_size=2,
|
||||||
|
pool_stride=2,
|
||||||
|
act=paddle.activation.Tanh())
|
||||||
|
# The first fully-connected layer
|
||||||
|
fc1 = paddle.layer.fc(input=conv_pool_2,
|
||||||
|
size=128,
|
||||||
|
act=paddle.activation.Tanh())
|
||||||
|
# The softmax layer, note that the hidden size should be 10,
|
||||||
|
# which is the number of unique digits
|
||||||
|
predict = paddle.layer.fc(input=fc1,
|
||||||
|
size=10,
|
||||||
|
act=paddle.activation.Softmax())
|
||||||
|
return predict
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
paddle.init(use_gpu=False, trainer_count=1)
|
||||||
|
|
||||||
|
# define network topology
|
||||||
|
images = paddle.layer.data(
|
||||||
|
name='pixel', type=paddle.data_type.dense_vector(784))
|
||||||
|
label = paddle.layer.data(
|
||||||
|
name='label', type=paddle.data_type.integer_value(10))
|
||||||
|
|
||||||
|
# Here we can build the prediction network in different ways. Please
|
||||||
|
# choose one by uncomment corresponding line.
|
||||||
|
predict = softmax_regression(images)
|
||||||
|
#predict = multilayer_perceptron(images)
|
||||||
|
#predict = convolutional_neural_network(images)
|
||||||
|
|
||||||
|
cost = paddle.layer.classification_cost(input=predict, label=label)
|
||||||
|
parameters = paddle.parameters.create(cost)
|
||||||
|
|
||||||
|
optimizer = paddle.optimizer.Momentum(
|
||||||
|
learning_rate=0.1 / 128.0,
|
||||||
|
momentum=0.9,
|
||||||
|
regularization=paddle.optimizer.L2Regularization(rate=0.0005 * 128))
|
||||||
|
|
||||||
|
trainer = paddle.trainer.SGD(cost=cost,
|
||||||
|
parameters=parameters,
|
||||||
|
update_equation=optimizer,
|
||||||
|
is_local=False,
|
||||||
|
pserver_spec="localhost:3000")
|
||||||
|
|
||||||
|
lists = []
|
||||||
|
|
||||||
|
def event_handler(event):
|
||||||
|
if isinstance(event, paddle.event.EndIteration):
|
||||||
|
if event.batch_id % 1000 == 0:
|
||||||
|
print "Pass %d, Batch %d, Cost %f, %s" % (
|
||||||
|
event.pass_id, event.batch_id, event.cost, event.metrics)
|
||||||
|
|
||||||
|
elif isinstance(event, paddle.event.EndPass):
|
||||||
|
result = trainer.test(reader=paddle.batch(
|
||||||
|
paddle.dataset.mnist.test(), batch_size=128))
|
||||||
|
print "Test with Pass %d, Cost %f, %s\n" % (
|
||||||
|
event.pass_id, result.cost, result.metrics)
|
||||||
|
lists.append((event.pass_id, result.cost,
|
||||||
|
result.metrics['classification_error_evaluator']))
|
||||||
|
|
||||||
|
trainer.train(
|
||||||
|
reader=paddle.batch(
|
||||||
|
paddle.reader.shuffle(
|
||||||
|
paddle.dataset.mnist.train(), buf_size=8192),
|
||||||
|
batch_size=128),
|
||||||
|
event_handler=event_handler,
|
||||||
|
num_passes=100)
|
||||||
|
|
||||||
|
# find the best pass
|
||||||
|
best = sorted(lists, key=lambda list: float(list[1]))[0]
|
||||||
|
print 'Best pass is %s, testing Avgcost is %s' % (best[0], best[1])
|
||||||
|
print 'The classification accuracy is %.2f%%' % (100 - float(best[2]) * 100)
|
||||||
|
|
||||||
|
test_creator = paddle.dataset.mnist.test()
|
||||||
|
test_data = []
|
||||||
|
for item in test_creator():
|
||||||
|
test_data.append((item[0], ))
|
||||||
|
if len(test_data) == 100:
|
||||||
|
break
|
||||||
|
|
||||||
|
# output is a softmax layer. It returns probabilities.
|
||||||
|
# Shape should be (100, 10)
|
||||||
|
probs = paddle.infer(
|
||||||
|
output_layer=predict, parameters=parameters, input=test_data)
|
||||||
|
print probs.shape
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -0,0 +1,60 @@
|
|||||||
|
import paddle.v2 as paddle
|
||||||
|
import paddle.v2.dataset.uci_housing as uci_housing
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# init
|
||||||
|
paddle.init(use_gpu=False, trainer_count=1)
|
||||||
|
|
||||||
|
# network config
|
||||||
|
x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
|
||||||
|
y_predict = paddle.layer.fc(input=x,
|
||||||
|
param_attr=paddle.attr.Param(name='w'),
|
||||||
|
size=1,
|
||||||
|
act=paddle.activation.Linear(),
|
||||||
|
bias_attr=paddle.attr.Param(name='b'))
|
||||||
|
y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
|
||||||
|
cost = paddle.layer.mse_cost(input=y_predict, label=y)
|
||||||
|
|
||||||
|
# create parameters
|
||||||
|
parameters = paddle.parameters.create(cost)
|
||||||
|
|
||||||
|
# create optimizer
|
||||||
|
optimizer = paddle.optimizer.Momentum(momentum=0)
|
||||||
|
|
||||||
|
trainer = paddle.trainer.SGD(cost=cost,
|
||||||
|
parameters=parameters,
|
||||||
|
update_equation=optimizer,
|
||||||
|
is_local=False,
|
||||||
|
pserver_spec="localhost:3000")
|
||||||
|
|
||||||
|
# event_handler to print training and testing info
|
||||||
|
def event_handler(event):
|
||||||
|
if isinstance(event, paddle.event.EndIteration):
|
||||||
|
if event.batch_id % 100 == 0:
|
||||||
|
print "Pass %d, Batch %d, Cost %f" % (
|
||||||
|
event.pass_id, event.batch_id, event.cost)
|
||||||
|
|
||||||
|
if isinstance(event, paddle.event.EndPass):
|
||||||
|
if (event.pass_id + 1) % 10 == 0:
|
||||||
|
result = trainer.test(
|
||||||
|
reader=paddle.batch(
|
||||||
|
uci_housing.test(), batch_size=2),
|
||||||
|
feeding={'x': 0,
|
||||||
|
'y': 1})
|
||||||
|
print "Test %d, %.2f" % (event.pass_id, result.cost)
|
||||||
|
|
||||||
|
# training
|
||||||
|
trainer.train(
|
||||||
|
reader=paddle.batch(
|
||||||
|
paddle.reader.shuffle(
|
||||||
|
uci_housing.train(), buf_size=500),
|
||||||
|
batch_size=2),
|
||||||
|
feeding={'x': 0,
|
||||||
|
'y': 1},
|
||||||
|
event_handler=event_handler,
|
||||||
|
num_passes=30)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue