Helin Wang
8 years ago
parent
7af02682f7
commit
a868d01065
@ -1,11 +0,0 @@
|
||||
include_directories(${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
go_library(adder SRCS adder.go)
|
||||
|
||||
if (WITH_TESTING)
|
||||
cc_test(cgo_test
|
||||
SRCS
|
||||
cgo_test.cc
|
||||
DEPS
|
||||
adder)
|
||||
endif()
|
||||
@ -1,10 +0,0 @@
|
||||
package main
|
||||
|
||||
import "C"
|
||||
|
||||
//export GoAdder
|
||||
func GoAdder(x, y int) int {
|
||||
return x + y
|
||||
}
|
||||
|
||||
func main() {} // Required but ignored
|
||||
@ -1,31 +1,12 @@
|
||||
cmake_minimum_required(VERSION 3.0)
|
||||
|
||||
if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
|
||||
message("-- Found gtest (include: ${GTEST_INCLUDE_DIR}, library: ${GTEST_LIBRARIES})")
|
||||
else()
|
||||
# find cmake directory modules
|
||||
get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
|
||||
get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
|
||||
get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
|
||||
get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
|
||||
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
|
||||
project(cxx_go C Go)
|
||||
|
||||
# enable c++11
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
||||
include(golang)
|
||||
include(flags)
|
||||
|
||||
# enable gtest
|
||||
set(THIRD_PARTY_PATH ./third_party)
|
||||
set(WITH_TESTING ON)
|
||||
include(external/gtest)
|
||||
endif()
|
||||
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
|
||||
|
||||
project(cxx_go CXX C Go)
|
||||
|
||||
include(cmake/golang.cmake)
|
||||
include(cmake/flags.cmake)
|
||||
|
||||
ExternalGoProject_Add(pserver github.com/PaddlePaddle/Paddle/paddle/go/pserver)
|
||||
add_go_library(client STATIC pserver)
|
||||
add_go_library(client STATIC)
|
||||
add_subdirectory(test)
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
#include <iostream>
|
||||
#include "gtest/gtest.h"
|
||||
#include "libadder.h"
|
||||
|
||||
TEST(Cgo, Invoke) { EXPECT_EQ(GoAdder(30, 12), 42); }
|
||||
@ -0,0 +1,12 @@
|
||||
cmake_minimum_required(VERSION 3.0)
|
||||
|
||||
get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
|
||||
|
||||
project(cxx_go C Go)
|
||||
|
||||
include(golang)
|
||||
include(flags)
|
||||
|
||||
add_go_library(recordio STATIC)
|
||||
add_subdirectory(test)
|
||||
@ -0,0 +1,208 @@
|
||||
package main
|
||||
|
||||
/*
|
||||
#include <string.h>
|
||||
|
||||
typedef int reader;
|
||||
typedef int writer;
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unsafe"
|
||||
|
||||
"github.com/PaddlePaddle/Paddle/paddle/go/recordio"
|
||||
)
|
||||
|
||||
var nullPtr = unsafe.Pointer(uintptr(0))
|
||||
|
||||
type writer struct {
|
||||
w *recordio.Writer
|
||||
f *os.File
|
||||
}
|
||||
|
||||
type reader struct {
|
||||
buffer chan []byte
|
||||
cancel chan struct{}
|
||||
}
|
||||
|
||||
func read(paths []string, buffer chan<- []byte, cancel chan struct{}) {
|
||||
var curFile *os.File
|
||||
var curScanner *recordio.Scanner
|
||||
var pathIdx int
|
||||
|
||||
var nextFile func() bool
|
||||
nextFile = func() bool {
|
||||
if pathIdx >= len(paths) {
|
||||
return false
|
||||
}
|
||||
|
||||
path := paths[pathIdx]
|
||||
pathIdx++
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nextFile()
|
||||
}
|
||||
|
||||
idx, err := recordio.LoadIndex(f)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
return nextFile()
|
||||
}
|
||||
|
||||
curFile = f
|
||||
curScanner = recordio.NewScanner(f, idx, 0, -1)
|
||||
return true
|
||||
}
|
||||
|
||||
more := nextFile()
|
||||
if !more {
|
||||
close(buffer)
|
||||
return
|
||||
}
|
||||
|
||||
closeFile := func() {
|
||||
err := curFile.Close()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
curFile = nil
|
||||
}
|
||||
|
||||
for {
|
||||
for curScanner.Scan() {
|
||||
select {
|
||||
case buffer <- curScanner.Record():
|
||||
case <-cancel:
|
||||
close(buffer)
|
||||
closeFile()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := curScanner.Error(); err != nil && err != io.EOF {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
closeFile()
|
||||
more := nextFile()
|
||||
if !more {
|
||||
close(buffer)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//export paddle_new_writer
|
||||
func paddle_new_writer(path *C.char) C.writer {
|
||||
p := C.GoString(path)
|
||||
f, err := os.Create(p)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return -1
|
||||
}
|
||||
|
||||
w := recordio.NewWriter(f, -1, -1)
|
||||
writer := &writer{f: f, w: w}
|
||||
return addWriter(writer)
|
||||
}
|
||||
|
||||
func cArrayToSlice(p unsafe.Pointer, len int) []byte {
|
||||
if p == nullPtr {
|
||||
return nil
|
||||
}
|
||||
|
||||
// create a Go clice backed by a C array, reference:
|
||||
// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
|
||||
//
|
||||
// Go garbage collector will not interact with this data, need
|
||||
// to be freed from C side.
|
||||
return (*[1 << 30]byte)(p)[:len:len]
|
||||
}
|
||||
|
||||
//export paddle_writer_write
|
||||
func paddle_writer_write(writer C.writer, buf *C.uchar, size C.int) int {
|
||||
w := getWriter(writer)
|
||||
b := cArrayToSlice(unsafe.Pointer(buf), int(size))
|
||||
_, err := w.w.Write(b)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return -1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
//export paddle_writer_release
|
||||
func paddle_writer_release(writer C.writer) {
|
||||
w := removeWriter(writer)
|
||||
w.w.Close()
|
||||
w.f.Close()
|
||||
}
|
||||
|
||||
//export paddle_new_reader
|
||||
func paddle_new_reader(path *C.char, bufferSize C.int) C.reader {
|
||||
p := C.GoString(path)
|
||||
ss := strings.Split(p, ",")
|
||||
var paths []string
|
||||
for _, s := range ss {
|
||||
match, err := filepath.Glob(s)
|
||||
if err != nil {
|
||||
log.Printf("error applying glob to %s: %v\n", s, err)
|
||||
return -1
|
||||
}
|
||||
|
||||
paths = append(paths, match...)
|
||||
}
|
||||
|
||||
if len(paths) == 0 {
|
||||
log.Println("no valid path provided.", p)
|
||||
return -1
|
||||
}
|
||||
|
||||
buffer := make(chan []byte, int(bufferSize))
|
||||
cancel := make(chan struct{})
|
||||
r := &reader{buffer: buffer, cancel: cancel}
|
||||
go read(paths, buffer, cancel)
|
||||
return addReader(r)
|
||||
}
|
||||
|
||||
//export paddle_reader_next_item
|
||||
func paddle_reader_next_item(reader C.reader, size *C.int) *C.uchar {
|
||||
r := getReader(reader)
|
||||
buf, ok := <-r.buffer
|
||||
if !ok {
|
||||
// channel closed and empty, reached EOF.
|
||||
*size = -1
|
||||
return (*C.uchar)(nullPtr)
|
||||
}
|
||||
|
||||
if len(buf) == 0 {
|
||||
// empty item
|
||||
*size = 0
|
||||
return (*C.uchar)(nullPtr)
|
||||
}
|
||||
|
||||
ptr := C.malloc(C.size_t(len(buf)))
|
||||
C.memcpy(ptr, unsafe.Pointer(&buf[0]), C.size_t(len(buf)))
|
||||
*size = C.int(len(buf))
|
||||
return (*C.uchar)(ptr)
|
||||
}
|
||||
|
||||
//export paddle_reader_release
|
||||
func paddle_reader_release(reader C.reader) {
|
||||
r := removeReader(reader)
|
||||
close(r.cancel)
|
||||
}
|
||||
|
||||
func main() {} // Required but ignored
|
||||
@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
/*
|
||||
typedef int reader;
|
||||
typedef int writer;
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import "sync"
|
||||
|
||||
var mu sync.Mutex
|
||||
var handleMap = make(map[C.reader]*reader)
|
||||
var curHandle C.reader
|
||||
var writerMap = make(map[C.writer]*writer)
|
||||
var curWriterHandle C.writer
|
||||
|
||||
func addReader(r *reader) C.reader {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
reader := curHandle
|
||||
curHandle++
|
||||
handleMap[reader] = r
|
||||
return reader
|
||||
}
|
||||
|
||||
func getReader(reader C.reader) *reader {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return handleMap[reader]
|
||||
}
|
||||
|
||||
func removeReader(reader C.reader) *reader {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
r := handleMap[reader]
|
||||
delete(handleMap, reader)
|
||||
return r
|
||||
}
|
||||
|
||||
func addWriter(w *writer) C.writer {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
writer := curWriterHandle
|
||||
curWriterHandle++
|
||||
writerMap[writer] = w
|
||||
return writer
|
||||
}
|
||||
|
||||
func getWriter(writer C.writer) *writer {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return writerMap[writer]
|
||||
}
|
||||
|
||||
func removeWriter(writer C.writer) *writer {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
w := writerMap[writer]
|
||||
delete(writerMap, writer)
|
||||
return w
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
cmake_minimum_required(VERSION 3.0)
|
||||
|
||||
include_directories(${CMAKE_BINARY_DIR})
|
||||
|
||||
add_executable(recordio_test test.c)
|
||||
add_dependencies(recordio_test recordio)
|
||||
set (CMAKE_EXE_LINKER_FLAGS "-pthread")
|
||||
target_link_libraries(recordio_test ${CMAKE_BINARY_DIR}/librecordio.a)
|
||||
@ -0,0 +1,31 @@
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "librecordio.h"
|
||||
|
||||
void panic() {
|
||||
// TODO(helin): fix: gtest using cmake is not working, using this
|
||||
// hacky way for now.
|
||||
*(void*)0;
|
||||
}
|
||||
|
||||
int main() {
|
||||
writer w = paddle_new_writer("/tmp/test");
|
||||
paddle_writer_write(w, "hello", 6);
|
||||
paddle_writer_write(w, "hi", 3);
|
||||
paddle_writer_release(w);
|
||||
|
||||
reader r = paddle_new_reader("/tmp/test", 10);
|
||||
int size;
|
||||
unsigned char* item = paddle_reader_next_item(r, &size);
|
||||
if (!strcmp(item, "hello") || size != 6) {
|
||||
panic();
|
||||
}
|
||||
free(item);
|
||||
|
||||
item = paddle_reader_next_item(r, &size);
|
||||
if (!strcmp(item, "hi") || size != 2) {
|
||||
panic();
|
||||
}
|
||||
free(item);
|
||||
}
|
||||
Loading…
Reference in new issue