Paddle/go/master/client_test.go

// Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package master_test

import (
	"fmt"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/PaddlePaddle/Paddle/go/master"
	"github.com/PaddlePaddle/recordio"
)

// tool function for testing output goroutine ids
func goid() int {
	var buf [64]byte
	n := runtime.Stack(buf[:], false)
	idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0]
	id, err := strconv.Atoi(idField)
	if err != nil {
		panic(fmt.Sprintf("cannot get goroutine id: %v", err))
	}
	return id
}

func TestNextRecord(t *testing.T) {
	const (
		path  = "/tmp/master_client_TestFull"
		total = 50
	)
	l, err := net.Listen("tcp", ":0")
	if err != nil {
		panic(err)
	}

	ss := strings.Split(l.Addr().String(), ":")
	p, err := strconv.Atoi(ss[len(ss)-1])
	if err != nil {
		panic(err)
	}
	go func(l net.Listener) {
		s, err := master.NewService(&master.InMemStore{}, 1, time.Second*60, 1)
		if err != nil {
			panic(err)
		}

		server := rpc.NewServer()
		err = server.Register(s)
		if err != nil {
			panic(err)
		}

		mux := http.NewServeMux()
		mux.Handle(rpc.DefaultRPCPath, server)
		err = http.Serve(l, mux)
		if err != nil {
			panic(err)
		}
	}(l)

	f, err := os.Create(path)
	if err != nil {
		panic(err)
	}

	w := recordio.NewWriter(f, 1, -1)
	for i := 0; i < total; i++ {
		_, err = w.Write([]byte{byte(i)})
		if err != nil {
			panic(err)
		}
	}

	err = w.Close()
	if err != nil {
		panic(err)
	}

	err = f.Close()
	if err != nil {
		panic(err)
	}

	// start several client to test task fetching
	var wg sync.WaitGroup
	for i := 0; i < 4; i++ {
		wg.Add(1)
		// test for multiple concurrent clients
		go func() {
			defer wg.Done()
			// each go-routine needs a single client connection instance
			c, e := master.NewClient(master.WithAddr(fmt.Sprintf(":%d", p)), master.WithBuffer(1))
			if e != nil {
				t.Fatal(e)
			}
			e = c.SetDataset([]string{path})
			if e != nil {
				panic(e)
			}

			// test for n passes
			for pass := 0; pass < 10; pass++ {
				c.StartGetRecords(pass)

				received := make(map[byte]bool)
				taskid := 0
				for {
					r, e := c.NextRecord()
					if e != nil {
						// ErrorPassAfter will wait, else break for next pass
						if e.Error() == master.ErrPassBefore.Error() ||
							e.Error() == master.ErrNoMoreAvailable.Error() {
							break
						}
						t.Fatal(pass, taskid, "Read error:", e)
					}
					if len(r) != 1 {
						t.Fatal(pass, taskid, "Length should be 1.", r)
					}
					if received[r[0]] {
						t.Fatal(pass, taskid, "Received duplicate.", received, r)
					}
					taskid++
					received[r[0]] = true
				}
			}
		}()
	}
	wg.Wait()
}