Add http_middleware to transfer (#402)

* validate ui query, add aggrFun support for resample

* add http_middleware to transfer

src/modules/transfer/config/config.go

@@ -43,8 +43,9 @@ type LoggerSection struct {
 }
 
 type HTTPSection struct {
-	Enabled bool   `yaml:"enabled"`
-	Access  string `yaml:"access"`
+	Mode         string `yaml:"mode"`
+	CookieName   string `yaml:"cookieName"`
+	CookieDomain string `yaml:"cookieDomain"`
 }
 
 type RPCSection struct {

src/modules/transfer/http/health_router.go

@@ -1,4 +1,4 @@
-package routes
+package http
 
 import (
 	"fmt"

src/modules/transfer/http/http_middleware.go

@@ -0,0 +1,124 @@
+package http
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/logger"
+	"github.com/toolkits/pkg/slice"
+
+	"github.com/didi/nightingale/src/common/address"
+	"github.com/didi/nightingale/src/models"
+	"github.com/didi/nightingale/src/modules/rdb/config"
+)
+
+func shouldBeLogin() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Set("username", mustUsername(c))
+		c.Next()
+	}
+}
+
+func shouldBeRoot() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		username := mustUsername(c)
+
+		user, err := models.UserGet("username=?", username)
+		dangerous(err)
+
+		if user.IsRoot != 1 {
+			bomb("forbidden")
+		}
+
+		c.Set("username", username)
+		c.Set("user", user)
+		c.Next()
+	}
+}
+
+func shouldBeService() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		remoteAddr := c.Request.RemoteAddr
+		idx := strings.LastIndex(remoteAddr, ":")
+		ip := ""
+		if idx > 0 {
+			ip = remoteAddr[0:idx]
+		}
+
+		if ip == "127.0.0.1" {
+			c.Next()
+			return
+		}
+
+		if ip != "" && slice.ContainsString(address.GetAddresses("rdb"), ip) {
+			c.Next()
+			return
+		}
+
+		token := c.GetHeader("X-Srv-Token")
+		if token == "" {
+			c.AbortWithError(http.StatusForbidden, fmt.Errorf("X-Srv-Token is blank"))
+			return
+		}
+
+		if !slice.ContainsString(config.Config.Tokens, token) {
+			c.AbortWithError(http.StatusForbidden, fmt.Errorf("X-Srv-Token[%s] invalid", token))
+			return
+		}
+
+		c.Next()
+	}
+}
+
+func mustUsername(c *gin.Context) string {
+	username := cookieUsername(c)
+	if username == "" {
+		username = headerUsername(c)
+	}
+
+	if username == "" {
+		bomb("unauthorized")
+	}
+
+	return username
+}
+
+func cookieUsername(c *gin.Context) string {
+	return models.UsernameByUUID(readCookieUser(c))
+}
+
+func headerUsername(c *gin.Context) string {
+	token := c.GetHeader("X-User-Token")
+	if token == "" {
+		return ""
+	}
+
+	ut, err := models.UserTokenGet("token=?", token)
+	if err != nil {
+		logger.Warningf("UserTokenGet[%s] fail: %v", token, err)
+		return ""
+	}
+
+	if ut == nil {
+		return ""
+	}
+
+	return ut.Username
+}
+
+// ------------
+
+func readCookieUser(c *gin.Context) string {
+	uuid, err := c.Cookie(config.Config.HTTP.CookieName)
+	if err != nil {
+		return ""
+	}
+
+	return uuid
+}
+
+func writeCookieUser(c *gin.Context, uuid string) {
+	c.SetCookie(config.Config.HTTP.CookieName, uuid, 3600*24, "/", config.Config.HTTP.CookieDomain, false, true)
+}

src/modules/transfer/http/http_server.go

@@ -0,0 +1,70 @@
+package http
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+
+	"github.com/didi/nightingale/src/common/address"
+	"github.com/didi/nightingale/src/common/middleware"
+	"github.com/didi/nightingale/src/modules/transfer/config"
+)
+
+var srv = &http.Server{
+	ReadTimeout:    10 * time.Second,
+	WriteTimeout:   10 * time.Second,
+	MaxHeaderBytes: 1 << 20,
+}
+
+var skipPaths = []string{"/api/rdb/auth/login"}
+
+func Start() {
+	c := config.Config
+
+	loggerMid := middleware.LoggerWithConfig(middleware.LoggerConfig{SkipPaths: skipPaths})
+	recoveryMid := middleware.Recovery()
+
+	if strings.ToLower(c.HTTP.Mode) == "release" {
+		gin.SetMode(gin.ReleaseMode)
+		middleware.DisableConsoleColor()
+	}
+
+	r := gin.New()
+	r.Use(loggerMid, recoveryMid)
+
+	Config(r)
+
+	srv.Addr = address.GetHTTPListen("transfer")
+	srv.Handler = r
+
+	go func() {
+		fmt.Println("http.listening:", srv.Addr)
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			fmt.Printf("listening %s occur error: %s\n", srv.Addr, err)
+			os.Exit(3)
+		}
+	}()
+}
+
+// Shutdown http server
+func Shutdown() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(ctx); err != nil {
+		fmt.Println("cannot shutdown http server:", err)
+		os.Exit(2)
+	}
+
+	// catching ctx.Done(). timeout of 5 seconds.
+	select {
+	case <-ctx.Done():
+		fmt.Println("shutdown http server timeout of 5 seconds.")
+	default:
+		fmt.Println("http server stopped")
+	}
+}

src/modules/transfer/http/push_router.go

@@ -1,4 +1,4 @@
-package routes
+package http
 
 import (
 	"github.com/didi/nightingale/src/common/dataobj"

src/modules/transfer/http/query_router.go

@@ -0,0 +1,158 @@
+package http
+
+import (
+	"github.com/didi/nightingale/src/common/dataobj"
+	"github.com/didi/nightingale/src/modules/transfer/backend"
+	"github.com/didi/nightingale/src/toolkits/http/render"
+	"github.com/didi/nightingale/src/toolkits/stats"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/errors"
+	"github.com/toolkits/pkg/logger"
+)
+
+func QueryData(c *gin.Context) {
+	stats.Counter.Set("data.api.qp10s", 1)
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	var input []dataobj.QueryData
+	errors.Dangerous(c.ShouldBindJSON(&input))
+	resp := dataSource.QueryData(input)
+	render.Data(c, resp, nil)
+}
+
+func QueryDataForUI(c *gin.Context) {
+	stats.Counter.Set("data.ui.qp10s", 1)
+	var input dataobj.QueryDataForUI
+	var respData []*dataobj.QueryDataForUIResp
+
+	dangerous(c.ShouldBindJSON(&input))
+	start := input.Start
+	end := input.End
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+	resp := dataSource.QueryDataForUI(input)
+	for _, d := range resp {
+		data := &dataobj.QueryDataForUIResp{
+			Start:    d.Start,
+			End:      d.End,
+			Endpoint: d.Endpoint,
+			Nid:      d.Nid,
+			Counter:  d.Counter,
+			DsType:   d.DsType,
+			Step:     d.Step,
+			Values:   d.Values,
+		}
+		respData = append(respData, data)
+	}
+
+	if len(input.Comparisons) > 1 {
+		for i := 1; i < len(input.Comparisons); i++ {
+			comparison := input.Comparisons[i]
+			input.Start = start - comparison
+			input.End = end - comparison
+			res := dataSource.QueryDataForUI(input)
+			for _, d := range res {
+				for j := range d.Values {
+					d.Values[j].Timestamp += comparison
+				}
+
+				data := &dataobj.QueryDataForUIResp{
+					Start:      d.Start,
+					End:        d.End,
+					Endpoint:   d.Endpoint,
+					Nid:        d.Nid,
+					Counter:    d.Counter,
+					DsType:     d.DsType,
+					Step:       d.Step,
+					Values:     d.Values,
+					Comparison: comparison,
+				}
+				respData = append(respData, data)
+			}
+		}
+	}
+
+	render.Data(c, respData, nil)
+}
+
+func GetMetrics(c *gin.Context) {
+	stats.Counter.Set("metric.qp10s", 1)
+	recv := dataobj.EndpointsRecv{}
+	errors.Dangerous(c.ShouldBindJSON(&recv))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryMetrics(recv)
+
+	render.Data(c, resp, nil)
+}
+
+func GetTagPairs(c *gin.Context) {
+	stats.Counter.Set("tag.qp10s", 1)
+	recv := dataobj.EndpointMetricRecv{}
+	errors.Dangerous(c.ShouldBindJSON(&recv))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryTagPairs(recv)
+	render.Data(c, resp, nil)
+}
+
+func GetIndexByClude(c *gin.Context) {
+	stats.Counter.Set("xclude.qp10s", 1)
+	recvs := make([]dataobj.CludeRecv, 0)
+	errors.Dangerous(c.ShouldBindJSON(&recvs))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryIndexByClude(recvs)
+	render.Data(c, resp, nil)
+}
+
+func GetIndexByFullTags(c *gin.Context) {
+	stats.Counter.Set("counter.qp10s", 1)
+	recvs := make([]dataobj.IndexByFullTagsRecv, 0)
+	errors.Dangerous(c.ShouldBindJSON(&recvs))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryIndexByFullTags(recvs)
+	render.Data(c, &listResp{List: resp, Count: len(resp)}, nil)
+}
+
+type listResp struct {
+	List  interface{} `json:"list"`
+	Count int         `json:"count"`
+}

src/modules/transfer/http/routes.go

@@ -1,4 +1,4 @@
-package routes
+package http
 
 import (
 	"github.com/gin-contrib/pprof"

src/modules/transfer/transfer.go

@@ -13,12 +13,10 @@ import (
 	"github.com/didi/nightingale/src/modules/transfer/backend"
 	"github.com/didi/nightingale/src/modules/transfer/config"
 	"github.com/didi/nightingale/src/modules/transfer/cron"
-	"github.com/didi/nightingale/src/modules/transfer/http/routes"
+	"github.com/didi/nightingale/src/modules/transfer/http"
 	"github.com/didi/nightingale/src/modules/transfer/rpc"
-	"github.com/didi/nightingale/src/toolkits/http"
 	"github.com/didi/nightingale/src/toolkits/stats"
 
-	"github.com/gin-gonic/gin"
 	"github.com/toolkits/pkg/file"
 	"github.com/toolkits/pkg/logger"
 	"github.com/toolkits/pkg/runner"
@@ -66,9 +64,10 @@ func main() {
 	go report.Init(cfg.Report, "rdb")
 	go rpc.Start()
 
-	r := gin.New()
-	routes.Config(r)
-	go http.Start(r, "transfer", cfg.Logger.Level)
+	// r := gin.New()
+	// routes.Config(r)
+	// go http.Start(r, "transfer", cfg.Logger.Level)
+	http.Start()
 
 	cleanup()
 }