support time limit for m3db query (#396)

.gitignore

@@ -54,4 +54,4 @@ _test
 /src/modules/transfer/transfer
 /src/modules/tsdb/tsdb
 
-
+tmp/

etc/rdb.yml

@@ -10,9 +10,8 @@ http:
 
 sso:
   enable: false
-  ssoAddr: "http://10.1.2.3:8071"
+  ssoAddr: "http://{sso-host}"
-  # TODO: redirectURL: "http://10.1.2.3:8072/auth-callback"
+  redirectURL: "http://{rdb-host}/auth-callback"
-  redirectURL: "http://10.1.2.3:8072/api/rdb/auth/callback"
   clientId: ""
   clientSecret: ""
   apiKey: ""

src/modules/transfer/backend/m3db/m3db.go

@@ -25,6 +25,7 @@ const (
 	METRIC_NAME   = "__name__"
 	SERIES_LIMIT  = 1000
 	DOCS_LIMIT    = 100
+	MAX_PONINTS   = 720
 )
 
 type M3dbSection struct {
@@ -51,6 +52,10 @@ type Client struct {
 	namespaceID ident.ID
 }
 
+func indexStartTime() time.Time {
+	return time.Now().Add(-time.Hour * 25)
+}
+
 func NewClient(cfg M3dbSection) (*Client, error) {
 	client, err := cfg.Config.NewClient(client.ConfigurationParameters{})
 	if err != nil {
@@ -267,7 +272,6 @@ func (p *Client) queryIndexByFullTags(session client.Session, input dataobj.Inde
 	ret = dataobj.IndexByFullTagsResp{
 		Metric: input.Metric,
 		Tags:   []string{},
-		Step:   10,
 		DsType: "GAUGE",
 	}
 
@@ -285,20 +289,22 @@ func (p *Client) queryIndexByFullTags(session client.Session, input dataobj.Inde
 	}
 
 	ret.Endpoints = input.Endpoints
+	tags := map[string]struct{}{}
 	for iter.Next() {
-		log.Printf("iter.next() ")
 		_, _, tagIter := iter.Current()
 		resp := xcludeResp(tagIter)
-		if len(resp.Tags) > 0 {
+		if len(resp.Tags) > 0 && len(resp.Tags[0]) > 0 {
-			ret.Tags = append(ret.Tags, resp.Tags[0])
+			tags[resp.Tags[0]] = struct{}{}
 		}
 	}
+	for k, _ := range tags {
+		ret.Tags = append(ret.Tags, k)
+	}
 	if err := iter.Err(); err != nil {
 		logger.Errorf("FetchTaggedIDs iter:", err)
 	}
 
 	return ret
-
 }
 
 // GetInstance: && (metric) (endpoint) (&& tags...)
@@ -438,12 +444,21 @@ func seriesIterWalk(iter encoding.SeriesIterator) (out *dataobj.TsdbQueryRespons
 
 	tagsIter := iter.Tags()
 	tags := map[string]string{}
+	var metric, endpoint string
+
 	for tagsIter.Next() {
 		tag := tagsIter.Current()
-		tags[tag.Name.String()] = tag.Value.String()
+		k := tag.Name.String()
+		v := tag.Value.String()
+		switch k {
+		case METRIC_NAME:
+			metric = v
+		case ENDPOINT_NAME, NID_NAME:
+			endpoint = v
+		default:
+			tags[k] = v
+		}
 	}
-	metric := tags[METRIC_NAME]
-	endpoint := tags[ENDPOINT_NAME]
 	counter, err := dataobj.GetCounter(metric, "", tags)
 
 	return &dataobj.TsdbQueryResponse{
@@ -483,7 +498,7 @@ func (cfg M3dbSection) validateTime(start, end int64, step *int) error {
 	}
 
 	if *step == 0 {
-		*step = int((end - start) / 720)
+		*step = int((end - start) / MAX_PONINTS)
 	}
 
 	if *step > cfg.MinStep {

src/modules/transfer/backend/m3db/query.go

@@ -134,7 +134,7 @@ func (cfg M3dbSection) queryMetricsOptions(input dataobj.EndpointsRecv) (index.Q
 		)},
 		index.AggregationOptions{
 			QueryOptions: index.QueryOptions{
-				StartInclusive: time.Time{},
+				StartInclusive: indexStartTime(),
 				EndExclusive:   time.Now(),
 				SeriesLimit:    cfg.SeriesLimit,
 				DocsLimit:      cfg.DocsLimit,
@@ -153,7 +153,7 @@ func (cfg M3dbSection) queryTagPairsOptions(input dataobj.EndpointMetricRecv) (i
 	return index.Query{idx.NewConjunctionQuery(q1, q2)},
 		index.AggregationOptions{
 			QueryOptions: index.QueryOptions{
-				StartInclusive: time.Time{},
+				StartInclusive: indexStartTime(),
 				EndExclusive:   time.Now(),
 				SeriesLimit:    cfg.SeriesLimit,
 				DocsLimit:      cfg.DocsLimit,
@@ -188,7 +188,7 @@ func (cfg M3dbSection) queryIndexByCludeOptions(input dataobj.CludeRecv) (index.
 	}
 
 	return query, index.QueryOptions{
-		StartInclusive: time.Time{},
+		StartInclusive: indexStartTime(),
 		EndExclusive:   time.Now(),
 		SeriesLimit:    cfg.SeriesLimit,
 		DocsLimit:      cfg.DocsLimit,
@@ -218,7 +218,7 @@ func (cfg M3dbSection) queryIndexByFullTagsOptions(input dataobj.IndexByFullTags
 	}
 
 	return query, index.QueryOptions{
-		StartInclusive: time.Time{},
+		StartInclusive: indexStartTime(),
 		EndExclusive:   time.Now(),
 		SeriesLimit:    cfg.SeriesLimit,
 		DocsLimit:      cfg.DocsLimit,

src/modules/transfer/backend/m3db/test/04-query-metrics.sh

@@ -7,7 +7,7 @@
 curl -X POST  \
 	http://localhost:8008/api/index/metrics \
 -d '{
-  "endpoints": []
+  "endpoints": ["10.178.24.116"]
 }'
 
 

src/modules/transfer/http/routes/query_router.go

@@ -148,5 +148,10 @@ func GetIndexByFullTags(c *gin.Context) {
 	}
 
 	resp := dataSource.QueryIndexByFullTags(recvs)
-	render.Data(c, resp, nil)
+	render.Data(c, &listResp{List: resp, Count: len(resp)}, nil)
+}
+
+type listResp struct {
+	List  interface{} `json:"list"`
+	Count int         `json:"count"`
 }