support anonymous struct field for monapi.plugins.template (#547)

* move get collectrule api from /api/mon to /v1/mon

* support anonymous struct field for monapi.plugins.template

* add tls with mysql, redis and mongodb

* add rdb.user.pwdExpiresAt

etc/prober.yml

@@ -7,8 +7,7 @@ logger:
 pluginsConfig: etc/plugins
 
 report:
-  enabled: true
   region: default 
-  interval: 4000
-  timeout: 3000
-  api: api/hbs/heartbeat
+
+collectRule:
+  token: monapi-internal-third-module-pass-fjsdi

src/models/user.go

@@ -63,6 +63,7 @@ type User struct {
 	LockedAt     int64     `json:"locked_at" description:"locked time"`
 	UpdatedAt    int64     `json:"updated_at" description:"user info change time"`
 	PwdUpdatedAt int64     `json:"pwd_updated_at" description:"password change time"`
+	PwdExpiresAt int64     `xorm:"-" json:"pwd_expires_at" description:"password expires time"`
 	LoggedAt     int64     `json:"logged_at" description:"last logged time"`
 	CreateAt     time.Time `json:"create_at" xorm:"<-"`
 }

src/modules/monapi/collector/template.go

@@ -14,12 +14,12 @@ import (
 var fieldCache sync.Map // map[reflect.Type]structFields
 
 type Field struct {
-	skip        bool               `json:"-"`
 	Name        string             `json:"name,omitempty"`
 	Label       string             `json:"label,omitempty"`
 	Default     interface{}        `json:"default,omitempty"`
 	Enum        []interface{}      `json:"enum,omitempty"`
 	Example     string             `json:"example,omitempty"`
+	Format      string             `json:"format,omitempty"`
 	Description string             `json:"description,omitempty"`
 	Required    bool               `json:"required,omitempty"`
 	Items       *Field             `json:"items,omitempty" description:"arrays's items"`
@@ -27,6 +27,11 @@ type Field struct {
 	Ref         string             `json:"$ref,omitempty" description:"name of the struct ref"`
 	Fields      []Field            `json:"fields,omitempty" description:"fields of struct type"`
 	Definitions map[string][]Field `json:"definitions,omitempty"`
+
+	// list      []Field
+	skip  bool `json:"-"`
+	index []int
+	typ   reflect.Type
 }
 
 func (p Field) String() string {
@@ -44,40 +49,93 @@ func cachedTypeContent(t reflect.Type) Field {
 
 func typeContent(t reflect.Type) Field {
 	definitions := map[string][]Field{t.String(): nil}
-	ret := Field{}
-
-	for i := 0; i < t.NumField(); i++ {
-		sf := t.Field(i)
-		isUnexported := sf.PkgPath != ""
-		if sf.Anonymous {
-			panic("unsupported anonymous field")
-		} else if isUnexported {
-			// Ignore unexported non-embedded fields.
-			continue
-		}
+	current := []Field{}
+	next := []Field{{typ: t}}
 
-		field := getTagOpt(sf)
-		if field.skip {
-			continue
-		}
-		ft := sf.Type
+	// Count of queued names for current level and the next.
+	var count, nextCount map[reflect.Type]int
 
-		fieldType(ft, &field, definitions)
+	// Types already visited at an earlier level.
+	visited := map[reflect.Type]bool{}
 
-		// Record found field and index sequence.
-		if field.Name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
-			ret.Fields = append(ret.Fields, field)
-			continue
-		}
+	// Fields found.
+	var fields []Field
 
-		panic("unsupported anonymous, struct field")
-	}
+	for len(next) > 0 {
+		current, next = next, current[:0]
+		count, nextCount = nextCount, map[reflect.Type]int{}
 
-	definitions[t.String()] = ret.Fields
+		for _, f := range current {
+			if visited[f.typ] {
+				continue
+			}
+			visited[f.typ] = true
+
+			// Scan f.typ for fields to include.
+			for i := 0; i < f.typ.NumField(); i++ {
+				sf := f.typ.Field(i)
+				isUnexported := sf.PkgPath != ""
+				if sf.Anonymous {
+					t := sf.Type
+					if t.Kind() == reflect.Ptr {
+						t = t.Elem()
+					}
+					if isUnexported && t.Kind() != reflect.Struct {
+						// Ignore embedded fields of unexported non-struct types.
+						continue
+					}
+					// Do not ignore embedded fields of unexported struct types
+					// since they may have exported fields.
+				} else if isUnexported {
+					// Ignore unexported non-embedded fields.
+					continue
+				}
+
+				field := getTagOpt(sf)
+				if field.skip {
+					continue
+				}
+				index := make([]int, len(f.index)+1)
+				copy(index, f.index)
+				index[len(f.index)] = i
+
+				ft := sf.Type
+				if ft.Name() == "" && ft.Kind() == reflect.Ptr {
+					// Follow pointer.
+					ft = ft.Elem()
+				}
+
+				fieldType(ft, &field, definitions)
+
+				// Record found field and index sequence.
+				if field.Name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
+					field.index = index
+					field.typ = ft
+
+					fields = append(fields, field)
+					if count[f.typ] > 1 {
+						// If there were multiple instances, add a second,
+						// so that the annihilation code will see a duplicate.
+						// It only cares about the distinction between 1 or 2,
+						// so don't bother generating any more copies.
+						fields = append(fields, fields[len(fields)-1])
+					}
+					continue
+				}
+
+				// Record new anonymous struct to explore in next round.
+				nextCount[ft]++
+				if nextCount[ft] == 1 {
+					next = append(next, Field{index: index, typ: ft})
+				}
+
+			}
+		}
+	}
 
-	ret.Definitions = definitions
+	definitions[t.String()] = fields
 
-	return ret
+	return Field{Fields: fields, Definitions: definitions}
 }
 
 // tagOptions is the string following a comma in a struct field's "json"
@@ -134,6 +192,7 @@ func getTagOpt(sf reflect.StructField) (opt Field) {
 	opt.Name = name
 	opt.Label = _s(sf.Tag.Get("label"))
 	opt.Example = sf.Tag.Get("example")
+	opt.Format = sf.Tag.Get("format")
 	opt.Description = _s(sf.Tag.Get("description"))
 	if s := sf.Tag.Get("enum"); s != "" {
 		if err := json.Unmarshal([]byte(s), &opt.Enum); err != nil {

src/modules/monapi/http/router.go

@@ -121,8 +121,7 @@ func Config(r *gin.Engine) {
 
 	collectRulesAnonymous := r.Group("/api/mon/collect-rules")
 	{
-		collectRulesAnonymous.GET("/endpoints/:endpoint/remote", collectRulesGetByRemoteEndpoint) // for prober
-		collectRulesAnonymous.GET("/endpoints/:endpoint/local", collectRulesGetByLocalEndpoint)   // for agent
+		collectRulesAnonymous.GET("/endpoints/:endpoint/local", collectRulesGetByLocalEndpoint) // for agent
 	}
 
 	stra := r.Group("/api/mon/stra").Use(GetCookieUser())
@@ -183,14 +182,10 @@ func Config(r *gin.Engine) {
 		indexProxy.POST("/counter/detail", indexReq)
 	}
 
-	/*
-		v1 := r.Group("/v1/mon")
-		{
-			v1.POST("/report-detector-heartbeat", detectorHeartBeat)
-			v1.GET("/detectors", detectorInstanceGets)
-			v1.GET("/rules", collectRulesGet)
-		}
-	*/
+	v1 := r.Group("/v1/mon")
+	{
+		v1.GET("/collect-rules/endpoints/:endpoint/remote", collectRulesGetByRemoteEndpoint) // for prober
+	}
 
 	if config.Get().Logger.Level == "DEBUG" {
 		pprof.Register(r, "/api/monapi/debug/pprof")

src/modules/monapi/plugins/mongodb/mongodb.go

@@ -50,7 +50,7 @@ type MongodbRule struct {
 	GatherPerdbStats    bool     `label:"Per DB stats" json:"gather_perdb_stats" description:"When true, collect per database stats" default:"false"`
 	GatherColStats      bool     `label:"Col stats" json:"gather_col_stats" description:"When true, collect per collection stats" default:"false"`
 	ColStatsDbs         []string `label:"Col stats dbs" json:"col_stats_dbs" description:"List of db where collections stats are collected, If empty, all db are concerned" example:"local" default:"[\"local\"]"`
-	// tlsint.ClientConfig
+	plugins.ClientConfig
 	// Ssl                 Ssl
 }
 
@@ -74,5 +74,6 @@ func (p *MongodbRule) TelegrafInput() (telegraf.Input, error) {
 		GatherColStats:      p.GatherColStats,
 		ColStatsDbs:         p.ColStatsDbs,
 		Log:                 plugins.GetLogger(),
+		ClientConfig:        p.ClientConfig.TlsClientConfig(),
 	}, nil
 }

src/modules/monapi/plugins/mysql/mysql.go

@@ -96,6 +96,7 @@ type MysqlRule struct {
 	GatherGlobalVars                    bool     `label:"Global Vars" json:"gather_global_variables" description:"gather metrics from PERFORMANCE_SCHEMA.GLOBAL_VARIABLES" default:"true"`
 	IntervalSlow                        string   `label:"Interval Slow" json:"interval_slow" description:"Some queries we may want to run less often (such as SHOW GLOBAL VARIABLES)" example:"30m"`
 	MetricVersion                       int      `label:"-" json:"-"`
+	plugins.ClientConfig
 }
 
 func (p *MysqlRule) Validate() error {
@@ -142,5 +143,6 @@ func (p *MysqlRule) TelegrafInput() (telegraf.Input, error) {
 		IntervalSlow:                        p.IntervalSlow,
 		MetricVersion:                       2,
 		Log:                                 plugins.GetLogger(),
+		ClientConfig:                        p.ClientConfig.TlsClientConfig(),
 	}, nil
 }

src/modules/monapi/plugins/redis/redis.go

@@ -54,7 +54,8 @@ type RedisCommand struct {
 type RedisRule struct {
 	Servers  []string        `label:"Servers" json:"servers,required" description:"specify servers" example:"tcp://localhost:6379"`
 	Commands []*RedisCommand `label:"Commands" json:"commands" description:"Optional. Specify redis commands to retrieve values"`
-	Password string          `label:"Password" json:"password" description:"specify server password"`
+	Password string          `label:"Password" json:"password" format:"password" description:"specify server password"`
+	plugins.ClientConfig
 }
 
 func (p *RedisRule) Validate() error {
@@ -107,9 +108,10 @@ func (p *RedisRule) TelegrafInput() (telegraf.Input, error) {
 	}
 
 	return &redis.Redis{
-		Servers:  p.Servers,
-		Commands: commands,
-		Password: p.Password,
-		Log:      plugins.GetLogger(),
+		Servers:      p.Servers,
+		Commands:     commands,
+		Password:     p.Password,
+		Log:          plugins.GetLogger(),
+		ClientConfig: p.ClientConfig.TlsClientConfig(),
 	}, nil
 }

src/modules/monapi/plugins/types.go

@@ -0,0 +1,37 @@
+package plugins
+
+import (
+	"github.com/didi/nightingale/src/toolkits/i18n"
+	"github.com/influxdata/telegraf/plugins/common/tls"
+)
+
+func init() {
+	i18n.DictRegister(langDict)
+}
+
+var (
+	langDict = map[string]map[string]string{
+		"zh": map[string]string{
+			"disables SSL certificate verification":                           "禁用SSL证书验证",
+			"verify certificates of TLS enabled servers using this CA bundle": "使用此CA文件验证服务器的证书",
+			"identify TLS client using this SSL certificate file":             "使用此SSL证书文件标识TLS客户端",
+			"identify TLS client using this SSL key file":                     "使用此SSL密钥文件标识TLS客户端",
+		},
+	}
+)
+
+type ClientConfig struct {
+	InsecureSkipVerify bool   `label:"Insecure Skip" json:"insecure_skip_verify" default:"false" description:"disables SSL certificate verification"`
+	TLSCA              string `label:"CA" json:"tls_ca" format:"file" description:"verify certificates of TLS enabled servers using this CA bundle"`
+	TLSCert            string `label:"Cert" json:"tls_cert" format:"file" description:"identify TLS client using this SSL certificate file"`
+	TLSKey             string `label:"Key" json:"tls_key" format:"file" description:"identify TLS client using this SSL key file"`
+}
+
+func (config ClientConfig) TlsClientConfig() tls.ClientConfig {
+	return tls.ClientConfig{
+		InsecureSkipVerify: config.InsecureSkipVerify,
+		TLSCA:              config.TLSCA,
+		TLSCert:            config.TLSCert,
+		TLSKey:             config.TLSKey,
+	}
+}

src/modules/monapi/scache/collectrule.go

@@ -172,7 +172,7 @@ func (p *collectRuleCache) syncPlacement() error {
 			if _, exists := nodesMap[d.Region]; !exists {
 				nodesMap[d.Region] = make(map[string]struct{})
 			}
-			nodesMap[d.Region][d.Identity+":"+d.RPCPort] = struct{}{}
+			nodesMap[d.Region][d.Identity+":"+d.HTTPPort] = struct{}{}
 		}
 	}
 

src/modules/prober/cache/collectrule.go

@@ -24,6 +24,7 @@ type CollectRuleCache struct {
 	TS      map[int64]int64
 	C       chan time.Time
 	timeout time.Duration
+	token   string
 }
 
 func NewCollectRuleCache(cf *config.CollectRuleSection) *CollectRuleCache {
@@ -31,8 +32,9 @@ func NewCollectRuleCache(cf *config.CollectRuleSection) *CollectRuleCache {
 		CollectRuleSection: cf,
 		Data:               make(map[int64]*models.CollectRule),
 		TS:                 make(map[int64]int64),
-		timeout:            time.Duration(cf.Timeout) * time.Millisecond,
 		C:                  make(chan time.Time, 1),
+		timeout:            time.Duration(cf.Timeout) * time.Millisecond,
+		token:              cf.Token,
 	}
 }
 
@@ -119,10 +121,10 @@ func (p *CollectRuleCache) syncCollectRule() error {
 			return fmt.Errorf("getIdent err %s", err)
 		}
 
-		url := fmt.Sprintf("http://%s/api/mon/collect-rules/endpoints/%s:%s/remote",
-			addrs[perm[i]], ident, report.Config.RPCPort)
-		err = httplib.Get(url).SetTimeout(p.timeout).ToJSON(&resp)
-		if err != nil {
+		url := fmt.Sprintf("http://%s/v1/mon/collect-rules/endpoints/%s:%s/remote",
+			addrs[perm[i]], ident, report.Config.HTTPPort)
+		if err = httplib.Get(url).SetTimeout(p.timeout).
+			Header("X-Srv-Token", p.token).ToJSON(&resp); err != nil {
 			logger.Warningf("get %s collect rule from remote failed, error:%v", url, err)
 			stats.Counter.Set("collectrule.get.err", 1)
 			continue

src/modules/rdb/auth/auth.go

@@ -44,3 +44,7 @@ func DeleteToken(accessToken string) error {
 func Start() error {
 	return defaultAuth.Start()
 }
+
+func PrepareUser(user *models.User) {
+	defaultAuth.PrepareUser(user)
+}

src/modules/rdb/auth/authenticator.go

@@ -294,6 +294,15 @@ func (p *Authenticator) Start() error {
 	return nil
 }
 
+func (p *Authenticator) PrepareUser(user *models.User) {
+	if !p.extraMode {
+		return
+	}
+
+	cf := cache.AuthConfig()
+	user.PwdExpiresAt = user.PwdUpdatedAt + cf.PwdExpiresIn*86400*30
+}
+
 // cleanup rdb.session & sso.token
 func (p *Authenticator) cleanupSession() {
 	now := time.Now().Unix()
@@ -432,7 +441,7 @@ func checkPassword(cf *models.AuthConfig, passwd string) error {
 	spCode := []byte{'!', '@', '#', '$', '%', '^', '&', '*', '_', '-', '~', '.', ',', '<', '>', '/', ';', ':', '|', '?', '+', '='}
 
 	if cf.PwdMinLenght > 0 && len(passwd) < cf.PwdMinLenght {
-		return _e("Password too short (min:%d) %s", cf.PwdMinLenght, cf.MustInclude())
+		return _e("Password too short (min:%d)", cf.PwdMinLenght)
 	}
 
 	passwdByte := []byte(passwd)
@@ -469,19 +478,19 @@ func checkPassword(cf *models.AuthConfig, passwd string) error {
 	}
 
 	if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_UPPER > 0 && indNum[0] == 0 {
-		return _e("Invalid Password, %s", cf.MustInclude())
+		return _e("Invalid Password, must include %s", _s("Upper char"))
 	}
 
 	if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_LOWER > 0 && indNum[1] == 0 {
-		return _e("Invalid Password, %s", cf.MustInclude())
+		return _e("Invalid Password, must include %s", _s("Lower char"))
 	}
 
 	if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_NUMBER > 0 && indNum[2] == 0 {
-		return _e("Invalid Password, %s", cf.MustInclude())
+		return _e("Invalid Password, must include %s", _s("Number"))
 	}
 
 	if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_SPEC_CHAR > 0 && indNum[3] == 0 {
-		return _e("Invalid Password, %s", cf.MustInclude())
+		return _e("Invalid Password, must include %s", _s("Special char"))
 	}
 
 	return nil

src/modules/rdb/config/i18n.go

@@ -96,8 +96,8 @@ var (
 			"Number":                                       "数字",
 			"Special char":                                 "特殊字符",
 			"Must include %s":                              "必须包含 %s",
-			"Invalid Password, %s":                         "密码不符合规范, %s",
-			"character: %s not supported":                  "不支持的字符 %s",
+			"Invalid Password, must include %s":            "密码不符合规范, 必须包括 %s",
+			"character: %s not supported %s":               "不支持的字符 %s, %s",
 			"Incorrect login/password %s times, you still have %s chances": "登陆失败%d次，你还有%d次机会",
 			"The limited sessions %d":                                      "会话数量限制，最多%d个会话",
 			"Password has been expired":                                    "密码已过期，请重置密码",
@@ -106,7 +106,7 @@ var (
 			"User is frozen":                                               "用户已休眠",
 			"User is writen off":                                           "用户已注销",
 			"Minimum password length %d":                                   "密码最小长度 %d",
-			"Password too short (min:%d) %s":                               "密码太短 (最小 %d) %s",
+			"Password too short (min:%d)":                                  "密码太短 (最小 %d)",
 			"%s format error":                                              "%s 所填内容不符合规范",
 			"%s %s format error":                                           "%s %s 所填内容不符合规范",
 			"username too long (max:%d)":                                   "用户名太长 (最长:%d)",

src/modules/rdb/http/router_funcs.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 
 	"github.com/didi/nightingale/src/models"
+	"github.com/didi/nightingale/src/modules/rdb/auth"
 	"github.com/didi/nightingale/src/toolkits/i18n"
 	"github.com/gin-gonic/gin"
 	"github.com/toolkits/pkg/errors"
@@ -159,6 +160,8 @@ func loginUser(c *gin.Context) *models.User {
 		bomb("unauthorized")
 	}
 
+	auth.PrepareUser(user)
+
 	return user
 }
 

src/modules/rdb/http/router_user.go

@@ -25,6 +25,7 @@ func userListGet(c *gin.Context) {
 
 	for i := 0; i < len(list); i++ {
 		list[i].UUID = ""
+		auth.PrepareUser(&list[i])
 	}
 
 	renderData(c, gin.H{
@@ -103,6 +104,9 @@ func userAddPost(c *gin.Context) {
 func userProfileGet(c *gin.Context) {
 	user := User(urlParamInt64(c, "id"))
 	user.UUID = ""
+
+	auth.PrepareUser(user)
+
 	renderData(c, user, nil)
 }