Merge branch 'master' of https://github.com/didi/nightingale

README.md

@@ -4,7 +4,7 @@ v3.x终于来了，文档正在编写中，稍安勿躁...
 
 # 升级说明
 
-v3.x的版本和v2.x差别巨大，如果短期没办法迁移，可以继续使用 [v.2.8.0](https://github.com/didi/nightingale/tree/v2.8.0) ，我们之所以决定升级到v3.x，具体原因 [请看这里](https://mp.weixin.qq.com/s/BoGcqPiIQIuiK7cM3PTvrw) ，简而言之，我们是希望夜莺逐渐演化为一个运维平台
+v3.x的版本和v2.x差别巨大，没办法平滑迁移，可以继续使用 [v.2.8.0](https://github.com/didi/nightingale/tree/v2.8.0) ，我们之所以决定升级到v3.x，具体原因 [请看这里](https://mp.weixin.qq.com/s/BoGcqPiIQIuiK7cM3PTvrw) ，简而言之，我们是希望夜莺逐渐演化为一个运维平台。如果v2.x用着也能满足需求，可以继续用v2.x，毕竟，适合自己的才是最好的
 
 # 新版效果
 
@@ -64,8 +64,8 @@ grep redis -r .
 
 ```shell script
 cd /home/n9e
-wget http://116.85.64.82/pub.0927.tar.gz
+wget http://116.85.64.82/pub.20200928.tar.gz
-tar zxvf pub.0927.tar.gz
+tar zxvf pub.20200928.tar.gz
 ```
 
 6、覆盖nginx.conf，建议大家还是看一下这个配置，熟悉一下nginx配置，夜莺不同web侧组件就是通过nginx的不同location区分的。覆盖完了配置记得reload一下或者重启nginx

etc/rdb.yml

@@ -51,6 +51,11 @@ redis:
     read: 3000
     write: 3000
 
+rabbitmq:
+  enable: false
+  addr: amqp://root:1234@127.0.0.1:5672/
+  queue: test
+
 sender:
   mail:
     # three choice: smtp|shell|api

src/common/dataobj/query_item.go

@@ -84,11 +84,11 @@ type CludeRecv struct {
 }
 
 type XcludeResp struct {
-	Endpoints []string `json:"endpoints"`
+	Endpoint string   `json:"endpoint"`
-	Metric    string   `json:"metric"`
+	Metric   string   `json:"metric"`
-	Tags      []string `json:"tags"`
+	Tags     []string `json:"tags"`
-	Step      int      `json:"step"`
+	Step     int      `json:"step"`
-	DsType    string   `json:"dstype"`
+	DsType   string   `json:"dstype"`
 }
 
 type IndexByFullTagsRecv struct {

src/modules/rdb/config/yaml.go

@@ -76,8 +76,9 @@ type timeoutSection struct {
 }
 
 type rabbitmqSection struct {
-	Addr  string `yaml:"addr"`
+	Enable bool   `yaml:"enable"`
-	Queue string `yaml:"queue"`
+	Addr   string `yaml:"addr"`
+	Queue  string `yaml:"queue"`
 }
 
 var Config *ConfigT

src/modules/rdb/rabbitmq/conn.go

@@ -1,9 +1,11 @@
 package rabbitmq
 
 import (
+	"fmt"
 	"log"
 
 	"github.com/streadway/amqp"
+	"github.com/toolkits/pkg/logger"
 )
 
 var (
@@ -19,6 +21,54 @@ func Init(url string) {
 	}
 }
 
+// ping 测试rabbitmq连接是否正常
+func ping() (err error) {
+	if conn == nil {
+		return fmt.Errorf("conn is nil")
+	}
+
+	ch, err := conn.Channel()
+	if err != nil {
+		logger.Error(err)
+		close()
+		return err
+	}
+
+	defer ch.Close()
+
+	err = ch.ExchangeDeclare("ping.ping", "topic", false, true, false, true, nil)
+	if err != nil {
+		logger.Error(err)
+		return err
+	}
+
+	msgContent := "ping.ping"
+	err = ch.Publish("ping.ping", "ping.ping", false, false, amqp.Publishing{
+		ContentType: "text/plain",
+		Body:        []byte(msgContent),
+	})
+	if err != nil {
+		logger.Error(err)
+		return err
+	}
+	err = ch.ExchangeDelete("ping.ping", false, false)
+	if err != nil {
+		logger.Error(err)
+	}
+
+	return err
+}
+
+func close() {
+	if conn != nil {
+		err := conn.Close()
+		if err != nil {
+			logger.Error(err)
+		}
+		conn = nil
+	}
+}
+
 func Shutdown() {
 	conn.Close()
 	exit <- true

src/modules/rdb/rabbitmq/queue_consume.go

@@ -3,26 +3,42 @@ package rabbitmq
 import (
 	"time"
 
+	"github.com/streadway/amqp"
 	"github.com/toolkits/pkg/logger"
 )
 
-func Consume(queueName string) {
+// Consume 消费消息
-	go func(queueName string) {
+func Consume(url, queueName string) {
-		for {
+	for {
+		select {
+		case <-exit:
+			return
+		default:
+			if err := ping(); err != nil {
+				logger.Error("rabbitmq conn failed")
+				conn, err = amqp.Dial(url)
+				if err != nil {
+					conn = nil
+					logger.Error(err)
+					time.Sleep(500 * time.Millisecond)
+					continue
+				}
+			}
+
 			sleep := consume(queueName)
 			if sleep {
 				time.Sleep(300 * time.Millisecond)
 			}
-
-			if _, ok := <-exit; ok {
-				return
-			}
 		}
-	}(queueName)
+	}
 }
 
 // 如果操作MQ出现问题，或者没有load到数据，就sleep一下
 func consume(queueName string) bool {
+	if conn == nil {
+		return true
+	}
+
 	ch, err := conn.Channel()
 	if err != nil {
 		logger.Error(err)
@@ -45,7 +61,7 @@ func consume(queueName string) bool {
 	}
 
 	err = ch.Qos(
-		0,     // prefetch count
+		80,    // prefetch count
 		0,     // prefetch size
 		false, // global
 	)

src/modules/rdb/rdb.go

@@ -68,9 +68,9 @@ func main() {
 	cron.InitWorker()
 
 	// 初始化 rabbitmq 处理部分异步逻辑
-	if len(config.Config.RabbitMQ.Addr) > 0 {
+	if config.Config.RabbitMQ.Enable {
 		rabbitmq.Init(config.Config.RabbitMQ.Addr)
-		rabbitmq.Consume(config.Config.RabbitMQ.Queue)
+		go rabbitmq.Consume(config.Config.RabbitMQ.Addr, config.Config.RabbitMQ.Queue)
 	}
 
 	go cron.ConsumeMail()
@@ -102,7 +102,7 @@ func endingProc() {
 	http.Shutdown()
 	redisc.CloseRedis()
 
-	if len(config.Config.RabbitMQ.Addr) > 0 {
+	if config.Config.RabbitMQ.Enable {
 		rabbitmq.Shutdown()
 	}
 

src/modules/transfer/backend/influxdb/model.go

@@ -36,43 +36,51 @@ func (query *ShowSeries) renderEndpoints() {
 		}
 		endpointPart = endpointPart[:len(endpointPart)-len("OR")]
 		endpointPart += ")"
-		query.RawQuery = fmt.Sprintf("\"%s\" WHERE \"%s\"", query.RawQuery, endpointPart)
+		query.RawQuery = fmt.Sprintf("%s WHERE %s", query.RawQuery, endpointPart)
 	}
 }
 
 func (query *ShowSeries) renderInclude() {
 	if len(query.Include) > 0 {
 		// include
+		if len(query.Include) == 1 && query.Include[0] == nil {
+			return
+		}
 		includePart := "("
 		for _, include := range query.Include {
 			for _, value := range include.Values {
 				includePart += fmt.Sprintf(" \"%s\"='%s' OR", include.Key, value)
 			}
 		}
-		includePart = includePart[:len(includePart)-len("OR")]
+		includePart = includePart[:len(includePart)-len("AND")]
 		includePart += ")"
 		if !strings.Contains(query.RawQuery, "WHERE") {
-			query.RawQuery += " WHERE"
+			query.RawQuery = fmt.Sprintf(" %s WHERE %s", query.RawQuery, includePart)
+		} else {
+			query.RawQuery = fmt.Sprintf(" %s AND %s", query.RawQuery, includePart)
 		}
-		query.RawQuery = fmt.Sprintf(" %s AND %s", query.RawQuery, includePart)
 	}
 }
 
 func (query *ShowSeries) renderExclude() {
 	if len(query.Exclude) > 0 {
 		// exclude
+		if len(query.Exclude) == 1 && query.Exclude[0] == nil {
+			return
+		}
 		excludePart := "("
 		for _, exclude := range query.Exclude {
 			for _, value := range exclude.Values {
-				excludePart += fmt.Sprintf(" \"%s\"='%s' OR", exclude.Key, value)
+				excludePart += fmt.Sprintf(" \"%s\"!='%s' AND", exclude.Key, value)
 			}
 		}
-		excludePart = excludePart[:len(excludePart)-len("OR")]
+		excludePart = excludePart[:len(excludePart)-len("AND")]
 		excludePart += ")"
 		if !strings.Contains(query.RawQuery, "WHERE") {
-			query.RawQuery += " WHERE"
+			query.RawQuery = fmt.Sprintf(" %s WHERE %s", query.RawQuery, excludePart)
+		} else {
+			query.RawQuery = fmt.Sprintf(" %s AND %s", query.RawQuery, excludePart)
 		}
-		query.RawQuery = fmt.Sprintf(" %s AND %s", query.RawQuery, excludePart)
 	}
 }