支持消息组合发送,优化代码

e19aa95a · 谢宇轩 · 64293201 · e19aa95a · e19aa95a · e19aa95a
Commit e19aa95a authored Jun 08, 2021 by 谢宇轩 😅
17 changed files
--- a/README.md
+++ b/README.md
@@ -6,3 +6,141 @@
 # Done
 1. 从kafka消费数据
 2. 添加消费者的上下文
+---------
+project : jiwei-service
+errno : 71
+logId : 2263624661
+request_params : 
+refer : 
+cookie : 
+host : 127.0.0.1:8185
+clientIp : 127.0.0.1
+topic : jiwei-service-topic
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+errmsg : PARAM 'source' REQUIRED
+source : logtranfers
+level : WARNING
+created_at : 2021-05-13 00:11:59 +0800 CST
+uri : /api/news/feedstream
+optime : 1620835919
+------------
+level : WARNING
+clientIp : 127.0.0.1
+project : jiwei-service
+cookie : 
+optime : 1620835919
+errmsg : PARAM 'source' REQUIRED
+source : logtranfers
+created_at : 2021-05-13 00:11:59 +0800 CST
+errno : 71
+refer : 
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+request_params : 
+topic : jiwei-service-topic
+logId : 2263653937
+uri : /api/news/newsdetail
+host : 127.0.0.1:8185
+------------
+topic : jiwei-service-topic
+level : WARNING
+created_at : 2021-05-13 00:12:00 +0800 CST
+refer : 
+clientIp : 127.0.0.1
+optime : 1620835919
+source : logtranfers
+request_params : 
+project : jiwei-service
+logId : 2263669088
+errmsg : PARAM 'source' REQUIRED
+errno : 71
+uri : /api/advert/advertitem
+cookie : 
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+host : 127.0.0.1:8185
+------------
+topic : jiwei-service-topic
+level : WARNING
+project : jiwei-service
+optime : 1620835919
+created_at : 2021-05-13 00:12:00 +0800 CST
+cookie : 
+host : 127.0.0.1:8185
+errmsg : PARAM 'source' REQUIRED
+source : logtranfers
+logId : 2263669418
+uri : /api/news/newsrelation
+refer : 
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+errno : 71
+clientIp : 127.0.0.1
+request_params : 
+------------
+clientIp : 127.0.0.1
+request_params : 
+level : WARNING
+uri : /api/news/feedstream
+refer : 
+host : 127.0.0.1:8185
+created_at : 2021-05-13 00:12:00 +0800 CST
+errno : 71
+optime : 1620835919
+source : logtranfers
+topic : jiwei-service-topic
+project : jiwei-service
+logId : 2263669593
+cookie : 
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+errmsg : PARAM 'source' REQUIRED
+------------
+errno : 71
+logId : 2263749932
+cookie : 
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+clientIp : 127.0.0.1
+request_params : 
+source : logtranfers
+topic : jiwei-service-topic
+created_at : 2021-05-13 00:12:00 +0800 CST
+host : 127.0.0.1:8185
+level : WARNING
+uri : /api/news/newsdetail
+optime : 1620835920
+errmsg : PARAM 'source' REQUIRED
+project : jiwei-service
+refer : 
+------------
+topic : jiwei-service-topic
+level : WARNING
+optime : 1620835920
+project : jiwei-service
+uri : /api/news/newsrelation
+request_params : 
+source : logtranfers
+refer : 
+cookie : 
+host : 127.0.0.1:8185
+clientIp : 127.0.0.1
+errmsg : PARAM 'source' REQUIRED
+created_at : 2021-05-13 00:12:00 +0800 CST
+errno : 71
+logId : 2263762670
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+------------
+request_params : 
+topic : jiwei-service-topic
+cookie : 
+ua : node-fetch/1.0 (+https://github.com/bitinn/node-fetch)
+host : 127.0.0.1:8185
+project : jiwei-service
+created_at : 2021-05-13 00:12:00 +0800 CST
+errno : 71
+clientIp : 127.0.0.1
+source : logtranfers
+level : WARNING
+uri : /api/news/feedstream
+refer : 
+optime : 1620835920
+logId : 2263762960
+errmsg : PARAM 'source' REQUIRED
+------------
+host : 127.0.0.1:8185
\ No newline at end of file
--- a/conf/etcd.go
+++ b/conf/etcd.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 	"github.com/coreos/etcd/clientv3"
@@ -12,15 +13,16 @@ import (
 var (
 	configPath = "/logagent/config/"
+	statusPath = "/logagent/active/"
 )
 type LogagentConfig []byte
 var cli *clientv3.Client
-func init() {
+func Init(confPath string) {
 	// 加载配置文件
-	if err := ini.MapTo(APPConfig, "./logtransfer.conf"); err != nil {
+	if err := ini.MapTo(APPConfig, confPath); err != nil {
 		log.Println("load ini file error: ", err)
 		return
 	}
@@ -53,10 +55,30 @@ func GetAllConfFromEtcd() []LogagentConfig {
 	configs := make([]LogagentConfig, 0)
 	for _, etcdResult := range resp.Kvs {
+		etcdKey := statusPath + string(etcdResult.Key[strings.LastIndex(string(etcdResult.Key), "/")+1:])
+		ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+		resp, err := cli.Get(ctx, etcdKey)
+		cancel()
+		if err != nil {
+			panic(fmt.Sprintf("Get Etcd config failed, err:%s \n", err))
+		}
+		if len(resp.Kvs) != 0 {
+			status := string(resp.Kvs[0].Value)
+			if status == "1" {
 				log.Printf("load config from:%s ", etcdResult.Key)
 				configs = append(configs, etcdResult.Value)
 			}
+		}
+	}
 	return configs
 }

--- a/conf/setting.go
+++ b/conf/setting.go
@@ -9,7 +9,6 @@ type LogTransferConf struct {
 // kafka 配置
 type Kafka struct {
 	Address   string `ini:"address"`
-	QueueSize string `ini:"queue_size"`
 }
 // ETCD 配置
@@ -20,6 +19,7 @@ type Etcd struct {
 // Es 属性
 type Es struct {
 	Address string `ini:"address"`
+	BulkSize int `ini:"bulk_size"`
 }
 var (

--- a/ding/client.go
+++ b/ding/client.go
+package ding
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+func GetDingClient(messageJson string) {
+	body := bytes.NewBuffer([]byte(messageJson))
+	// 构造request对象
+	request, err := http.NewRequest("POST", "https://oapi.dingtalk.com/robot/send?access_token=3bd7a0b3e0558b4a2c3650d42cdca704fd95b4108852c46a86b6339b083b92e1", body)
+	request.Header.Set("Content-Type", "application/json")
+	if err != nil {
+		log.Fatal(err)
+	}
+	client := &http.Client{}
+	resp, err := client.Do(request)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	fmt.Println("response Status:", resp.Status)
+	fmt.Println("response Headers:", resp.Header)
+	resultBody, _ := ioutil.ReadAll(resp.Body)
+	fmt.Println("response Body:", string(resultBody))
+}
--- a/ding/message.go
+++ b/ding/message.go
+package ding
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+)
+const (
+	MARKTYPE       = "markdown"
+	TEXTTYPE       = "text"
+	CONTENT_TEMPLE = "#  %s服务状态异常\r\n > 异常类型:%s  \r\n > 触发时间:%s  \r\n ###### Warning"
+)
+type DDAt struct {
+	IsAtAll   bool     `json:"isAtAll"`
+	AtMobiles []string `json:"atMobiles"`
+}
+type TextContent struct {
+	Content string `json:"content,omitempty"`
+}
+type MarkdownContent struct {
+	Title string `json:"title,omitempty"`
+	Text  string `json:"text,omitempty"`
+}
+type DDMessage struct {
+	MsgType  string          `json:"msgtype"`
+	Markdown MarkdownContent `json:"markdown,omitempty"`
+	Text     TextContent     `json:"text,omitempty"`
+	At       DDAt            `json:"at"`
+}
+/**
+* 获取可以使用的消息对象Json
+*
+* @return string
+ */
+func (m *DDMessage) process() string {
+	data, err := json.Marshal(m)
+	if err != nil {
+		log.Panic(err)
+	}
+	return fmt.Sprintf("%s\n", data)
+}
+/**
+* 初始化消息
+*
+* @return string
+ */
+func initMessage(status string, appName string, time string) string {
+	var message DDMessage
+	message.MsgType = MARKTYPE
+	message.Markdown.Title = "日志报警"
+	message.Markdown.Text = fmt.Sprintf(CONTENT_TEMPLE, appName, status, time)
+	message.At.IsAtAll = false
+	message.At.AtMobiles = []string{}
+	messageJson := message.process()
+	log.Println(messageJson)
+	return messageJson
+}
--- a/go.mod
+++ b/go.mod
@@ -12,13 +12,19 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/uuid v1.2.0 // indirect
+	github.com/olivere/elastic/v7 v7.0.24
 	github.com/segmentio/kafka-go v0.4.13
 	go.uber.org/zap v1.16.0 // indirect
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
+	golang.org/x/mod v0.3.0
 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
 	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/tools v0.1.0 // indirect
-	google.golang.org/grpc v1.26.0 // indirect
+	google.golang.org/grpc v1.33.2
 	gopkg.in/ini.v1 v1.62.0
 )
+replace github.com/coreos/go-systemd => github.com/coreos/go-systemd/v22 v22.0.0
+replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
--- a/go.sum
+++ b/go.sum
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/aws/aws-sdk-go v1.38.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -12,6 +14,8 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -19,6 +23,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s=
 github.com/envoyproxy/go-control-plane v0.9.6/go.mod h1:GFqM7v0B62MraO4PWRedIbhThr/Rf7ev6aHOOPXeaDA=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
@@ -26,9 +31,13 @@ github.com/envoyproxy/go-control-plane v0.9.8/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgx
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -39,6 +48,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -49,11 +59,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
@@ -61,14 +76,26 @@ github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/olivere/elastic v1.0.1 h1:UeafjZg+TifCVPhCJNPof0pUHig6vbXuJEbC/A+Ouo0=
+github.com/olivere/elastic v6.2.35+incompatible h1:MMklYDy2ySi01s123CB2WLBuDMzFX4qhFcA5tKWJPgM=
+github.com/olivere/elastic/v7 v7.0.24 h1:9ZcCQP3Pvgese7TaypYiVAL49sCEphyIwkVxtRf8jb8=
+github.com/olivere/elastic/v7 v7.0.24/go.mod h1:OuWmD2DiuYhddWegBKPWQuelVKBLrW0fa/VUYgxuGTY=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/segmentio/kafka-go v0.4.13 h1:0i28vWX02rOuQave8Z8dS1lCDH8lgSRKqnDqqHHU7UY=
 github.com/segmentio/kafka-go v0.4.13/go.mod h1:BVDwBTF24avtlj4l8/xsWNb4papVeg16+jO6/0qjvhA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -78,6 +105,7 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
@@ -99,6 +127,7 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -108,6 +137,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
@@ -159,6 +189,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20210406143921-e86de6bf7a46 h1:f4STrQZf8jaowsiUitigvrqMCCM4QJH1A2JCSI7U1ow=
 google.golang.org/genproto v0.0.0-20210406143921-e86de6bf7a46/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
@@ -168,6 +199,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ
 google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.36.1 h1:cmUfbeGKnz9+2DD/UYsMQXeqbHZqZDs4eQwW0sFOpBY=
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c=
@@ -190,6 +223,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
 gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

--- a/logtransfer
+++ b/logtransfer
--- a/logtransfer.conf
+++ b/logtransfer.conf
@@ -2,13 +2,13 @@
 # Kafka配置
 [kafka]
-address=localhost:9091
+address=47.107.239.240:9091,47.107.239.240:9092
-queue_size=1000
 # Kafka配置
 [etcd]
-address=localhost:23790
+address=47.107.239.240:23379
 # ES 配置
 [es]
-address="http://localhost:9200/"
+address="http://47.107.239.240:9320/"
\ No newline at end of file
+bulk_size=50
\ No newline at end of file
--- a/main.go
+++ b/main.go
 package main
 import (
+	"flag"
+	"fmt"
 	"github.com/y7ut/logtransfer/transfer"
 )
+const version = "1.0.0"
+var c = flag.String("c", "./logtransfer.conf", "使用配置文件启动")
+var v = flag.Bool("v", false, "查看当前程序版本")
 func main() {
-	transfer.Run()
+	// 加载配置文件
+	flag.Parse()
+	if *v {
+		fmt.Println("Jiwei Logtransfer \nversion: ", version)
+		return
+	}
+	transfer.Run(*c)
 }
--- a/kafka/kafka.go
+++ b/kafka/kafka.go
-package kafka
+package queue
 import (
-	"context"
 	"log"
+	"strings"
 	"github.com/segmentio/kafka-go"
 	"github.com/y7ut/logtransfer/conf"
 )
-type Customer struct {
+func InitReader(topic string, groupId string) *kafka.Reader {
-	Reader *kafka.Reader
-	Ctx context.Context
-	Cancel context.CancelFunc
-}
-func InitReader(topic string, groupId string) *Customer {
 	// 先去创建一下这个分组
 	CreateCustomerGroup(topic, groupId)
 	// make a writer that produces to topic-A, using the least-bytes distribution
 	r := kafka.NewReader(kafka.ReaderConfig{
-		Brokers:     []string{conf.APPConfig.Kafka.Address},
+		Brokers:   strings.Split(conf.APPConfig.Kafka.Address, ","),
 		Topic:     topic,
 		GroupID:   groupId,
 		Partition: 0,
 		MinBytes:  10e3, // 10KB
 		MaxBytes:  10e6, // 10MB
 	})
-	context, cancel := context.WithCancel(context.Background())
-	return &Customer{Reader: r, Ctx:context, Cancel: cancel}
+	return r
 }
 func CreateCustomerGroup(topic string, groupId string) {
 	config := kafka.ConsumerGroupConfig{
 		ID:          groupId,
-		Brokers:     []string{conf.APPConfig.Kafka.Address},
+		Brokers:     strings.Split(conf.APPConfig.Kafka.Address, ","),
 		Topics:      []string{topic},
 		StartOffset: kafka.LastOffset,
 	}

--- a/storage/es.go
+++ b/storage/es.go
 package storage
+import (
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"github.com/y7ut/logtransfer/conf"
+)
+func GetClient() *elastic.Client {
+	var err error
+	client, err := elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(conf.APPConfig.Es.Address))
+	if err != nil {
+		fmt.Println("connect es error", err)
+		return nil
+	}
+	return client
+}
--- a/transfer/collector.go
+++ b/transfer/collector.go
@@ -18,14 +18,17 @@ func loadCollectors() []Collector {
 	configs := conf.GetAllConfFromEtcd()
 	collectors := make([]Collector, 0)
 	for _, v := range configs {
-		log.Printf("json decode config:%s ", v)
 		var currentCollector []Collector
 		err := json.Unmarshal(v, &currentCollector)
 		if err != nil {
-			log.Printf("json decode config:%s  err: %s", v, err)
+			log.Printf("json decode config(%s) err :  err: %s", v, err)
 		}
+		if currentCollector !=nil {
+			log.Printf("Init config:%s ", v)
 			collectors = append(collectors, currentCollector...)
 		}
+	}
 	return collectors
 }

--- a/transfer/customer.go
+++ b/transfer/customer.go
+package transfer
+import "github.com/segmentio/kafka-go"
+type Customer struct {
+	Reader         *kafka.Reader
+	HandlePipeline *PipeLine
+	Format         Formater
+	done           chan struct{}
+}
+func (c Customer) Exit() {
+	go func() {
+		c.done <- struct{}{}
+	}()
+}
+func (c Customer) Listen() chan struct{} {
+	return c.done
+}
+func registerManger(c *Customer) {
+	mu.Lock()
+	CustomerManger[c.Reader.Config().Topic] = c
+	mu.Unlock()
+}
+func getCustomer(topic string) (customer *Customer, ok bool) {
+	mu.Lock()
+	customer, ok = CustomerManger[topic]
+	mu.Unlock()
+	return customer, ok
+}
+func uninstallCustomer(topic string) {
+	mu.Lock()
+	delete(CustomerManger, topic)
+	mu.Unlock()
+}
--- a/transfer/format.go
+++ b/transfer/format.go
+package transfer
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+)
+var (
+	contentRegexp       = regexp.MustCompile(`\[(?s:(.*?))\]`)
+	serviceWfLogKeyWord = []string{"errno", "logId", "uri", "refer", "cookie", "ua", "host", "clientIp", "optime", "request_params", "errmsg"}
+)
+type Formater func(string, string) Matedate
+// service错误日志的处理
+func FormatServiceWfLog(sourceKey string,message string)  Matedate {
+	result := make(Matedate,0)
+	levelIndex := strings.Index(message, ":")
+	result["topic"] = sourceKey
+	result["level"] = message[:levelIndex]
+	result["project"] = "jiwei-service"
+	message = message[levelIndex:]
+	loc, _ := time.LoadLocation("Local")
+	logTime, _ := time.ParseInLocation(": 06-01-02 15:04:05 ", message[:strings.Index(message, "[")],loc)
+	result["created_at"] = logTime
+	keyword := serviceWfLogKeyWord
+	for _, word := range keyword {
+		flysnowRegexp := regexp.MustCompile(fmt.Sprintf(`%s\[(?s:(.*?))\]`, word))
+		logContent := flysnowRegexp.FindString(message)
+		result[word] = contentRegexp.FindStringSubmatch(logContent)[1]
+	}
+	return result
+}
+// service错误日志的处理
+func DefaultLog(sourceKey string, message string)  Matedate {
+	result := make(Matedate,0)
+	result["topic"] = sourceKey
+	result["message"] = message
+	return result
+}
--- a/transfer/handler.go
+++ b/transfer/handler.go
+package transfer
+import (
+	"fmt"
+)
+type Matedate map[string]interface{}
+type Handler func(Matedate) Matedate
+type PipeLine struct {
+	Next    *PipeLine
+	Current Handler
+}
+func (p PipeLine) Enter(m Matedate) Matedate {
+	if p.Current == nil {
+		return m
+	}
+	if p.Next == nil {
+		return p.Current(m)
+	}
+	return p.Next.Enter(p.Current(m))
+}
+// 持久化到ES
+func Edit(m Matedate) Matedate {
+	m["source"] = "logtranfers"
+	return m
+}
+// 持久化到ES
+func SaveES(m Matedate) Matedate {
+	messages <- m
+	return m
+}
+// 打印
+func Dump(m Matedate) Matedate {
+	for k, v := range m {
+		fmt.Printf("%s : %s\n", k, v)
+	}
+	fmt.Println("------------")
+	return m
+}
--- a/transfer/kernel.go
+++ b/transfer/kernel.go
@@ -6,47 +6,26 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
-	"github.com/y7ut/logtransfer/kafka"
+	elastic "github.com/olivere/elastic/v7"
+	"github.com/y7ut/logtransfer/conf"
+	"github.com/y7ut/logtransfer/queue"
 )
 var (
-	Start          = make(chan *kafka.Customer)
+	Start          = make(chan *Customer)
 	Close          = make(chan string)
-	CustomerManger = make(map[string]*kafka.Customer, 0)
+	CustomerManger = make(map[string]*Customer, 0)
-	MaxRetryTime   = 3
+	MaxRetryTime   = 10
 	mu             sync.Mutex
 	closeWg        sync.WaitGroup
+	messages       = make(chan Matedate, conf.APPConfig.Es.BulkSize)
 )
-// type Kernel struct{
-// 	CustomerManger map[string]*kafka.Customer
-// 	mu 		       sync.Mutex
-// }
-func registerManger(c *kafka.Customer) {
-	mu.Lock()
-	CustomerManger[c.Reader.Config().Topic] = c
-	mu.Unlock()
-}
-func getCustomer(topic string) (customer *kafka.Customer, ok bool) {
-	mu.Lock()
-	customer, ok = CustomerManger[topic]
-	mu.Unlock()
-	return customer, ok
-}
-func uninstallCustomer(topic string) {
-	mu.Lock()
-	delete(CustomerManger, topic)
-	mu.Unlock()
-}
 func getRegisterTopics() (topics []string) {
 	mu.Lock()
 	for topic := range CustomerManger {
@@ -57,22 +36,33 @@ func getRegisterTopics() (topics []string) {
 }
 // 核心启动
-func Run() {
+func Run(confPath string) {
+	conf.Init(confPath)
+	esClient, err := elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(conf.APPConfig.Es.Address))
+	if err != nil {
+		fmt.Println("connect es error", err)
+		panic(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	go MatedateSender(ctx, esClient)
 	go func() {
-		for{
+		for {
 			select {
 			case customer := <-Start:
-				log.Println("Starting Customer...")
 				registerManger(customer)
-				go ReadingMessage(customer)
+				go ReadingMessage(ctx, customer)
 			case closer := <-Close:
-				_, ok := getCustomer(closer)
+				c, ok := getCustomer(closer)
 				if !ok {
 					log.Printf(" Customer %s unstall Failed ", closer)
 				}
-				// log.Printf(" Customer %s exit with offset(%d)", closer, c.Reader.Offset())
+				c.Exit()
 				closeWg.Done()
 			}
@@ -81,8 +71,19 @@ func Run() {
 	for topic := range ChooseTopic() {
 		GroupID := topic + "_group"
-		currentCustomer := kafka.InitReader(topic, GroupID)
+		r := queue.InitReader(topic, GroupID)
-		log.Printf("Init Customer of (%s) success!", topic)
+		currentFormater := DefaultLog
+		// TODO:使用ETCD下发需要的pipe handler
+		if strings.Contains(topic, "service") {
+			currentFormater = FormatServiceWfLog
+		}
+		// Edit->Save->Dump 这个顺序
+		pipe := &PipeLine{Current: Edit, Next: &PipeLine{Current: Dump, Next: &PipeLine{Current: SaveES, Next: nil}}}
+		currentCustomer := &Customer{Reader: r, done: make(chan struct{}), HandlePipeline: pipe, Format: currentFormater}
+		log.Printf("Check Customer group of [%s] success!", GroupID)
 		Start <- currentCustomer
 	}
@@ -92,16 +93,22 @@ func Run() {
 			log.Println("Safe Exit with:", sign)
 			currentTopics := getRegisterTopics()
-			closeWg.Add(len(currentTopics))
 			for _, topic := range currentTopics {
-				log.Printf(" Customer %s unstalling...", topic)
+				closeWg.Add(1)
 				Close <- topic
-			}
+				log.Printf(" Customer %s unstalling...", topic)
 				closeWg.Wait()
+			}
+			close(messages)
 			log.Printf(" Success unstall %d Transfer", len(currentTopics))
 			os.Exit(0)
 		}
 	}
+	defer cancel()
 }
 func sign() <-chan os.Signal {
@@ -112,54 +119,77 @@ func sign() <-chan os.Signal {
 }
 // 從Kafka中消費消息，注意这里会提交commit offset
-func ReadingMessage(c *kafka.Customer) {
+func ReadingMessage(ctx context.Context, c *Customer) {
 	defer c.Reader.Close()
-	log.Println("Start Customer success!")
+	log.Printf("Start Customer Group[%s] success!", c.Reader.Config().GroupID)
+	// var trycount int
+	// var cstSh, _ = time.LoadLocation("Asia/Shanghai") //上海时区
-	var trycount int
 	for {
 		select {
-		case <-c.Ctx.Done():
+		case <-c.Listen():
-			closeWg.Add(1)
+			// 监听需要关闭的信号
-			Close<- c.Reader.Config().Topic
-				closeWg.Wait()
 			log.Println("Close customer of Topic :", c.Reader.Config().Topic)
 			return
 		default:
-			// 使用超时上下文
+			// // 使用超时上下文, 但是这样是非阻塞，超过deadline就报错了
-			timeoutCtx, cancel := context.WithTimeout(c.Ctx, 5*time.Second)
+			// // timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
-			m, err := c.Reader.FetchMessage(timeoutCtx)
+			// // 这里使用阻塞的上下文
-			cancel()
+			m, err := c.Reader.ReadMessage(ctx)
 			if err != nil {
-				log.Println("failed to Reader messages:", err)
+				// 退出
-				trycount++
+				c.Exit()
-				if trycount >= MaxRetryTime {
-					log.Println("Too many retries, Customer has Suspended  : ", err)
-					c.Cancel()
-				}
 				continue
 			}
-			// 临时打印
+			// 流入pipe
-			fmt.Printf("message at topic/partition/offset %v/%v/%v: %s = %s\n", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))
+			c.HandlePipeline.Enter(c.Format(string(c.Reader.Config().Topic), string(m.Value)))
+		}
+	}
+}
+func MatedateSender(ctx context.Context, esClient *elastic.Client) {
+	tick := time.NewTicker(3 * time.Second)
+	var (
+		SenderMu sync.Mutex
+	)
-			// 使用超时上下文
+	bulkRequest := esClient.Bulk()
-			nextTimeoutCtx, nextCancel := context.WithTimeout(c.Ctx, 5*time.Second)
+	for {
-			err = c.Reader.CommitMessages(nextTimeoutCtx, m)
+		select {
-			nextCancel()
+		case m := <-messages:
+			indexRequest := elastic.NewBulkIndexRequest().Index("logs").Doc(m)
+			bulkRequest.Add(indexRequest)
+		case <-tick.C:
+			// Do sends the bulk requests to Elasticsearch
+			SenderMu.Lock()
+			count := bulkRequest.NumberOfActions()
+			if count > 0{
+				log.Printf("Send message to Es: %d : \n", bulkRequest.NumberOfActions())
+				_, err := bulkRequest.Do(ctx)
 				if err != nil {
-				cancel()
+					log.Println("Save Es Error:", err)
-				log.Println("failed to commit messages:", err)
-				trycount++
-				if trycount >= MaxRetryTime {
-					log.Println("Too many retries, Customer has Suspended  : ", err)
-					c.Cancel()
 				}
-				continue
+				bulkRequest.Reset()
 			}
+			SenderMu.Unlock()
-			// 手动限速
+		case <-ctx.Done():
-			time.Sleep(1 * time.Second)
+			// Do sends the bulk requests to Elasticsearch
+			SenderMu.Lock()
+			_, err := bulkRequest.Do(ctx)
+			if err != nil {
+				log.Println("Save Es Error:", err)
+			}
+			bulkRequest.Reset()
+			SenderMu.Unlock()
+			log.Println("Exiting...")
+			return
 		}
 	}
 }