Skip to content

Commit be7c560

Browse files
crimson-gaocrimson-gao
committed
supp x-log-query-info
1 parent bd5389d commit be7c560

File tree

4 files changed

+170
-30
lines changed

4 files changed

+170
-30
lines changed

‎log_store.go

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -657,12 +657,15 @@ func (s *LogStore) GetLogLinesV2(req *GetLogRequest) (*GetLogLinesResponse, erro
657657
var logs []json.RawMessage
658658
_ = json.Unmarshal(data, &logs)
659659

660-
v2Rsp := convV3ToV2LogResp(v3Rsp, httpRsp.Header)
660+
v2Rsp, err := toLogRespV2(v3Rsp, httpRsp.Header)
661+
if err != nil {
662+
return nil, err
663+
}
664+
661665
lineRsp := GetLogLinesResponse{
662666
GetLogsResponse: *v2Rsp,
663667
Lines: logs,
664668
}
665-
666669
return &lineRsp, nil
667670
}
668671

@@ -770,23 +773,26 @@ func (s *LogStore) GetLogsV2(req *GetLogRequest) (*GetLogsResponse, error) {
770773
if err != nil {
771774
return nil, err
772775
}
773-
return convV3ToV2LogResp(resp, httpRsp.Header), nil
776+
return toLogRespV2(resp, httpRsp.Header)
774777
}
775778

776-
// @note: field [Contents] and header [x-log-query-info] is not supported in V3
777-
func convV3ToV2LogResp(v3Resp *GetLogsV3Response, respHeader http.Header) *GetLogsResponse {
778-
convMetaToHeader(v3Resp, respHeader)
779+
func toLogRespV2(v3Resp *GetLogsV3Response, respHeader http.Header) (*GetLogsResponse, error) {
780+
queryInfo, err := v3Resp.Meta.constructQueryInfo()
781+
if err != nil {
782+
return nil, fmt.Errorf("fail to construct x-log-query-info: %w", err)
783+
}
784+
writeHeader(v3Resp, respHeader, queryInfo)
779785
return &GetLogsResponse{
780786
Logs: v3Resp.Logs,
781787
Progress: v3Resp.Meta.Progress,
782788
Count: v3Resp.Meta.Count,
783789
HasSQL: v3Resp.Meta.HasSQL,
784-
Contents: "",
790+
Contents: queryInfo,
785791
Header: respHeader,
786-
}
792+
}, nil
787793
}
788794

789-
func convMetaToHeader(v3Resp *GetLogsV3Response, header http.Header) {
795+
func writeHeader(v3Resp *GetLogsV3Response, header http.Header, queryInfo string) {
790796
header.Add(GetLogsCountHeader, strconv.FormatInt(v3Resp.Meta.Count, 10))
791797
header.Add(ProcessedRows, strconv.FormatInt(v3Resp.Meta.ProcessedRows, 10))
792798
header.Add(ProgressHeader, v3Resp.Meta.Progress)
@@ -800,6 +806,7 @@ func convMetaToHeader(v3Resp *GetLogsV3Response, header http.Header) {
800806
header.Add(CpuCores, strconv.FormatFloat(v3Resp.Meta.CpuCores, 'E', -1, 64))
801807
header.Add(PowerSql, strconv.FormatBool(v3Resp.Meta.PowerSql))
802808
header.Add(InsertedSql, v3Resp.Meta.InsertedSql)
809+
header.Add(GetLogsQueryInfo, queryInfo)
803810
}
804811

805812
// GetLogsV3 query logs with [from, to) time range

‎log_store_test.go

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ func (s *GetLogsTestSuite) TestGetLogsV2() {
7373
From: int64(t - 900),
7474
To: int64(t + 10),
7575
Lines: 100,
76+
Query: "val1 | with_pack_meta",
7677
}
7778
// old
7879
ls := convertLogstore(s.client, s.env.ProjectName, s.env.LogstoreName)
@@ -84,22 +85,15 @@ func (s *GetLogsTestSuite) TestGetLogsV2() {
8485
s.Require().NoError(err)
8586

8687
// these headers need not to be compared
87-
filtered := map[string]bool{
88-
HTTPHeaderDate: true,
89-
ElapsedMillisecond: true,
90-
RequestIDHeader: true,
91-
HTTPHeaderLogDate: true,
92-
HTTPHeaderContentType: true,
93-
HTTPHeaderContentLength: true,
94-
GetLogsQueryInfo: true, // not support yet
95-
}
88+
filtered := map[string]bool{}
9689
addFilter := func(key string) {
9790
filtered[strings.ToLower(key)] = true
9891
}
9992
addFilter(HTTPHeaderDate)
10093
addFilter(ElapsedMillisecond)
10194
addFilter(RequestIDHeader)
10295
addFilter(HTTPHeaderLogDate)
96+
addFilter("x-log-time")
10397
addFilter(HTTPHeaderContentType)
10498
addFilter(HTTPHeaderContentLength)
10599
addFilter(GetLogsQueryInfo)
@@ -116,9 +110,53 @@ func (s *GetLogsTestSuite) TestGetLogsV2() {
116110
s.EqualValues(resp.Count, resp2.Count)
117111
s.EqualValues(resp.HasSQL, resp2.HasSQL)
118112

119-
// fmt.Printf("%#v\n", resp)
120-
// fmt.Printf("%#v\n", resp2)
113+
// compare query info
114+
queryInfoV2 := resp.Contents
115+
queryInfoV3 := resp2.Contents
116+
fmt.Println(queryInfoV2)
117+
fmt.Println(queryInfoV3)
118+
var info2 interface{}
119+
var info3 interface{}
120+
s.Require().NoError(json.Unmarshal([]byte(queryInfoV2), &info2))
121+
s.Require().NoError(json.Unmarshal([]byte(queryInfoV3), &info3))
122+
s.EqualValues(info2, info3)
121123
}
124+
125+
func (s *GetLogsTestSuite) TestConstructQueryInfo() {
126+
v3Meta := &GetLogsV3ResponseMeta{
127+
Keys: nil,
128+
Terms: nil,
129+
Marker: nil,
130+
Mode: nil,
131+
PhraseQueryInfo: nil,
132+
Shard: nil,
133+
ScanBytes: nil,
134+
IsAccurate: nil,
135+
ColumnTypes: nil,
136+
Highlights: nil,
137+
}
138+
contents, err := v3Meta.constructQueryInfo()
139+
s.Require().NoError(err)
140+
s.Equal("{}", contents)
141+
b := false
142+
v3Meta.IsAccurate = &b
143+
contents, err = v3Meta.constructQueryInfo()
144+
s.Require().NoError(err)
145+
s.Equal("{\"isAccurate\":0}", contents)
146+
b = true
147+
contents, err = v3Meta.constructQueryInfo()
148+
s.Require().NoError(err)
149+
s.Equal("{\"isAccurate\":1}", contents)
150+
151+
v3Meta.Keys = make([]string, 0)
152+
shard := 0
153+
v3Meta.Shard = &shard
154+
contents, err = v3Meta.constructQueryInfo()
155+
s.Require().NoError(err)
156+
157+
s.Equal("{\"shard\":0,\"isAccurate\":1}", contents)
158+
}
159+
122160
func (s *GetLogsTestSuite) TestMarshalLines() {
123161
logs := []map[string]string{
124162
{

‎model.go

Lines changed: 78 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ type MetaTerm struct {
9999
Key string `json:"key"`
100100
Term string `json:"term"`
101101
}
102-
type PhraseQueryInfo struct {
102+
type PhraseQueryInfoV3 struct {
103103
ScanAll *bool `json:"scanAll,omitempty"`
104104
BeginOffset *int64 `json:"beginOffset,omitempty"`
105105
EndOffset *int64 `json:"endOffset,omitempty"`
@@ -122,15 +122,83 @@ type GetLogsV3ResponseMeta struct {
122122
PowerSql bool `json:"powerSql"`
123123
InsertedSql string `json:"insertedSQL"`
124124

125-
Keys []string `json:"keys,omitempty"`
126-
Terms []MetaTerm `json:"terms,omitempty"`
127-
Marker *string `json:"string,omitempty"`
128-
Mode *int `json:"mode,omitempty"`
129-
PhraseQueryInfo *PhraseQueryInfo `json:"phraseQueryInfo,omitempty"`
130-
Shard *int `json:"shard,omitempty"`
131-
ScanBytes *int64 `json:"scanBytes,omitempty"`
132-
IsAccurate *bool `json:"isAccurate,omitempty"`
133-
ColumnTypes []string `json:"columnTypes,omitempty"`
125+
Keys []string `json:"keys,omitempty"`
126+
Terms []MetaTerm `json:"terms,omitempty"`
127+
Marker *string `json:"string,omitempty"`
128+
Mode *int `json:"mode,omitempty"`
129+
PhraseQueryInfo *PhraseQueryInfoV3 `json:"phraseQueryInfo,omitempty"`
130+
Shard *int `json:"shard,omitempty"`
131+
ScanBytes *int64 `json:"scanBytes,omitempty"`
132+
IsAccurate *bool `json:"isAccurate,omitempty"`
133+
ColumnTypes []string `json:"columnTypes,omitempty"`
134+
Highlights []map[string]string `json:"highlights,omitempty"`
135+
}
136+
137+
type PhraseQueryInfoV2 struct {
138+
ScanAll string `json:"scanAll,omitempty"`
139+
BeginOffset string `json:"beginOffset,omitempty"`
140+
EndOffset string `json:"endOffset,omitempty"`
141+
EndTime string `json:"endTime,omitempty"`
142+
}
143+
144+
func (s *PhraseQueryInfoV3) toPhraseQueryInfoV2() *PhraseQueryInfoV2 {
145+
if s == nil {
146+
return nil
147+
}
148+
return &PhraseQueryInfoV2{
149+
ScanAll: BoolPtrToStringNum(s.ScanAll),
150+
BeginOffset: Int64PtrToString(s.BeginOffset),
151+
EndOffset: Int64PtrToString(s.EndOffset),
152+
EndTime: Int64PtrToString(s.EndTime),
153+
}
154+
}
155+
156+
type QueryInfoV2 struct {
157+
Keys []string `json:"keys,omitempty"`
158+
Terms [][]string `json:"terms,omitempty"` // [[term, key], [term2, key2]]
159+
Limited string `json:"limited,omitempty"`
160+
Marker *string `json:"marker,omitempty"`
161+
Mode *int `json:"mode,omitempty"`
162+
PhraseQueryInfo *PhraseQueryInfoV2 `json:"phraseQueryInfo,omitempty"`
163+
Shard *int `json:"shard,omitempty"`
164+
ScanBytes *int64 `json:"scanBytes,omitempty"`
165+
IsAccurate *int64 `json:"isAccurate,omitempty"`
166+
ColumnTypes []string `json:"columnTypes,omitempty"`
167+
Highlights []map[string]string `json:"highlight,omitempty"`
168+
}
169+
170+
func (meta *GetLogsV3ResponseMeta) constructQueryInfo() (string, error) {
171+
var terms [][]string
172+
for _, term := range meta.Terms {
173+
terms = append(terms, []string{term.Term, term.Key})
174+
}
175+
var isAccurate *int64
176+
if meta.IsAccurate != nil {
177+
res := BoolToInt64(*meta.IsAccurate)
178+
isAccurate = &res
179+
}
180+
limited := ""
181+
if meta.Limited != 0 {
182+
limited = strconv.FormatInt(meta.Limited, 10)
183+
}
184+
queryInfo := &QueryInfoV2{
185+
Keys: meta.Keys,
186+
Terms: terms,
187+
Limited: limited,
188+
Marker: meta.Marker,
189+
Mode: meta.Mode,
190+
PhraseQueryInfo: meta.PhraseQueryInfo.toPhraseQueryInfoV2(),
191+
Shard: meta.Shard,
192+
ScanBytes: meta.ScanBytes,
193+
IsAccurate: isAccurate,
194+
ColumnTypes: meta.ColumnTypes,
195+
Highlights: meta.Highlights,
196+
}
197+
contents, err := json.Marshal(queryInfo)
198+
if err != nil {
199+
return "", err
200+
}
201+
return string(contents), nil
134202
}
135203

136204
// GetLogsV3Response defines response from GetLogs call

‎utils.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package sls
2+
3+
import "strconv"
4+
5+
func BoolToInt64(b bool) int64 {
6+
if b {
7+
return 1
8+
}
9+
return 0
10+
}
11+
12+
func BoolPtrToStringNum(b *bool) string {
13+
if b == nil {
14+
return ""
15+
}
16+
if *b {
17+
return "1"
18+
}
19+
return "0"
20+
}
21+
22+
func Int64PtrToString(i *int64) string {
23+
if i == nil {
24+
return ""
25+
}
26+
return strconv.FormatInt(*i, 10)
27+
}

0 commit comments

Comments
 (0)