Merge branch 'main' into ddelemeny/refactor-multisearch

Author: fmassot

README.md

@@ -29,20 +29,20 @@ Quickwit 0.8 is compatible with 0.4.x versions only.
 
 You can either download the plugin manually and unzip it into the plugin directory or use the env variable `GF_INSTALL_PLUGINS` to install it.
 
-### 0.4.5 for Quickwit 0.8
+### 0.4.6 for Quickwit 0.8
 
 Run `grafana-oss` container with the env variable:
 
 ```bash
-docker run -p 3000:3000 -e GF_INSTALL_PLUGINS="https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.4.5/quickwit-quickwit-datasource-0.4.5.zip;quickwit-quickwit-datasource" grafana/grafana-oss run
+docker run -p 3000:3000 -e GF_INSTALL_PLUGINS="https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.4.6/quickwit-quickwit-datasource-0.4.6.zip;quickwit-quickwit-datasource" grafana/grafana-oss run
 ```
 
 Or download the plugin manually and start Grafana
 
 ```bash
-wget https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.4.5/quickwit-quickwit-datasource-0.4.5.zip
+wget https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.4.6/quickwit-quickwit-datasource-0.4.6.zip
 mkdir -p plugins
-unzip quickwit-quickwit-datasource-0.4.5.zip -d plugins/quickwit-quickwit-datasource-0.4.5
+unzip quickwit-quickwit-datasource-0.4.6.zip -d plugins/quickwit-quickwit-datasource-0.4.6
 docker run -p 3000:3000 -e GF_PATHS_PLUGINS=/data/plugins -v ${PWD}/plugins:/data/plugins grafana/grafana-oss run
 ```
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "quickwit-datasource",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "Quickwit datasource",
   "scripts": {
     "build": "webpack -c ./.config/webpack/webpack.config.ts --env production",

package.json

@@ -15,13 +15,20 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 )
 
+type ReadyStatus struct {
+	IsReady bool
+	Err     error
+}
+
 type DatasourceInfo struct {
 	ID                         int64
 	HTTPClient                 *http.Client
 	URL                        string
 	Database                   string
 	ConfiguredFields           ConfiguredFields
 	MaxConcurrentShardRequests int64
+	ReadyStatus                chan ReadyStatus
+	ShouldInit                 bool
 }
 
 // TODO: Move ConfiguredFields closer to handlers, the client layer doesn't need this stuff

pkg/quickwit/client/client.go

@@ -57,9 +57,6 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
 		return nil, err
 	}
 
-	timeField, toOk := jsonData["timeField"].(string)
-	timeOutputFormat, tofOk := jsonData["timeOutputFormat"].(string)
-
 	logLevelField, ok := jsonData["logLevelField"].(string)
 	if !ok {
 		logLevelField = ""
@@ -74,6 +71,7 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
 	if !ok {
 		index = ""
 	}
+	// XXX : Legacy check, should not happen ?
 	if index == "" {
 		index = settings.Database
 	}
@@ -92,18 +90,11 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
 		maxConcurrentShardRequests = 256
 	}
 
-	if !toOk || !tofOk {
-		timeField, timeOutputFormat, err = GetTimestampFieldInfos(index, settings.URL, httpCli)
-		if nil != err {
-			return nil, err
-		}
-	}
-
 	configuredFields := es.ConfiguredFields{
-		TimeField:        timeField,
-		TimeOutputFormat: timeOutputFormat,
 		LogLevelField:    logLevelField,
 		LogMessageField:  logMessageField,
+		TimeField:        "",
+		TimeOutputFormat: "",
 	}
 
 	model := es.DatasourceInfo{
@@ -113,16 +104,62 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
 		Database:                   index,
 		MaxConcurrentShardRequests: int64(maxConcurrentShardRequests),
 		ConfiguredFields:           configuredFields,
+		ReadyStatus:                make(chan es.ReadyStatus, 1),
+		ShouldInit:                 true,
 	}
-	return &QuickwitDatasource{dsInfo: model}, nil
+
+	ds := &QuickwitDatasource{dsInfo: model}
+
+	// Create an initialization goroutine
+	go func(ds *QuickwitDatasource, readyStatus chan<- es.ReadyStatus) {
+		var status es.ReadyStatus = es.ReadyStatus{
+			IsReady: false,
+			Err:     nil,
+		}
+		for {
+			// Will retry init everytime the channel is consumed until ready
+			if !status.IsReady || ds.dsInfo.ShouldInit {
+				qwlog.Debug("Initializing Datasource")
+				status.IsReady = true
+				status.Err = nil
+
+				indexMetadataList, err := GetIndexesMetadata(ds.dsInfo.Database, ds.dsInfo.URL, ds.dsInfo.HTTPClient)
+				if err != nil {
+					status.IsReady = false
+					status.Err = fmt.Errorf("failed to get index metadata : %w", err)
+				} else if len(indexMetadataList) == 0 {
+					status.IsReady = false
+					status.Err = fmt.Errorf("no index found for %s", ds.dsInfo.Database)
+				} else {
+					timeField, timeOutputFormat, err := GetTimestampFieldInfos(indexMetadataList)
+					if nil != err {
+						status.IsReady = false
+						status.Err = err
+					} else if "" == timeField {
+						status.IsReady = false
+						status.Err = fmt.Errorf("timefield is empty for %s", ds.dsInfo.Database)
+					} else if "" == timeOutputFormat {
+						status.Err = fmt.Errorf("timefield's output_format is empty, logs timestamps will not be parsed correctly for %s", ds.dsInfo.Database)
+					}
+
+					ds.dsInfo.ConfiguredFields.TimeField = timeField
+					ds.dsInfo.ConfiguredFields.TimeOutputFormat = timeOutputFormat
+					ds.dsInfo.ShouldInit = false
+				}
+			}
+			readyStatus <- status
+		}
+	}(ds, model.ReadyStatus)
+	return ds, nil
 }
 
 // Dispose here tells plugin SDK that plugin wants to clean up resources when a new instance
 // created. As soon as datasource settings change detected by SDK old datasource instance will
 // be disposed and a new one will be created using NewSampleDatasource factory function.
 func (ds *QuickwitDatasource) Dispose() {
-	// Clean up datasource instance resources.
-	// TODO
+	// FIXME: The ReadyStatus channel should probably be closed here, but doing it
+	// causes odd calls to healthcheck to fail. Needs investigation
+	// close(ds.dsInfo.ReadyStatus)
 }
 
 // CheckHealth handles health checks sent from Grafana to the plugin.
@@ -131,13 +168,39 @@ func (ds *QuickwitDatasource) Dispose() {
 // a datasource is working as expected.
 func (ds *QuickwitDatasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
 	res := &backend.CheckHealthResult{}
-
 	res.Status = backend.HealthStatusOk
 	res.Message = "plugin is running"
+
+	ds.dsInfo.ShouldInit = true
+	status := <-ds.dsInfo.ReadyStatus
+
+	if nil != status.Err {
+		res.Status = backend.HealthStatusError
+		res.Message = fmt.Errorf("Failed to initialize datasource: %w", status.Err).Error()
+	} else if "" == ds.dsInfo.ConfiguredFields.TimeField {
+		res.Status = backend.HealthStatusError
+		res.Message = fmt.Sprintf("timefield is missing from index config \"%s\"", ds.dsInfo.Database)
+	} else if "" == ds.dsInfo.ConfiguredFields.TimeOutputFormat {
+		res.Status = backend.HealthStatusError
+		res.Message = fmt.Sprintf("timefield's output_format is missing from index config \"%s\"", ds.dsInfo.Database)
+	}
+	qwlog.Debug(res.Message)
+
 	return res, nil
 }
 
 func (ds *QuickwitDatasource) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+	// Ensure ds is initialized, we need timestamp infos
+	status := <-ds.dsInfo.ReadyStatus
+	if !status.IsReady {
+		qwlog.Debug(fmt.Errorf("Datasource initialization failed: %w", status.Err).Error())
+		response := &backend.QueryDataResponse{
+			Responses: backend.Responses{},
+		}
+		response.Responses["__qwQueryDataError"] = backend.ErrDataResponse(backend.StatusInternal, "Datasource initialization failed")
+		return response, nil
+	}
+
 	return queryData(ctx, req.Queries, &ds.dsInfo)
 }