Add sync telemetry

BemiHQ · Feb 12, 2025 · 89db433 · 89db433
1 parent 8f67293
commit 89db433
Show file tree

Hide file tree

Showing 5 changed files with 75 additions and 11 deletions.
diff --git a/.env.sample b/.env.sample
@@ -19,6 +19,8 @@ BEMIDB_STORAGE_PATH=./iceberg
 # AWS_ACCESS_KEY_ID=[REPLACE_ME]
 # AWS_SECRET_ACCESS_KEY=[REPLACE_ME]
 
+# DISABLE_ANONYMOUS_ANALYTICS=true
+
 # Postgres syncing
 PG_DATABASE_URL=postgres://[USER]:[PASSWORD]@localhost:5432/[DATABASE]
 # PG_SYNC_INTERVAL=1h

diff --git a/README.md b/README.md
@@ -209,16 +209,17 @@ psql postgres://localhost:54321/bemidb -c \
 
 #### Other common options
 
-| CLI argument              | Environment variable    | Default value                   | Description                                          |
-|---------------------------|-------------------------|---------------------------------|------------------------------------------------------|
-| `--storage-type`          | `BEMIDB_STORAGE_TYPE`   | `LOCAL`                         | Storage type: `LOCAL` or `S3`                        |
-| `--storage-path`          | `BEMIDB_STORAGE_PATH`   | `iceberg`                       | Path to the storage folder                           |
-| `--log-level`             | `BEMIDB_LOG_LEVEL`      | `INFO`                          | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE` |
-| `--aws-s3-endpoint`       | `AWS_S3_ENDPOINT`       | `s3.amazonaws.com`              | AWS S3 endpoint                                      |
-| `--aws-region`            | `AWS_REGION`            | Required with `S3` storage type | AWS region                                           |
-| `--aws-s3-bucket`         | `AWS_S3_BUCKET`         | Required with `S3` storage type | AWS S3 bucket name                                   |
-| `--aws-access-key-id`     | `AWS_ACCESS_KEY_ID`     | Required with `S3` storage type | AWS access key ID                                    |
-| `--aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | Required with `S3` storage type | AWS secret access key                                |
+| CLI argument                   | Environment variable          | Default value                  | Description                                                                |
+|--------------------------------|-------------------------------|--------------------------------|----------------------------------------------------------------------------|
+| `--storage-type`               | `BEMIDB_STORAGE_TYPE`         | `LOCAL`                        | Storage type: `LOCAL` or `S3`                                              |
+| `--storage-path`               | `BEMIDB_STORAGE_PATH`         | `iceberg`                      | Path to the storage folder                                                 |
+| `--log-level`                  | `BEMIDB_LOG_LEVEL`            | `INFO`                         | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`                       |
+| `--disable-anonymous-analytics`| `DISABLE_ANONYMOUS_ANALYTICS` | `false`                        | Disable collection of anonymous usage metadata (OS type, database host)    |
+| `--aws-s3-endpoint`            | `AWS_S3_ENDPOINT`             | `s3.amazonaws.com`             | AWS S3 endpoint                                                            |
+| `--aws-region`                 | `AWS_REGION`                  | Required with `S3` storage type | AWS region                                                                |
+| `--aws-s3-bucket`              | `AWS_S3_BUCKET`               | Required with `S3` storage type | AWS S3 bucket name                                                        |
+| `--aws-access-key-id`          | `AWS_ACCESS_KEY_ID`           | Required with `S3` storage type | AWS access key ID                                                         |
+| `--aws-secret-access-key`      | `AWS_SECRET_ACCESS_KEY`       | Required with `S3` storage type | AWS secret access key                                                     |
 
 Note that CLI arguments take precedence over environment variables. I.e. you can override the environment variables with CLI arguments.
 

diff --git a/src/config.go b/src/config.go
@@ -32,6 +32,8 @@ const (
 	ENV_PG_INCLUDE_TABLES  = "PG_INCLUDE_TABLES"
 	ENV_PG_EXCLUDE_TABLES  = "PG_EXCLUDE_TABLES"
 
+	ENV_DISABLE_ANONYMOUS_ANALYTICS     = "DISABLE_ANONYMOUS_ANALYTICS"
+
 	DEFAULT_PORT              = "54321"
 	DEFAULT_DATABASE          = "bemidb"
 	DEFAULT_USER              = ""
@@ -78,6 +80,7 @@ type Config struct {
 	StoragePath       string
 	Aws               AwsConfig
 	Pg                PgConfig
+	DisableAnalytics  bool
 }
 
 type configParseValues struct {
@@ -117,6 +120,7 @@ func registerFlags() {
 	flag.StringVar(&_config.Aws.S3Bucket, "aws-s3-bucket", os.Getenv(ENV_AWS_S3_BUCKET), "AWS S3 bucket name")
 	flag.StringVar(&_config.Aws.AccessKeyId, "aws-access-key-id", os.Getenv(ENV_AWS_ACCESS_KEY_ID), "AWS access key ID")
 	flag.StringVar(&_config.Aws.SecretAccessKey, "aws-secret-access-key", os.Getenv(ENV_AWS_SECRET_ACCESS_KEY), "AWS secret access key")
+	flag.BoolVar(&_config.DisableAnalytics, "disable-anonymous-analytics", os.Getenv(ENV_DISABLE_ANONYMOUS_ANALYTICS) == "true", "Disable anonymous analytics collection")
 }
 
 func parseFlags() {

diff --git a/src/syncer.go b/src/syncer.go
@@ -1,12 +1,17 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/csv"
+	"encoding/json"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"strings"
+	"time"
 
 	"github.com/jackc/pgx/v5"
 )
@@ -22,6 +27,12 @@ type Syncer struct {
 	icebergReader *IcebergReader
 }
 
+type TelemetryData struct {
+	DbHost     string `json:"dbHost"`
+	OsName     string `json:"osName"`
+	DbConnHash string `json:"dbConnHash"`
+}
+
 func NewSyncer(config *Config) *Syncer {
 	if config.Pg.DatabaseUrl == "" {
 		panic("Missing PostgreSQL database URL")
@@ -35,6 +46,7 @@ func NewSyncer(config *Config) *Syncer {
 func (syncer *Syncer) SyncFromPostgres() {
 	ctx := context.Background()
 	databaseUrl := syncer.urlEncodePassword(syncer.config.Pg.DatabaseUrl)
+	syncer.sendTelemetry(databaseUrl)
 
 	conn, err := pgx.Connect(ctx, databaseUrl)
 	PanicIfError(err)
@@ -322,3 +334,42 @@ func (syncer *Syncer) deleteOldIcebergSchemaTables(pgSchemaTables []PgSchemaTabl
 		}
 	}
 }
+
+func (syncer *Syncer) isLocalHost(hostname string) bool {
+	switch hostname {
+	case "localhost", "127.0.0.1", "::1", "0.0.0.0":
+		return true
+	}
+	return false
+}
+
+func (syncer *Syncer) sendTelemetry(databaseUrl string) {
+	if syncer.config.DisableAnalytics {
+		LogInfo(syncer.config, "Telemetry is disabled")
+		return
+	}
+
+	dbUrl, err := url.Parse(databaseUrl)
+	if err != nil {
+		return
+	}
+
+	hostname := dbUrl.Hostname()
+	if syncer.isLocalHost(hostname) {
+		return
+	}
+
+	data := TelemetryData{
+		DbHost:     hostname,
+		OsName:     runtime.GOOS,
+		DbConnHash: StringToSha256Hash(databaseUrl),
+	}
+
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return
+	}
+
+	client := http.Client{Timeout: 5 * time.Second}
+	_, _ = client.Post("http://api.bemidb.com/api/analytics", "application/json", bytes.NewBuffer(jsonData))
+}
diff --git a/src/utils.go b/src/utils.go
@@ -6,10 +6,11 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
-	"golang.org/x/crypto/pbkdf2"
 	"os"
 	"strconv"
 	"unicode"
+
+	"golang.org/x/crypto/pbkdf2"
 )
 
 func PanicIfError(err error, message ...string) {
@@ -77,6 +78,11 @@ func StringContainsUpper(str string) bool {
 	return false
 }
 
+func StringToSha256Hash(input string) string {
+	sum := sha256Hash([]byte(input))
+	return fmt.Sprintf("%x", sum)
+}
+
 func hmacSha256Hash(key []byte, message []byte) []byte {
 	hash := hmac.New(sha256.New, key)
 	hash.Write(message)