diff --git a/.env.sample b/.env.sample index cf74f98..fdcc419 100644 --- a/.env.sample +++ b/.env.sample @@ -19,6 +19,8 @@ BEMIDB_STORAGE_PATH=./iceberg # AWS_ACCESS_KEY_ID=[REPLACE_ME] # AWS_SECRET_ACCESS_KEY=[REPLACE_ME] +# DISABLE_ANONYMOUS_ANALYTICS=true + # Postgres syncing PG_DATABASE_URL=postgres://[USER]:[PASSWORD]@localhost:5432/[DATABASE] # PG_SYNC_INTERVAL=1h diff --git a/README.md b/README.md index 2181944..06673c3 100644 --- a/README.md +++ b/README.md @@ -209,16 +209,17 @@ psql postgres://localhost:54321/bemidb -c \ #### Other common options -| CLI argument | Environment variable | Default value | Description | -|---------------------------|-------------------------|---------------------------------|------------------------------------------------------| -| `--storage-type` | `BEMIDB_STORAGE_TYPE` | `LOCAL` | Storage type: `LOCAL` or `S3` | -| `--storage-path` | `BEMIDB_STORAGE_PATH` | `iceberg` | Path to the storage folder | -| `--log-level` | `BEMIDB_LOG_LEVEL` | `INFO` | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE` | -| `--aws-s3-endpoint` | `AWS_S3_ENDPOINT` | `s3.amazonaws.com` | AWS S3 endpoint | -| `--aws-region` | `AWS_REGION` | Required with `S3` storage type | AWS region | -| `--aws-s3-bucket` | `AWS_S3_BUCKET` | Required with `S3` storage type | AWS S3 bucket name | -| `--aws-access-key-id` | `AWS_ACCESS_KEY_ID` | Required with `S3` storage type | AWS access key ID | -| `--aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | Required with `S3` storage type | AWS secret access key | +| CLI argument | Environment variable | Default value | Description | +|--------------------------------|-------------------------------|--------------------------------|----------------------------------------------------------------------------| +| `--storage-type` | `BEMIDB_STORAGE_TYPE` | `LOCAL` | Storage type: `LOCAL` or `S3` | +| `--storage-path` | `BEMIDB_STORAGE_PATH` | `iceberg` | Path to the storage folder | +| `--log-level` | `BEMIDB_LOG_LEVEL` | `INFO` | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE` | +| `--disable-anonymous-analytics`| `DISABLE_ANONYMOUS_ANALYTICS` | `false` | Disable collection of anonymous usage metadata (OS type, database host) | +| `--aws-s3-endpoint` | `AWS_S3_ENDPOINT` | `s3.amazonaws.com` | AWS S3 endpoint | +| `--aws-region` | `AWS_REGION` | Required with `S3` storage type | AWS region | +| `--aws-s3-bucket` | `AWS_S3_BUCKET` | Required with `S3` storage type | AWS S3 bucket name | +| `--aws-access-key-id` | `AWS_ACCESS_KEY_ID` | Required with `S3` storage type | AWS access key ID | +| `--aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | Required with `S3` storage type | AWS secret access key | Note that CLI arguments take precedence over environment variables. I.e. you can override the environment variables with CLI arguments. diff --git a/src/config.go b/src/config.go index 8c4272c..cccc119 100644 --- a/src/config.go +++ b/src/config.go @@ -32,6 +32,8 @@ const ( ENV_PG_INCLUDE_TABLES = "PG_INCLUDE_TABLES" ENV_PG_EXCLUDE_TABLES = "PG_EXCLUDE_TABLES" + ENV_DISABLE_ANONYMOUS_ANALYTICS = "DISABLE_ANONYMOUS_ANALYTICS" + DEFAULT_PORT = "54321" DEFAULT_DATABASE = "bemidb" DEFAULT_USER = "" @@ -78,6 +80,7 @@ type Config struct { StoragePath string Aws AwsConfig Pg PgConfig + DisableAnalytics bool } type configParseValues struct { @@ -117,6 +120,7 @@ func registerFlags() { flag.StringVar(&_config.Aws.S3Bucket, "aws-s3-bucket", os.Getenv(ENV_AWS_S3_BUCKET), "AWS S3 bucket name") flag.StringVar(&_config.Aws.AccessKeyId, "aws-access-key-id", os.Getenv(ENV_AWS_ACCESS_KEY_ID), "AWS access key ID") flag.StringVar(&_config.Aws.SecretAccessKey, "aws-secret-access-key", os.Getenv(ENV_AWS_SECRET_ACCESS_KEY), "AWS secret access key") + flag.BoolVar(&_config.DisableAnalytics, "disable-anonymous-analytics", os.Getenv(ENV_DISABLE_ANONYMOUS_ANALYTICS) == "true", "Disable anonymous analytics collection") } func parseFlags() { diff --git a/src/main.go b/src/main.go index 11163b7..260dee0 100644 --- a/src/main.go +++ b/src/main.go @@ -6,7 +6,7 @@ import ( "time" ) -const VERSION = "0.31.4" +const VERSION = "0.31.5" func main() { config := LoadConfig() diff --git a/src/syncer.go b/src/syncer.go index f8cb781..0f99c98 100644 --- a/src/syncer.go +++ b/src/syncer.go @@ -1,12 +1,17 @@ package main import ( + "bytes" "context" "encoding/csv" + "encoding/json" "fmt" + "net/http" "net/url" "os" + "runtime" "strings" + "time" "github.com/jackc/pgx/v5" ) @@ -22,6 +27,12 @@ type Syncer struct { icebergReader *IcebergReader } +type TelemetryData struct { + DbHost string `json:"dbHost"` + OsName string `json:"osName"` + DbConnHash string `json:"dbConnHash"` +} + func NewSyncer(config *Config) *Syncer { if config.Pg.DatabaseUrl == "" { panic("Missing PostgreSQL database URL") @@ -35,6 +46,7 @@ func NewSyncer(config *Config) *Syncer { func (syncer *Syncer) SyncFromPostgres() { ctx := context.Background() databaseUrl := syncer.urlEncodePassword(syncer.config.Pg.DatabaseUrl) + syncer.sendTelemetry(databaseUrl) conn, err := pgx.Connect(ctx, databaseUrl) PanicIfError(err) @@ -322,3 +334,42 @@ func (syncer *Syncer) deleteOldIcebergSchemaTables(pgSchemaTables []PgSchemaTabl } } } + +func (syncer *Syncer) isLocalHost(hostname string) bool { + switch hostname { + case "localhost", "127.0.0.1", "::1", "0.0.0.0": + return true + } + return false +} + +func (syncer *Syncer) sendTelemetry(databaseUrl string) { + if syncer.config.DisableAnalytics { + LogInfo(syncer.config, "Telemetry is disabled") + return + } + + dbUrl, err := url.Parse(databaseUrl) + if err != nil { + return + } + + hostname := dbUrl.Hostname() + if syncer.isLocalHost(hostname) { + return + } + + data := TelemetryData{ + DbHost: hostname, + OsName: runtime.GOOS, + DbConnHash: StringToSha256Hash(databaseUrl), + } + + jsonData, err := json.Marshal(data) + if err != nil { + return + } + + client := http.Client{Timeout: 5 * time.Second} + _, _ = client.Post("http://api.bemidb.com/api/analytics", "application/json", bytes.NewBuffer(jsonData)) +} diff --git a/src/utils.go b/src/utils.go index 92cde53..0970f8c 100644 --- a/src/utils.go +++ b/src/utils.go @@ -6,10 +6,11 @@ import ( "crypto/sha256" "encoding/base64" "fmt" - "golang.org/x/crypto/pbkdf2" "os" "strconv" "unicode" + + "golang.org/x/crypto/pbkdf2" ) func PanicIfError(err error, message ...string) { @@ -77,6 +78,11 @@ func StringContainsUpper(str string) bool { return false } +func StringToSha256Hash(input string) string { + sum := sha256Hash([]byte(input)) + return fmt.Sprintf("%x", sum) +} + func hmacSha256Hash(key []byte, message []byte) []byte { hash := hmac.New(sha256.New, key) hash.Write(message)