Load e2e env from file

.github/workflows/flow.yml

@@ -30,7 +30,7 @@ jobs:
       catalog:
         image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine
         ports:
-          - 5432:5432
+          - 9901:5432
         env:
           PGUSER: postgres
           POSTGRES_PASSWORD: postgres
@@ -213,13 +213,13 @@ jobs:
           openssl req -new -key certs/cert.key -out certs/cert.csr -subj /CN=minio.local &&
           openssl x509 -req -days 3650 -in certs/cert.csr -signkey certs/cert.key -out certs/cert.crt &&
           chown -R 1001 certs &&
-          ./minio server ./minio-data --certs-dir ./certs --address :9999 &
+          ./minio server ./minio-data --certs-dir ./certs --address :9001 &
           sleep 2 &&
-          ./mc alias set myminiopeerdb http://localhost:9999 minio miniosecret &&
+          ./mc alias set myminiopeerdb http://localhost:9001 _peerdb_minioadmin _peerdb_minioadmin &&
           ./mc mb myminiopeerdb/peerdb
         env:
-          MINIO_ROOT_USER: minio
-          MINIO_ROOT_PASSWORD: miniosecret
+          MINIO_ROOT_USER: _peerdb_minioadmin
+          MINIO_ROOT_PASSWORD: _peerdb_minioadmin
           AWS_EC2_METADATA_DISABLED: true
 
       - name: create postgres extensions, increase logical replication limits, and setup catalog database
@@ -304,7 +304,7 @@ jobs:
                 <shard>
                   <replica>
                     <host>localhost</host>
-                    <port>9001</port>
+                    <port>29001</port>
                   </replica>
                 </shard>
               </cicluster>
@@ -331,7 +331,7 @@ jobs:
             <tmp_path>var/lib/clickhouse/tmp</tmp_path>
             <user_files_path>var/lib/clickhouse/user_files</user_files_path>
             <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
-            <tcp_port>9001</tcp_port>
+            <tcp_port>29001</tcp_port>
             <http_port remove="1"/>
             <postgresql_port remove="1"/>
             <mysql_port remove="1"/>
@@ -359,7 +359,7 @@ jobs:
                 <shard>
                   <replica>
                     <host>localhost</host>
-                    <port>9001</port>
+                    <port>29001</port>
                   </replica>
                 </shard>
               </cicluster>
@@ -409,6 +409,9 @@ jobs:
       - name: Install gotestsum
         run: |
           go install gotest.tools/gotestsum@latest
+      - name: Add host.docker.internal to /etc/hosts
+        run: |
+          echo "127.0.0.1 host.docker.internal" | sudo tee -a /etc/hosts
       - name: run tests
         run: |
           temporal server start-dev --namespace default --headless &
@@ -425,20 +428,20 @@ jobs:
         working-directory: ./flow
         env:
           GOCOVERDIR: coverage
-          AWS_ENDPOINT_URL_S3: http://localhost:9999
-          AWS_ACCESS_KEY_ID: minio
-          AWS_SECRET_ACCESS_KEY: miniosecret
+          AWS_ENDPOINT_URL_S3: http://localhost:9001
+          AWS_ACCESS_KEY_ID: _peerdb_minioadmin
+          AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
           AWS_REGION: us-east-1
-          AWS_ENDPOINT_URL_S3_TLS: https://localhost:9998
-          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
-          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
+          AWS_ENDPOINT_URL_S3_TLS: https://localhost:9003
+          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin
+          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
           PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
-          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
+          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9001
           PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb
-          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
-          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
+          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin
+          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
           PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_REGION: us-east-1
-          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
+          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9001
           PEERDB_SNOWFLAKE_AWS_S3_BUCKET_NAME: peerdb
           TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
           TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
@@ -450,7 +453,7 @@ jobs:
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
           PEERDB_CATALOG_HOST: localhost
-          PEERDB_CATALOG_PORT: 5432
+          PEERDB_CATALOG_PORT: 9901
           PEERDB_CATALOG_USER: postgres
           PEERDB_CATALOG_PASSWORD: postgres
           PEERDB_CATALOG_DATABASE: postgres
@@ -464,6 +467,7 @@ jobs:
           CI_MONGO_URI: mongodb://localhost:27017
           CI_MONGO_USERNAME: "csuser"
           CI_MONGO_PASSWORD: "cspass"
+          TOXIPROXY_HOST: localhost
           TOXIPROXY_POSTGRES_HOST: catalog
           ENABLE_OTEL_METRICS: ${{ (matrix.db-version.pg == '16' || matrix.db-version.mysql == 'mysql-pos') && 'true' || 'false' }}
           OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: http://localhost:4317

flow/.gitignore

@@ -1,5 +1,6 @@
 *.sh
 *.test
+!.env.test
 coverage.out
 # intellij files
 .idea/

flow/e2e/.env.test

@@ -0,0 +1,18 @@
+AWS_ENDPOINT_URL_S3=http://host.docker.internal:9001
+AWS_ACCESS_KEY_ID=_peerdb_minioadmin
+AWS_SECRET_ACCESS_KEY=_peerdb_minioadmin
+AWS_REGION=us-east-1
+AWS_ENDPOINT_URL_S3_TLS=https://host.docker.internal:9003
+PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID=_peerdb_minioadmin
+PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY=_peerdb_minioadmin
+PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION=us-east-1
+PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3=http://host.docker.internal:9001
+PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME=peerdb
+PEERDB_CATALOG_HOST=host.docker.internal
+PEERDB_CATALOG_PORT=9901
+PEERDB_CATALOG_USER=postgres
+PEERDB_CATALOG_PASSWORD=postgres
+PEERDB_CATALOG_DATABASE=postgres
+TOXIPROXY_HOST=host.docker.internal
+TOXIPROXY_POSTGRES_HOST=host.docker.internal
+TZ=UTC

flow/e2e/clickhouse.go

@@ -70,8 +70,8 @@ func (s ClickHouseSuite) Peer() *protos.Peer {
 			Type: protos.DBType_CLICKHOUSE,
 			Config: &protos.Peer_ClickhouseConfig{
 				ClickhouseConfig: &protos.ClickhouseConfig{
-					Host:       "localhost",
-					Port:       9001,
+					Host:       "host.docker.internal",
+					Port:       29001,
 					Database:   dbname,
 					DisableTls: true,
 					S3:         s.s3Helper.S3Config,
@@ -93,7 +93,7 @@ func (s ClickHouseSuite) PeerForDatabase(dbname string) *protos.Peer {
 		Type: protos.DBType_CLICKHOUSE,
 		Config: &protos.Peer_ClickhouseConfig{
 			ClickhouseConfig: &protos.ClickhouseConfig{
-				Host:       "localhost",
+				Host:       "host.docker.internal",
 				Port:       9000,
 				Database:   dbname,
 				DisableTls: true,

flow/e2e/pg.go

@@ -305,7 +305,8 @@ func SetupPostgresWithToxiproxy(t *testing.T, suffix string, port uint32) (*Post
 
 	// Create config pointing to proxy
 	config := internal.GetCatalogPostgresConfigFromEnv(t.Context())
-	config.Host = "localhost"
+
+	config.Host = os.Getenv("TOXIPROXY_HOST")
 	config.Port = port
 	// Don't set RequireTls - let it use the default from env
 

flow/e2e/test_utils.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"os"
 	"slices"
 	"strings"
 	"testing"
@@ -37,9 +38,16 @@ import (
 )
 
 func init() {
-	// it's okay if the .env file is not present
-	// we will use the default values
-	_ = godotenv.Load()
+	// First read the environment from .env (gitignored, may not exist)
+	// Then the default one from .env.test
+	envPath := ".env"
+	if _, err := os.Stat(envPath); err == nil {
+		_ = godotenv.Load(".env", ".env.test")
+	} else if errors.Is(err, os.ErrNotExist) {
+		_ = godotenv.Load(".env.test")
+	} else {
+		panic(err)
+	}
 }
 
 type Suite interface {