Merge branch 'next'

.env.example

@@ -1,14 +1,30 @@
-# TOBEY_DEBUG=false
-# TOBEY_SKIP_CACHE=false
+# Controls debug mode, uncomment to enable debug mode, by default it is disabled.
+# TOBEY_DEBUG=true 
 
-TOBEY_PROGRESS_DSN=http://progress:8080
-TOBEY_RABBITMQ_DSN=amqp://guest:guest@rabbitmq:5672/
-TOBEY_REDIS_DSN=redis:6379/0
+# Controls caching access, uncomment to disabled caching. By default caching is enabled.
+# TOBEY_SKIP_CACHE=true
 
-# TOBEY_REQS_PER_S=2
+# Controls the number of workers per instance, by default 5.
+# TOBEY_WORKERS=5
 
-# A space separated list of telemetry to send. Available telemetry: metrics,
-# traces. To disable telemetry provide an empty value. When enabling telemetry
-# appropriate OTLP endpoints must provided as well.
-TOBEY_TELEMETRY="traces"
-OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://jaeger:4318/v1/traces
+# DSN specifying where crawl results should be stored, by default no results are stored. Here we store results 
+# in the "results" directory, relative to the current working directory.
+TOBEY_RESULTS_DSN=disk://results 
+# TOBEY_RESULTS_DSN=webhook://host/path
+
+# DSN for progress reporting. By default, a console progress reporter is used. Uncomment to report progress to the 
+# Factorial service or disable progress reporting.
+# TOBEY_PROGRESS_DSN=factorial://localhost:8080 
+# TOBEY_PROGRESS_DSN=
+
+# If you have multiple instances of Tobey, you can use a Redis instance to coordinate the work queue. By default
+# no coordination is done, uncomment to enable coordination.
+# TOBEY_REDIS_DSN=redis://localhost:6379 
+
+# A space separated list of telemetry to send. Available telemetry: metrics, traces, pulse. By default no telemetry 
+# is send. Uncomment to enable metrics and traces.
+# TOBEY_TELEMETRY=metrics traces 
+
+# In order to send telemetry you need to provide the OTLP endpoints, as well.
+# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=
+# OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=

.env.example.factorial

@@ -0,0 +1,26 @@
+# Controls debug mode, uncomment to enable debug mode, by default it is disabled.
+# TOBEY_DEBUG=true 
+
+# Controls caching access, uncomment to disabled caching. By default caching is enabled.
+# TOBEY_SKIP_CACHE=true
+
+# DSN specifying where crawl results should be stored, by default no results are stored. Here we store results 
+# in the "results" directory, relative to the current working directory.
+# TOBEY_RESULTS_DSN=disk://results 
+TOBEY_RESULTS_DSN=webhook://?enable_dynamic_config
+
+# DSN for progress reporting. By default no progress is reported, uncomment to report progress to the 
+# Factorial service.
+TOBEY_PROGRESS_DSN=factorial://localhost:8080 
+
+# If you have multiple instances of Tobey, you can use a Redis instance to coordinate the work queue. By default
+# no coordination is done, uncomment to enable coordination.
+TOBEY_REDIS_DSN=redis:6379/0
+
+# A space separated list of telemetry to send. Available telemetry: metrics, traces, pulse. By default no telemetry 
+# is send. Uncomment to enable metrics and traces.
+TOBEY_TELEMETRY="traces"
+
+# In order to send telemetry you need to provide the OTLP endpoints, as well.
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://jaeger:4318/v1/traces
+OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=

.github/workflows/release.yml

@@ -0,0 +1,53 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
+
+    - name: Run Tests
+      run: go test -v ./...
+
+  release:
+    needs: test
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
+
+    - name: Build for Linux
+      run: |
+        GOOS=linux GOARCH=amd64 go build -o tobey-linux-amd64
+        GOOS=linux GOARCH=arm64 go build -o tobey-linux-arm64
+
+    - name: Build for macOS
+      run: |
+        GOOS=darwin GOARCH=amd64 go build -o tobey-darwin-amd64
+        GOOS=darwin GOARCH=arm64 go build -o tobey-darwin-arm64
+
+    - name: Create Release
+      uses: softprops/action-gh-release@v1
+      with:
+        files: |
+          tobey-linux-amd64
+          tobey-linux-arm64
+          tobey-darwin-amd64
+          tobey-darwin-arm64
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Tests
+
+on:
+  push:
+    branches: [ main, next ]
+  pull_request:
+    branches: [ main, next ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
+
+    - name: Run Tests
+      run: go test -v ./... 

.gitignore

@@ -1,2 +1,3 @@
 cache
 tobey
+.env

Makefile

@@ -1,10 +1,14 @@
 .PHONY: dev
 dev:
-	TOBEY_DEBUG=true go run .
+	TOBEY_SKIP_CACHE=true TOBEY_DEBUG=true TOBEY_RESULTS_DSN=disk:///tmp/tobey go run . -host 127.0.0.1
+
+.PHONY: pulse
+pulse:
+	go run cmd/pulse/main.go
 
 .PHONY: test
 test:
-	TOBEY_SKIP_CACHE=true TOBEY_DEBUG=true go test
+	TOBEY_SKIP_CACHE=true TOBEY_DEBUG=true go test -v ./...
 
 .PHONY: clean
 clean: