feat: add robots.txt file

.github/workflows/app-pull-request.yml

@@ -63,7 +63,7 @@ jobs:
           working-directory: ./app
 
       - name: Test
-        run: go test ./...
+        run: make test
 
       - name: Docker build
-        run: docker build --tag cds-superset-docs --platform linux/arm64 .
+        run: make docker

app/Makefile

@@ -1,8 +1,20 @@
+docker:
+	docker build --tag cds-superset-docs --platform linux/arm64 .
+
+fmt:
+	go fmt ./...
+
 install:
 	go mod download
 
+lint:
+	golangci-lint run ./...
+
 run: install
 	set -a && . $$(pwd)/.env && set +a &&\
 	go run cmd/server/main.go
 
-.PHONY: install run
+test:
+	go test -v ./...
+
+.PHONY: docker fmt install lint run test

app/cmd/server/main.go

@@ -38,6 +38,7 @@ func main() {
 
 	// Set up routes
 	http.Handle("/static/", http.StripPrefix("/static/", handlers.NewStaticHandler("static")))
+	http.Handle("/robots.txt", middleware.SecurityHeaders(handlers.NewRobotsHandler()))
 	http.Handle("/", middleware.SecurityHeaders(handlers.NewPageHandler(siteNames, wordPressClient)))
 
 	// Determine if this is a Lambda or HTTP server startup

app/internal/handlers/robots.go

@@ -0,0 +1,30 @@
+package handlers
+
+import (
+	"log"
+	"net/http"
+)
+
+// RobotsHandler serves the robots.txt file to control web crawler behavior
+type RobotsHandler struct {
+	robotsContent []byte
+}
+
+// NewRobotsHandler creates a new handler for robots.txt requests
+func NewRobotsHandler() *RobotsHandler {
+	return &RobotsHandler{
+		robotsContent: []byte("User-agent: *\nDisallow: /"),
+	}
+}
+
+// ServeHTTP implements the http.Handler interface
+func (h *RobotsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=604800")
+
+	w.WriteHeader(http.StatusOK)
+	_, err := w.Write(h.robotsContent)
+	if err != nil {
+		log.Printf("Error writing robots.txt response: %v", err)
+	}
+}

app/internal/handlers/robots_test.go

@@ -0,0 +1,115 @@
+package handlers
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// TestNewRobotsHandler verifies that the NewRobotsHandler function
+// initializes a RobotsHandler with the correct default content
+func TestNewRobotsHandler(t *testing.T) {
+	handler := NewRobotsHandler()
+
+	expectedContent := "User-agent: *\nDisallow: /"
+	actualContent := string(handler.robotsContent)
+
+	if actualContent != expectedContent {
+		t.Errorf("Expected robots content to be %q, got %q", expectedContent, actualContent)
+	}
+}
+
+// TestRobotsHandlerServeHTTP verifies that the RobotsHandler responds correctly to HTTP requests
+func TestRobotsHandlerServeHTTP(t *testing.T) {
+	handler := NewRobotsHandler()
+
+	// Create a test HTTP request
+	req, err := http.NewRequest("GET", "/robots.txt", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ResponseRecorder to record the response
+	rr := httptest.NewRecorder()
+
+	// Call the handler function
+	handler.ServeHTTP(rr, req)
+
+	// Check the status code
+	if status := rr.Code; status != http.StatusOK {
+		t.Errorf("Handler returned wrong status code: got %v want %v", status, http.StatusOK)
+	}
+
+	// Check the content type
+	expectedContentType := "text/plain; charset=utf-8"
+	if contentType := rr.Header().Get("Content-Type"); contentType != expectedContentType {
+		t.Errorf("Handler returned wrong content type: got %v want %v", contentType, expectedContentType)
+	}
+
+	// Check the cache control header
+	expectedCacheControl := "public, max-age=604800"
+	if cacheControl := rr.Header().Get("Cache-Control"); cacheControl != expectedCacheControl {
+		t.Errorf("Handler returned wrong cache control header: got %v want %v", cacheControl, expectedCacheControl)
+	}
+
+	// Check the response body
+	expectedBody := "User-agent: *\nDisallow: /"
+	if body := rr.Body.String(); body != expectedBody {
+		t.Errorf("Handler returned unexpected body: got %v want %v", body, expectedBody)
+	}
+}
+
+// TestRobotsHandlerWithCustomContent verifies that the RobotsHandler works with custom robots.txt content
+func TestRobotsHandlerWithCustomContent(t *testing.T) {
+	customContent := "User-agent: Googlebot\nAllow: /\nUser-agent: *\nDisallow: /private/"
+
+	handler := &RobotsHandler{
+		robotsContent: []byte(customContent),
+	}
+
+	req, err := http.NewRequest("GET", "/robots.txt", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr := httptest.NewRecorder()
+	handler.ServeHTTP(rr, req)
+
+	if status := rr.Code; status != http.StatusOK {
+		t.Errorf("Handler returned wrong status code: got %v want %v", status, http.StatusOK)
+	}
+
+	if body := rr.Body.String(); body != customContent {
+		t.Errorf("Handler returned unexpected body: got %v want %v", body, customContent)
+	}
+}
+
+// TestRobotsHandlerDifferentMethods checks that the handler responds the same way
+// regardless of HTTP method used
+func TestRobotsHandlerDifferentMethods(t *testing.T) {
+	methods := []string{"GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"}
+	handler := NewRobotsHandler()
+
+	for _, method := range methods {
+		t.Run(method, func(t *testing.T) {
+			req, err := http.NewRequest(method, "/robots.txt", nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			rr := httptest.NewRecorder()
+			handler.ServeHTTP(rr, req)
+
+			if status := rr.Code; status != http.StatusOK {
+				t.Errorf("%s: Handler returned wrong status code: got %v want %v",
+					method, status, http.StatusOK)
+			}
+
+			expectedBody := "User-agent: *\nDisallow: /"
+			if body := rr.Body.String(); body != expectedBody {
+				t.Errorf("%s: Handler returned unexpected body: got %v want %v",
+					method, body, expectedBody)
+			}
+		})
+	}
+}