feat: implement multi header support

README.md

@@ -318,7 +318,7 @@ Tests can be altered using four lists:
   - `protocol`: overrides the protocol
   - `uri`: overrides the uri
   - `version`: overrides the HTTP version. E.g. "HTTP/1.1"
-  - `headers`: overrides headers, the format is a map of strings
+  - `ordered_headers`: overrides headers, the format is a list of `name` / `value` pairs
   - `method`: overrides the method used to perform the request
   - `data`: overrides data sent in the request
   - `autocomplete_headers`: overrides header autocompletion (currently sets `Connection: close` and `Content-Length` for requests with body data)

config/types.go

@@ -7,9 +7,8 @@ import (
 	"fmt"
 	"regexp"
 
-	schema "github.com/coreruleset/ftw-tests-schema/v2/types/overrides"
-
-	"github.com/coreruleset/go-ftw/ftwhttp"
+	schema "github.com/coreruleset/ftw-tests-schema/v2/types"
+	overrides_schema "github.com/coreruleset/ftw-tests-schema/v2/types/overrides"
 )
 
 // RunMode represents the mode of the test run
@@ -53,8 +52,8 @@ type FTWConfiguration struct {
 }
 
 type PlatformOverrides struct {
-	schema.FTWOverrides
-	OverridesMap map[uint][]*schema.TestOverride
+	overrides_schema.FTWOverrides
+	OverridesMap map[uint][]*overrides_schema.TestOverride
 }
 
 // FTWTestOverride holds four lists:
@@ -72,15 +71,17 @@ type FTWTestOverride struct {
 
 // Overrides represents the overridden inputs that have to be applied to tests
 type Overrides struct {
-	DestAddr   *string        `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
-	Port       *int           `yaml:"port,omitempty" koanf:"port,omitempty"`
-	Protocol   *string        `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
-	URI        *string        `yaml:"uri,omitempty" koanf:"uri,omitempty"`
-	Version    *string        `yaml:"version,omitempty" koanf:"version,omitempty"`
-	Headers    ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
-	Method     *string        `yaml:"method,omitempty" koanf:"method,omitempty"`
-	Data       *string        `yaml:"data,omitempty" koanf:"data,omitempty"`
-	SaveCookie *bool          `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
+	DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
+	Port     *int    `yaml:"port,omitempty" koanf:"port,omitempty"`
+	Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
+	URI      *string `yaml:"uri,omitempty" koanf:"uri,omitempty"`
+	Version  *string `yaml:"version,omitempty" koanf:"version,omitempty"`
+	// Deprecated: use OrderedHeaders instead
+	Headers        map[string]string    `yaml:"headers,omitempty" koanf:"headers,omitempty"`
+	OrderedHeaders []schema.HeaderTuple `yaml:"ordered_headers,omitempty" koanf:"ordered_headers,omitempty"`
+	Method         *string              `yaml:"method,omitempty" koanf:"method,omitempty"`
+	Data           *string              `yaml:"data,omitempty" koanf:"data,omitempty"`
+	SaveCookie     *bool                `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
 	// Deprecated: replaced with AutocompleteHeaders
 	StopMagic               *bool   `yaml:"stop_magic" koanf:"stop_magic,omitempty"`
 	AutocompleteHeaders     *bool   `yaml:"autocomplete_headers" koanf:"autocomplete_headers,omitempty"`

ftwhttp/client_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	header_names "github.com/coreruleset/go-ftw/ftwhttp/header_names"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/suite"
 	"golang.org/x/time/rate"
@@ -136,7 +137,11 @@ func (s *clientTestSuite) TestGetTrackedTime() {
 		Version: "HTTP/1.1",
 	}
 
-	h := Header{"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost"}
+	h := NewHeaderWithEntries([]*HeaderTuple{
+		{"Accept", "*/*"},
+		{"User-Agent", "go-ftw test agent"},
+		{"Host", "localhost"},
+	})
 
 	data := []byte(`test=me&one=two&one=twice`)
 	req := NewRequest(rl, h, data, true)
@@ -170,10 +175,11 @@ func (s *clientTestSuite) TestClientMultipartFormDataRequest() {
 		Version: "HTTP/1.1",
 	}
 
-	h := Header{
-		"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost",
-		"Content-Type": "multipart/form-data; boundary=--------397236876",
-	}
+	h := NewHeader()
+	h.Add("Accept", "*/*")
+	h.Add("User-Agent", "go-ftw test agent")
+	h.Add("Host", "localhost")
+	h.Add(header_names.ContentType, "multipart/form-data; boundary=--------397236876")
 
 	data := []byte(`----------397236876
 Content-Disposition: form-data; name="fileRap"; filename="test.txt"
@@ -255,7 +261,10 @@ func (s *clientTestSuite) TestClientRateLimits() {
 		Version: "HTTP/1.1",
 	}
 
-	h := Header{"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost"}
+	h := NewHeader()
+	h.Add("Accept", "*/*")
+	h.Add("User-Agent", "go-ftw test agent")
+	h.Add("Host", "localhost")
 	req := NewRequest(rl, h, nil, true)
 
 	// We need to do at least 2 calls so there is a wait between both.

ftwhttp/connection_test.go

@@ -39,7 +39,10 @@ func (s *connectionTestSuite) TestMultipleRequestTypes() {
 		Version: "HTTP/1.1",
 	}
 
-	h := Header{"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost"}
+	h := NewHeader()
+	h.Add("Accept", "*/*")
+	h.Add("User-Agent", "go-ftw test agent")
+	h.Add("Host", "localhost")
 
 	data := []byte(`test=me&one=two`)
 	req = NewRequest(rl, h, data, true)

ftwhttp/header.go

@@ -4,149 +4,161 @@
 package ftwhttp
 
 import (
-	"bytes"
+	"bufio"
 	"io"
 	"net/textproto"
-	"sort"
+	"strings"
 
 	"github.com/rs/zerolog/log"
 )
 
 const (
-	// ContentTypeHeader gives you the string for content type
-	ContentTypeHeader string = "Content-Type"
+	// HeaderSeparator is used to separate header name and value
+	HeaderSeparator = ": "
+	// HeaderDelimiter marks then end of a header (CRLF)
+	HeaderDelimiter = "\r\n"
 )
 
-// Based on https://golang.org/src/net/http/header.go
-
-// Header is a simplified version of headers, where there is only one header per key.
-// The original golang stdlib uses a proper string slice to map this.
-type Header map[string]string
-
-// stringWriter implements WriteString on a Writer.
-type stringWriter struct {
-	w io.Writer
+// Header is a representation of the HTTP header section.
+// It holds an ordered list of HeaderTuples.
+type Header struct {
+	canonicalNames map[string]uint
+	entries        []HeaderTuple
 }
 
-// WriteString writes the string on a Writer
-func (w stringWriter) WriteString(s string) (n int, err error) {
-	return w.w.Write([]byte(s))
+// HeaderTuple is a representation of an HTTP header. It consists
+// of a name and value.
+type HeaderTuple struct {
+	Name  string
+	Value string
 }
 
-// Add adds the (key, value) pair to the headers if it does not exist
-// The key is case-insensitive
-func (h Header) Add(key, value string) {
-	if h.Get(key) == "" {
-		h.Set(key, value)
+// Creates an empty Header. You should not initialize the struct directly.
+func NewHeader() *Header {
+	return &Header{
+		canonicalNames: map[string]uint{},
+		entries:        []HeaderTuple{},
 	}
 }
 
-// Set sets the header entries associated with key to
-// the single element value. It replaces any existing
-// values associated with a case-insensitive key.
-func (h Header) Set(key, value string) {
-	h.Del(key)
-	h[key] = value
-}
-
-// Get gets the value associated with the given key.
-// If there are no values associated with the key, Get returns "".
-// The key is case-insensitive
-func (h Header) Get(key string) string {
-	if h == nil {
-		return ""
+// Creates an empty Header. You should not initialize the struct directly.
+func NewHeaderWithEntries(entries []*HeaderTuple) *Header {
+	header := NewHeader()
+	for _, tuple := range entries {
+		header.Add(tuple.Name, tuple.Value)
 	}
-	v := h[h.getKeyMatchingCanonicalKey(key)]
-
-	return v
-}
-
-// Value is a wrapper to Get
-func (h Header) Value(key string) string {
-	return h.Get(key)
+	return header
 }
 
-// Del deletes the value associated with key.
-// The key is case-insensitive
-func (h Header) Del(key string) {
-	delete(h, h.getKeyMatchingCanonicalKey(key))
-}
-
-// Write writes a header in wire format.
-func (h Header) Write(w io.Writer) error {
-	ws, ok := w.(io.StringWriter)
+// Add a new HTTP header to the Header.
+func (h *Header) Add(name string, value string) {
+	key := canonicalKey(name)
+	count, ok := h.canonicalNames[key]
 	if !ok {
-		ws = stringWriter{w}
+		count = 0
 	}
+	h.canonicalNames[key] = count + 1
+	h.entries = append(h.entries, HeaderTuple{name, value})
+}
 
-	sorted := h.getSortedHeadersByName()
-
-	for _, key := range sorted {
-		// we want all headers "as-is"
-		s := key + ": " + h[key] + "\r\n"
-		if _, err := ws.WriteString(s); err != nil {
-			return err
+// Set replaces any existing HTTP headers of the same canonical
+// name with this new entry.
+func (h *Header) Set(name string, value string) {
+	key := canonicalKey(name)
+	retainees := []HeaderTuple{}
+	for _, tuple := range h.entries {
+		if canonicalKey(tuple.Name) != key {
+			retainees = append(retainees, tuple)
 		}
 	}
+	h.entries = retainees
+	h.Add(name, value)
+}
 
-	return nil
+// Returns true if the Header contains any HTTP header that
+// matches the canonical name.
+func (h *Header) HasAny(name string) bool {
+	key := canonicalKey(name)
+	_, ok := h.canonicalNames[key]
+	return ok
+}
 
+// Returns true if the Header contains any HTTP header that
+// matches the canonical name and canoncial value.
+// Values are compared using strings.EqualFold.
+func (h *Header) HasAnyValue(name string, value string) bool {
+	identity := func(a string) string { return a }
+	return h.hasAnyValue(name, value, identity, identity, strings.EqualFold)
 }
 
-// WriteBytes writes a header in a ByteWriter.
-func (h Header) WriteBytes(b *bytes.Buffer) (int, error) {
-	sorted := h.getSortedHeadersByName()
-	count := 0
-	for _, key := range sorted {
-		// we want all headers "as-is"
-		s := key + ": " + h[key] + "\r\n"
-		log.Trace().Msgf("Writing header: %s", s)
-		n, err := b.Write([]byte(s))
-		count += n
-		if err != nil {
-			return count, err
-		}
-	}
+// Returns true if the Header contains any HTTP header that
+// matches the canonical name and has a value containing the
+// specified substring.
+// Both, the header value and the search string are lower-cased
+// before performing the search.
+func (h *Header) HasAnyValueContaining(name string, value string) bool {
+	return h.hasAnyValue(name, value, strings.ToLower, strings.ToLower, strings.Contains)
+}
 
-	return count, nil
+// Returns all HeaderTuples that match the canonical header name.
+// If no matches are found the returned array will be empty.
+func (h *Header) GetAll(name string) []HeaderTuple {
+	return h.getAll(canonicalKey(name), canonicalKey)
 }
 
-// Clone returns a copy of h
-func (h Header) Clone() Header {
-	clone := make(Header)
+// Write writes the header to the provided writer
+func (h *Header) Write(writer io.Writer) error {
+	buf := bufio.NewWriter(writer)
+	for index, tuple := range h.entries {
+		if log.Trace().Enabled() {
+			log.Trace().Msgf("Writing header %d: %s: %s", index, tuple.Name, tuple.Value)
+		}
+		if _, err := buf.WriteString(tuple.Name + HeaderSeparator + tuple.Value + HeaderDelimiter); err != nil {
+			return err
+		}
 
-	for n, v := range h {
-		clone[n] = v
 	}
-
-	return clone
+	return buf.Flush()
 }
 
-// sortHeadersByName gets headers sorted by name
-// This way the output is predictable, for tests
-func (h Header) getSortedHeadersByName() []string {
-	keys := make([]string, 0, len(h))
-	for k := range h {
-		keys = append(keys, k)
+// Creates a clone of the Header.
+// If the Header is nil or empty, a non-nil empty Header will be returned.
+func (h *Header) Clone() *Header {
+	newHeader := NewHeader()
+	if h == nil {
+		return newHeader
+	}
+	for _, tuple := range h.entries {
+		newHeader.Add(tuple.Name, tuple.Value)
 	}
-	sort.Strings(keys)
+	return newHeader
+}
 
-	return keys
+func canonicalKey(key string) string {
+	return textproto.CanonicalMIMEHeaderKey(key)
 }
 
-// getKeyMatchingCanonicalKey finds a key matching with the given one, provided both are canonicalised
-func (h Header) getKeyMatchingCanonicalKey(searchKey string) string {
-	searchKey = canonicalKey(searchKey)
-	for k := range h {
-		if searchKey == canonicalKey(k) {
-			return k
+func (h *Header) getAll(name string, canonicalizer func(key string) string) []HeaderTuple {
+	matches := []HeaderTuple{}
+	for _, tuple := range h.entries {
+		if canonicalizer(tuple.Name) == name {
+			matches = append(matches, tuple)
 		}
 	}
 
-	return ""
+	return matches
 }
 
-// canonicalKey transforms given to the canonical form
-func canonicalKey(key string) string {
-	return textproto.CanonicalMIMEHeaderKey(key)
+func (h *Header) hasAnyValue(name string, value string, valueTransfomer func(a string) string, needleTransformer func(a string) string, comparator func(a string, b string) bool) bool {
+	key := canonicalKey(name)
+	if _, ok := h.canonicalNames[key]; !ok {
+		return ok
+	}
+	transformedValue := valueTransfomer(value)
+	for _, tuple := range h.entries {
+		if canonicalKey(tuple.Name) == key && comparator(needleTransformer(tuple.Value), transformedValue) {
+			return true
+		}
+	}
+	return false
 }

ftwhttp/header_names/constants.go

@@ -0,0 +1,7 @@
+package header_names
+
+const (
+	Connection    = "Connection"
+	ContentType   = "Content-Type"
+	ContentLength = "Content-Length"
+)