Implement 'variables' extensions and fix numerous other issues

README.md

@@ -8,11 +8,18 @@ implementation in Go.
 
 - envelope ([RFC 5228])
 - fileinto ([RFC 5228])
+- encoded-character ([RFC 5228])
 - imap4flags ([RFC 5232])
+- variables ([RFC 5229])
 
 ## Example
 
 See ./cmd/sieve-run.
 
+## Known issues
+
+- `:matches` `*` is greedy (RFC 5229 requires non-greedy matching).
+
 [RFC 5228]: https://datatracker.ietf.org/doc/html/rfc5228
+[RFC 5229]: https://datatracker.ietf.org/doc/html/rfc5229
 [RFC 5232]: https://datatracker.ietf.org/doc/html/rfc5232

go.mod

@@ -2,4 +2,11 @@ module github.com/foxcpp/go-sieve
 
 go 1.20
 
-require github.com/davecgh/go-spew v1.1.1
+require (
+	github.com/davecgh/go-spew v1.1.1
+)
+
+require (
+	github.com/emersion/go-message v0.18.0 // indirect
+	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
+)

go.sum

@@ -1,2 +1,38 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emersion/go-message v0.18.0 h1:7LxAXHRpSeoO/Wom3ZApVZYG7c3d17yCScYce8WiXA8=
+github.com/emersion/go-message v0.18.0/go.mod h1:Zi69ACvzaoV/MBnrxfVBPV3xWEuCmC2nEN39oJF4B8A=
+github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY=
+github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

interp/action.go

@@ -7,29 +7,30 @@ import (
 
 type CmdStop struct{}
 
-func (c CmdStop) Execute(ctx context.Context, d *RuntimeData) error {
+func (c CmdStop) Execute(_ context.Context, _ *RuntimeData) error {
 	return ErrStop
 }
 
 type CmdFileInto struct {
 	Mailbox string
-	Flags   *Flags
+	Flags   Flags
 }
 
-func (c CmdFileInto) Execute(ctx context.Context, d *RuntimeData) error {
+func (c CmdFileInto) Execute(_ context.Context, d *RuntimeData) error {
+	mailbox := expandVars(d, c.Mailbox)
 	found := false
 	for _, m := range d.Mailboxes {
-		if m == c.Mailbox {
+		if m == mailbox {
 			found = true
 		}
 	}
 	if found {
 		return nil
 	}
-	d.Mailboxes = append(d.Mailboxes, c.Mailbox)
+	d.Mailboxes = append(d.Mailboxes, mailbox)
 	d.ImplicitKeep = false
 	if c.Flags != nil {
-		d.Flags = *canonicalFlags(make([]string, len(*c.Flags)), nil, d.FlagAliases)
+		d.Flags = canonicalFlags(expandVarsList(d, c.Flags), nil, d.FlagAliases)
 	}
 	return nil
 }
@@ -39,14 +40,16 @@ type CmdRedirect struct {
 }
 
 func (c CmdRedirect) Execute(ctx context.Context, d *RuntimeData) error {
-	ok, err := d.Policy.RedirectAllowed(ctx, d, c.Addr)
+	addr := expandVars(d, c.Addr)
+
+	ok, err := d.Policy.RedirectAllowed(ctx, d, addr)
 	if err != nil {
 		return err
 	}
 	if !ok {
 		return nil
 	}
-	d.RedirectAddr = append(d.RedirectAddr, c.Addr)
+	d.RedirectAddr = append(d.RedirectAddr, addr)
 	d.ImplicitKeep = false
 
 	if len(d.RedirectAddr) > d.Script.opts.MaxRedirects {
@@ -56,13 +59,13 @@ func (c CmdRedirect) Execute(ctx context.Context, d *RuntimeData) error {
 }
 
 type CmdKeep struct {
-	Flags *Flags
+	Flags Flags
 }
 
 func (c CmdKeep) Execute(_ context.Context, d *RuntimeData) error {
 	d.Keep = true
 	if c.Flags != nil {
-		d.Flags = *canonicalFlags(make([]string, len(*c.Flags)), nil, d.FlagAliases)
+		d.Flags = canonicalFlags(expandVarsList(d, c.Flags), nil, d.FlagAliases)
 	}
 	return nil
 }
@@ -76,41 +79,43 @@ func (c CmdDiscard) Execute(_ context.Context, d *RuntimeData) error {
 }
 
 type CmdSetFlag struct {
-	Flags *Flags
+	Flags Flags
 }
 
 func (c CmdSetFlag) Execute(_ context.Context, d *RuntimeData) error {
 	if c.Flags != nil {
-		d.Flags = *canonicalFlags(*c.Flags, nil, d.FlagAliases)
+		d.Flags = canonicalFlags(expandVarsList(d, c.Flags), nil, d.FlagAliases)
 	}
 	return nil
 }
 
 type CmdAddFlag struct {
-	Flags *Flags
+	Flags Flags
 }
 
 func (c CmdAddFlag) Execute(_ context.Context, d *RuntimeData) error {
 	if c.Flags != nil {
+		flags := expandVarsList(d, c.Flags)
+
 		if d.Flags == nil {
-			d.Flags = make([]string, len(*c.Flags))
-			copy(d.Flags, *c.Flags)
+			d.Flags = make([]string, len(flags))
+			copy(d.Flags, flags)
 		} else {
 			// Use canonicalFlags to remove duplicates
-			d.Flags = *canonicalFlags(append(d.Flags, *c.Flags...), nil, d.FlagAliases)
+			d.Flags = canonicalFlags(append(d.Flags, flags...), nil, d.FlagAliases)
 		}
 	}
 	return nil
 }
 
 type CmdRemoveFlag struct {
-	Flags *Flags
+	Flags Flags
 }
 
 func (c CmdRemoveFlag) Execute(_ context.Context, d *RuntimeData) error {
 	if c.Flags != nil {
 		// Use canonicalFlags to remove duplicates
-		d.Flags = *canonicalFlags(d.Flags, c.Flags, d.FlagAliases)
+		d.Flags = canonicalFlags(d.Flags, expandVarsList(d, c.Flags), d.FlagAliases)
 	}
 	return nil
 }

interp/dovecot_testsuite.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io/fs"
 	"net/textproto"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -25,13 +26,20 @@ type CmdDovecotTest struct {
 func (c CmdDovecotTest) Execute(ctx context.Context, d *RuntimeData) error {
 	testData := d.Copy()
 	testData.testName = c.TestName
+	testData.testFailMessage = ""
 
 	d.Script.opts.T.Run(c.TestName, func(t *testing.T) {
+		for _, testName := range testData.Script.opts.DisabledTests {
+			if c.TestName == testName {
+				t.Skip("test is disabled by DisabledTests")
+			}
+		}
+
 		for _, cmd := range c.Cmds {
 			if err := cmd.Execute(ctx, testData); err != nil {
 				if errors.Is(err, ErrStop) {
 					if testData.testFailMessage != "" {
-						t.Error("test_fail called:", testData.testFailMessage)
+						t.Errorf("test_fail at %v called: %v", testData.testFailAt, testData.testFailMessage)
 					}
 					return
 				}
@@ -44,11 +52,37 @@ func (c CmdDovecotTest) Execute(ctx context.Context, d *RuntimeData) error {
 }
 
 type CmdDovecotTestFail struct {
+	At      lexer.Position
 	Message string
 }
 
-func (c CmdDovecotTestFail) Execute(ctx context.Context, d *RuntimeData) error {
-	d.testFailMessage = c.Message
+func (c CmdDovecotTestFail) Execute(_ context.Context, d *RuntimeData) error {
+	d.testFailMessage = expandVars(d, c.Message)
+	d.testFailAt = c.At
+	return ErrStop
+}
+
+type CmdDovecotConfigSet struct {
+	Unset bool
+	Key   string
+	Value string
+}
+
+func (c CmdDovecotConfigSet) Execute(_ context.Context, d *RuntimeData) error {
+	switch c.Key {
+	case "sieve_variables_max_variable_size":
+		if c.Unset {
+			d.Script.opts.MaxVariableLen = 4000
+		} else {
+			val, err := strconv.Atoi(c.Value)
+			if err != nil {
+				return err
+			}
+			d.Script.opts.MaxVariableLen = val
+		}
+	default:
+		return fmt.Errorf("unknown test_config_set key: %v", c.Key)
+	}
 	return nil
 }
 
@@ -57,7 +91,9 @@ type CmdDovecotTestSet struct {
 	VariableValue string
 }
 
-func (c CmdDovecotTestSet) Execute(ctx context.Context, d *RuntimeData) error {
+func (c CmdDovecotTestSet) Execute(_ context.Context, d *RuntimeData) error {
+	value := expandVars(d, c.VariableValue)
+
 	switch c.VariableName {
 	case "message":
 		r := textproto.NewReader(bufio.NewReader(strings.NewReader(c.VariableValue)))
@@ -71,14 +107,26 @@ func (c CmdDovecotTestSet) Execute(ctx context.Context, d *RuntimeData) error {
 			Header: msgHdr,
 		}
 	case "envelope.from":
+		value = strings.TrimSuffix(strings.TrimPrefix(value, "<"), ">")
+
 		d.Envelope = EnvelopeStatic{
-			From: c.VariableValue,
+			From: value,
 			To:   d.Envelope.EnvelopeTo(),
+			Auth: d.Envelope.AuthUsername(),
 		}
 	case "envelope.to":
+		value = strings.TrimSuffix(strings.TrimPrefix(value, "<"), ">")
+
 		d.Envelope = EnvelopeStatic{
 			From: d.Envelope.EnvelopeFrom(),
-			To:   c.VariableValue,
+			To:   value,
+			Auth: d.Envelope.AuthUsername(),
+		}
+	case "envelope.auth":
+		d.Envelope = EnvelopeStatic{
+			From: d.Envelope.EnvelopeFrom(),
+			To:   d.Envelope.EnvelopeTo(),
+			Auth: value,
 		}
 	default:
 		d.Variables[c.VariableName] = c.VariableValue
@@ -91,7 +139,7 @@ type TestDovecotCompile struct {
 	ScriptPath string
 }
 
-func (t TestDovecotCompile) Check(ctx context.Context, d *RuntimeData) (bool, error) {
+func (t TestDovecotCompile) Check(_ context.Context, d *RuntimeData) (bool, error) {
 	if d.Namespace == nil {
 		return false, fmt.Errorf("RuntimeData.Namespace is not set, cannot load scripts")
 	}
@@ -151,7 +199,7 @@ func (t TestDovecotRun) Check(ctx context.Context, d *RuntimeData) (bool, error)
 type TestDovecotTestError struct {
 }
 
-func (t TestDovecotTestError) Check(ctx context.Context, d *RuntimeData) (bool, error) {
+func (t TestDovecotTestError) Check(_ context.Context, _ *RuntimeData) (bool, error) {
 	// go-sieve has a very different error formatting and stops lexing/parsing/loading
 	// on first error, therefore we skip all test_errors checks as they are
 	// Pigeonhole-specific.