feat: add s3 and simplify API

Signed-off-by: Donnie Adams <[email protected]>

Author: thedadams

README.md

@@ -1 +1,24 @@
-# workspace-provider
+# workspace-provider
+
+There are two providers that can be used to create and manage workspaces: directory and S3.
+
+## Directory
+
+The directory provider provides a directory-based workspace. This provider is used by default.
+
+## S3
+
+The S3 provider provides a S3-based workspace.
+
+This provider supports loading the default AWS configuration. You can control this configuration using the following environment variables:
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+- `AWS_REGION`
+
+You must set the following environment variables:
+- `WORKSPACE_PROVIDER_S3_BUCKET`
+
+### Usage with S3-compatible providers (e.g. Cloudflare R2)
+
+You can use the above referenced AWS environment variables to configure the S3 provider, setting the value of the environment variable to the corresponding value from your provider.
+Additionally, you should also set the `WORKSPACE_PROVIDER_S3_BASE_ENDPOINT` environment variable to the endpoint of your provider. For example, if you are using Cloudflare R2, you can set `WORKSPACE_PROVIDER_S3_BASE_ENDPOINT` to `https://<ACCOUND_ID>.r2.cloudflarestorage.com`.

go.mod

@@ -4,13 +4,31 @@ go 1.23.2
 
 require (
 	github.com/adrg/xdg v0.5.0
+	github.com/aws/aws-sdk-go-v2 v1.32.2
+	github.com/aws/aws-sdk-go-v2/config v1.27.43
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3
 	github.com/google/uuid v1.6.0
 	github.com/gptscript-ai/cmd v0.0.0-20240907001148-ffd49061124a
 	github.com/gptscript-ai/go-gptscript v0.9.5-rc3.0.20240820135702-14a861da41fa
 	github.com/spf13/cobra v1.8.1
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
+	github.com/aws/smithy-go v1.22.0 // indirect
 	github.com/getkin/kin-openapi v0.124.0 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.8 // indirect

go.sum

@@ -1,5 +1,41 @@
 github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY=
 github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4=
+github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
+github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
+github.com/aws/aws-sdk-go-v2/config v1.27.43 h1:p33fDDihFC390dhhuv8nOmX419wjOSDQRb+USt20RrU=
+github.com/aws/aws-sdk-go-v2/config v1.27.43/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3 h1:xxHGZ+wUgZNACQmxtdvP5tgzfsxGS3vPpTP5Hy3iToE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo=
+github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
+github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/cli/cpfile.go

@@ -2,18 +2,14 @@ package cli
 
 import (
 	"context"
-	"io"
 	"os"
 	"path/filepath"
 
-	"github.com/gptscript-ai/workspace-provider/pkg/client"
 	"github.com/spf13/cobra"
 )
 
 type cpFile struct {
 	root *workspaceProvider
-
-	client.WriteOptions
 }
 
 func (c *cpFile) Customize(cmd *cobra.Command) {
@@ -40,12 +36,5 @@ func (c *cpFile) copyFile(ctx context.Context, workspaceID, src string) error {
 	}
 	defer source.Close()
 
-	file, err := c.root.client.WriteFile(ctx, workspaceID, filepath.Base(src), c.WriteOptions)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	_, err = io.Copy(file, source)
-	return err
+	return c.root.client.WriteFile(ctx, workspaceID, filepath.Base(src), source)
 }

pkg/cli/ls.go

@@ -1,18 +1,16 @@
 package cli
 
 import (
-	"encoding/json"
 	"fmt"
+	"strings"
 
-	"github.com/gptscript-ai/workspace-provider/pkg/client"
 	"github.com/spf13/cobra"
 )
 
 type ls struct {
 	root *workspaceProvider
 
-	client.LsOptions
-	JSON bool `usage:"Output as JSON" env:"LS_JSON"`
+	Prefix string `usage:"Only list files with this prefix" env:"LS_PREFIX"`
 }
 
 func (l *ls) Customize(c *cobra.Command) {
@@ -22,42 +20,20 @@ func (l *ls) Customize(c *cobra.Command) {
 }
 
 func (l *ls) Run(cmd *cobra.Command, args []string) error {
-	var workspaceContents []client.WorkspaceContent
 	for _, arg := range args {
-		contents, err := l.root.client.Ls(cmd.Context(), arg, l.LsOptions)
+		contents, err := l.root.client.Ls(cmd.Context(), arg, l.Prefix)
 		if err != nil {
 			return err
 		}
 
-		if l.JSON {
-			workspaceContents = append(workspaceContents, contents)
-			continue
-		}
-
-		printContent(contents)
-	}
-
-	if l.JSON {
-		b, err := json.Marshal(workspaceContents)
-		if err != nil {
-			return fmt.Errorf("failed to marshal workspace contents: %w", err)
-		}
-
-		fmt.Println(string(b))
+		printContent(arg, contents)
 	}
 
 	return nil
 }
 
-func printContent(content client.WorkspaceContent) {
-	fmt.Printf("%s:\n", content.ID)
-	fmt.Printf("%s:\n", content.Path)
-	for _, child := range content.Children {
-		if child.FileName != "" {
-			fmt.Printf("%s\n", child.FileName)
-		} else if child.Path != "" {
-			printContent(child)
-		}
-	}
+func printContent(id string, content []string) {
+	fmt.Printf("%s:\n", id)
+	fmt.Println(strings.Join(content, "\n"))
 	fmt.Print("\n\n")
 }

pkg/cli/mkdir.go

@@ -0,0 +1,30 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+type removeAllWithPrefix struct {
+	root *workspaceProvider
+}
+
+func (r *removeAllWithPrefix) Customize(c *cobra.Command) {
+	c.Args = cobra.MinimumNArgs(2)
+	c.Use = "rm-with-prefix [OPTIONS] ID PREFIX..."
+	c.Short = "Remove all files with a given prefix"
+}
+
+func (r *removeAllWithPrefix) Run(cmd *cobra.Command, args []string) error {
+	workspaceID := args[0]
+	for _, arg := range args[1:] {
+		if err := r.root.client.RemoveAllWithPrefix(cmd.Context(), workspaceID, arg); err != nil {
+			return err
+		}
+
+		fmt.Printf("files with prefix %s deleted from workspace %s\n", arg, args[0])
+	}
+
+	return nil
+}

pkg/cli/removewithprefix.go

@@ -1,17 +1,14 @@
 package cli
 
 import (
-	"errors"
 	"fmt"
 	"strings"
 
-	"github.com/gptscript-ai/workspace-provider/pkg/client"
 	"github.com/spf13/cobra"
 )
 
 type rm struct {
-	root           *workspaceProvider
-	IgnoreNotFound bool `usage:"Ignore not found errors"`
+	root *workspaceProvider
 }
 
 func (r *rm) Customize(c *cobra.Command) {
@@ -31,11 +28,6 @@ func (r *rm) Run(cmd *cobra.Command, args []string) error {
 	for _, arg := range args {
 		fmt.Println(arg)
 		if err := r.root.client.Rm(cmd.Context(), arg); err != nil {
-			var notFound *client.WorkspaceNotFoundError
-			if r.IgnoreNotFound && errors.As(err, &notFound) {
-				fmt.Printf("workspace %s not found\n", arg)
-				continue
-			}
 			return err
 		}
 

pkg/cli/rm.go

@@ -1,17 +1,13 @@
 package cli
 
 import (
-	"errors"
 	"fmt"
 
-	"github.com/gptscript-ai/workspace-provider/pkg/client"
 	"github.com/spf13/cobra"
 )
 
 type rmFile struct {
 	root *workspaceProvider
-
-	IgnoreNotFound bool `usage:"Ignore not found errors"`
 }
 
 func (r *rmFile) Customize(c *cobra.Command) {
@@ -24,11 +20,6 @@ func (r *rmFile) Run(cmd *cobra.Command, args []string) error {
 	workspaceID := args[0]
 	for _, arg := range args[1:] {
 		if err := r.root.client.DeleteFile(cmd.Context(), workspaceID, arg); err != nil {
-			var notFound *client.FileNotFoundError
-			if r.IgnoreNotFound && errors.As(err, &notFound) {
-				fmt.Printf("file %s not found in workspace %s\n", arg, workspaceID)
-				continue
-			}
 			return err
 		}