Implement ContainersAll, ContainerNamesAll, ObjectsAll and ObjectNamesAll

These wrap the non All versions using Limit and Marker to read all the items

Author: ncw

notes.txt

@@ -11,8 +11,6 @@ Rename to go-swift to match user agent string?
 
 Reconnect on auth error - 401 when token expires isn't tested
 
-Implement higher level read all files / containers which uses limit and marker to loop
-
 Make more api compatible with python cloudfiles?
 
 Retry operations on timeout / network errors?

swift.go

@@ -429,6 +429,64 @@ func (c *Connection) Containers(opts *ContainersOpts) ([]Container, error) {
 	return containers, err
 }
 
+// containersAllOpts makes a copy of opts if set or makes a new one and
+// overrides Limit and Marker
+func containersAllOpts(opts *ContainersOpts) *ContainersOpts {
+	var newOpts ContainersOpts
+	if opts != nil {
+		newOpts = *opts
+	}
+	if newOpts.Limit == 0 {
+		newOpts.Limit = 10000
+	}
+	newOpts.Marker = ""
+	return &newOpts
+}
+
+// ContainersAll is like Containers but it returns all the Containers
+//
+// It calls Containers multiple times using the Marker parameter
+//
+// It has a default Limit parameter but you may pass in your own
+func (c *Connection) ContainersAll(opts *ContainersOpts) ([]Container, error) {
+	opts = containersAllOpts(opts)
+	containers := make([]Container, 0)
+	for {
+		newContainers, err := c.Containers(opts)
+		if err != nil {
+			return nil, err
+		}
+		containers = append(containers, newContainers...)
+		if len(newContainers) < opts.Limit {
+			break
+		}
+		opts.Marker = newContainers[len(newContainers)-1].Name
+	}
+	return containers, nil
+}
+
+// ContainerNamesAll is like ContainerNamess but it returns all the Containers
+//
+// It calls ContainerNames multiple times using the Marker parameter
+//
+// It has a default Limit parameter but you may pass in your own
+func (c *Connection) ContainerNamesAll(opts *ContainersOpts) ([]string, error) {
+	opts = containersAllOpts(opts)
+	containers := make([]string, 0)
+	for {
+		newContainers, err := c.ContainerNames(opts)
+		if err != nil {
+			return nil, err
+		}
+		containers = append(containers, newContainers...)
+		if len(newContainers) < opts.Limit {
+			break
+		}
+		opts.Marker = newContainers[len(newContainers)-1]
+	}
+	return containers, nil
+}
+
 /* ------------------------------------------------------------ */
 
 // ObjectOpts is options for Objects() and ObjectNames()
@@ -548,6 +606,64 @@ func (c *Connection) Objects(container string, opts *ObjectsOpts) ([]Object, err
 	return objects, err
 }
 
+// objectsAllOpts makes a copy of opts if set or makes a new one and
+// overrides Limit and Marker
+func objectsAllOpts(opts *ObjectsOpts) *ObjectsOpts {
+	var newOpts ObjectsOpts
+	if opts != nil {
+		newOpts = *opts
+	}
+	if newOpts.Limit == 0 {
+		newOpts.Limit = 10000
+	}
+	newOpts.Marker = ""
+	return &newOpts
+}
+
+// ObjectsAll is like Objects but it returns all the Objects
+//
+// It calls Objects multiple times using the Marker parameter
+//
+// It has a default Limit parameter but you may pass in your own
+func (c *Connection) ObjectsAll(container string, opts *ObjectsOpts) ([]Object, error) {
+	opts = objectsAllOpts(opts)
+	objects := make([]Object, 0)
+	for {
+		newObjects, err := c.Objects(container, opts)
+		if err != nil {
+			return nil, err
+		}
+		objects = append(objects, newObjects...)
+		if len(newObjects) < opts.Limit {
+			break
+		}
+		opts.Marker = newObjects[len(newObjects)-1].Name
+	}
+	return objects, nil
+}
+
+// ObjectNamesAll is like ObjectNamess but it returns all the Objects
+//
+// It calls ObjectNames multiple times using the Marker parameter
+//
+// It has a default Limit parameter but you may pass in your own
+func (c *Connection) ObjectNamesAll(container string, opts *ObjectsOpts) ([]string, error) {
+	opts = objectsAllOpts(opts)
+	objects := make([]string, 0)
+	for {
+		newObjects, err := c.ObjectNames(container, opts)
+		if err != nil {
+			return nil, err
+		}
+		objects = append(objects, newObjects...)
+		if len(newObjects) < opts.Limit {
+			break
+		}
+		opts.Marker = newObjects[len(newObjects)-1]
+	}
+	return objects, nil
+}
+
 // Account contains information about this account.
 type Account struct {
 	BytesUsed  int64 // total number of bytes used

swift_test.go

@@ -144,6 +144,44 @@ func TestContainer(t *testing.T) {
 	//fmt.Println(headers)
 }
 
+func TestContainersAll(t *testing.T) {
+	containers1, err := c.ContainersAll(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containers2, err := c.Containers(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(containers1) != len(containers2) {
+		t.Fatal("Wrong length")
+	}
+	for i := range containers1 {
+		if containers1[i] != containers2[i] {
+			t.Fatal("Not the same")
+		}
+	}
+}
+
+func TestContainersAllWithLimit(t *testing.T) {
+	containers1, err := c.ContainersAll(&swift.ContainersOpts{Limit: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	containers2, err := c.Containers(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(containers1) != len(containers2) {
+		t.Fatal("Wrong length")
+	}
+	for i := range containers1 {
+		if containers1[i] != containers2[i] {
+			t.Fatal("Not the same")
+		}
+	}
+}
+
 func TestContainerUpdate(t *testing.T) {
 	err := c.ContainerUpdate(CONTAINER, m2.ContainerHeaders())
 	if err != nil {
@@ -176,6 +214,44 @@ func TestContainerNames(t *testing.T) {
 	// fmt.Println(containers)
 }
 
+func TestContainerNamesAll(t *testing.T) {
+	containers1, err := c.ContainerNamesAll(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containers2, err := c.ContainerNames(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(containers1) != len(containers2) {
+		t.Fatal("Wrong length")
+	}
+	for i := range containers1 {
+		if containers1[i] != containers2[i] {
+			t.Fatal("Not the same")
+		}
+	}
+}
+
+func TestContainerNamesAllWithLimit(t *testing.T) {
+	containers1, err := c.ContainerNamesAll(&swift.ContainersOpts{Limit: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	containers2, err := c.ContainerNames(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(containers1) != len(containers2) {
+		t.Fatal("Wrong length")
+	}
+	for i := range containers1 {
+		if containers1[i] != containers2[i] {
+			t.Fatal("Not the same")
+		}
+	}
+}
+
 func TestObjectPutString(t *testing.T) {
 	err := c.ObjectPutString(CONTAINER, OBJECT, CONTENTS, "")
 	if err != nil {
@@ -273,6 +349,28 @@ func TestObjectNames(t *testing.T) {
 	//fmt.Println(objects)
 }
 
+func TestObjectNamesAll(t *testing.T) {
+	objects, err := c.ObjectNamesAll(CONTAINER, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(objects) != 1 || objects[0] != OBJECT {
+		t.Error("Incorrect listing", objects)
+	}
+	//fmt.Println(objects)
+}
+
+func TestObjectNamesAllWithLimit(t *testing.T) {
+	objects, err := c.ObjectNamesAll(CONTAINER, &swift.ObjectsOpts{Limit: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(objects) != 1 || objects[0] != OBJECT {
+		t.Error("Incorrect listing", objects)
+	}
+	//fmt.Println(objects)
+}
+
 func TestObjects(t *testing.T) {
 	objects, err := c.Objects(CONTAINER, &swift.ObjectsOpts{Delimiter: '/'})
 	if err != nil {
@@ -289,6 +387,28 @@ func TestObjects(t *testing.T) {
 	// fmt.Println(objects)
 }
 
+func TestObjectsAll(t *testing.T) {
+	objects, err := c.ObjectsAll(CONTAINER, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(objects) != 1 || objects[0].Name != OBJECT {
+		t.Error("Incorrect listing", objects)
+	}
+	//fmt.Println(objects)
+}
+
+func TestObjectsAllWithLimit(t *testing.T) {
+	objects, err := c.ObjectsAll(CONTAINER, &swift.ObjectsOpts{Limit: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(objects) != 1 || objects[0].Name != OBJECT {
+		t.Error("Incorrect listing", objects)
+	}
+	//fmt.Println(objects)
+}
+
 func TestObjectNamesWithPath(t *testing.T) {
 	objects, err := c.ObjectNames(CONTAINER, &swift.ObjectsOpts{Delimiter: '/', Path: ""})
 	if err != nil {