Improve performance by using bufio with large objects by default

Author: cezarsa

dlo.go

@@ -12,16 +12,16 @@ type DynamicLargeObjectCreateFile struct {
 // DynamicLargeObjectCreateFile creates a dynamic large object
 // returning an object which satisfies io.Writer, io.Seeker, io.Closer
 // and io.ReaderFrom.  The flags are as passes to the
-// LargeObjectCreate method.
+// largeObjectCreate method.
 func (c *Connection) DynamicLargeObjectCreateFile(opts *LargeObjectOpts) (LargeObjectFile, error) {
-	lo, err := c.LargeObjectCreate(opts)
+	lo, err := c.largeObjectCreate(opts)
 	if err != nil {
 		return nil, err
 	}
 
-	return &DynamicLargeObjectCreateFile{
+	return withBuffer(opts, &DynamicLargeObjectCreateFile{
 		largeObjectCreateFile: *lo,
-	}, nil
+	}), nil
 }
 
 // DynamicLargeObjectCreate creates or truncates an existing dynamic

largeobjects.go

@@ -1,6 +1,7 @@
 package swift
 
 import (
+	"bufio"
 	"bytes"
 	"crypto/rand"
 	"crypto/sha1"
@@ -103,6 +104,7 @@ type LargeObjectOpts struct {
 	MinChunkSize     int64   // Minimum chunk size, automatically set for SLO's based on info
 	SegmentContainer string  // Name of the container to place segments
 	SegmentPrefix    string  // Prefix to use for the segments
+	NoBuffer         bool    // Prevents using a bufio.Writer to write segments
 }
 
 type LargeObjectFile interface {
@@ -113,12 +115,12 @@ type LargeObjectFile interface {
 	Flush() error
 }
 
-// LargeObjectCreate creates a large object at opts.Container, opts.ObjectName.
+// largeObjectCreate creates a large object at opts.Container, opts.ObjectName.
 //
 // opts.Flags can have the following bits set
 //   os.TRUNC  - remove the contents of the large object if it exists
 //   os.APPEND - write at the end of the large object
-func (c *Connection) LargeObjectCreate(opts *LargeObjectOpts) (*largeObjectCreateFile, error) {
+func (c *Connection) largeObjectCreate(opts *LargeObjectOpts) (*largeObjectCreateFile, error) {
 	var (
 		segmentPath      string
 		segmentContainer string
@@ -398,3 +400,50 @@ func (file *largeObjectCreateFile) writeSegment(buf []byte, writeSegmentIdx int,
 	}
 	return &Object{Name: segmentName, Bytes: int64(segmentSize), Hash: headers["Etag"]}, sizeToRead, nil
 }
+
+func withBuffer(opts *LargeObjectOpts, lo LargeObjectFile) LargeObjectFile {
+	if !opts.NoBuffer {
+		return &bufferedLargeObjectFile{
+			LargeObjectFile: lo,
+			bw:              bufio.NewWriterSize(lo, int(opts.ChunkSize)),
+		}
+	}
+	return lo
+}
+
+type bufferedLargeObjectFile struct {
+	LargeObjectFile
+	bw *bufio.Writer
+}
+
+func (blo *bufferedLargeObjectFile) Close() error {
+	err := blo.bw.Flush()
+	if err != nil {
+		return err
+	}
+	return blo.LargeObjectFile.Close()
+}
+
+func (blo *bufferedLargeObjectFile) Write(p []byte) (n int, err error) {
+	return blo.bw.Write(p)
+}
+
+func (blo *bufferedLargeObjectFile) Seek(offset int64, whence int) (int64, error) {
+	err := blo.bw.Flush()
+	if err != nil {
+		return 0, err
+	}
+	return blo.LargeObjectFile.Seek(offset, whence)
+}
+
+func (blo *bufferedLargeObjectFile) Size() int64 {
+	return blo.LargeObjectFile.Size() + int64(blo.bw.Buffered())
+}
+
+func (blo *bufferedLargeObjectFile) Flush() error {
+	err := blo.bw.Flush()
+	if err != nil {
+		return err
+	}
+	return blo.LargeObjectFile.Flush()
+}

slo.go

@@ -35,7 +35,7 @@ type swiftSegment struct {
 
 // StaticLargeObjectCreateFile creates a static large object returning
 // an object which satisfies io.Writer, io.Seeker, io.Closer and
-// io.ReaderFrom.  The flags are as passed to the LargeObjectCreate
+// io.ReaderFrom.  The flags are as passed to the largeObjectCreate
 // method.
 func (c *Connection) StaticLargeObjectCreateFile(opts *LargeObjectOpts) (LargeObjectFile, error) {
 	info, err := c.cachedQueryInfo()
@@ -49,13 +49,13 @@ func (c *Connection) StaticLargeObjectCreateFile(opts *LargeObjectOpts) (LargeOb
 	if realMinChunkSize > opts.MinChunkSize {
 		opts.MinChunkSize = realMinChunkSize
 	}
-	lo, err := c.LargeObjectCreate(opts)
+	lo, err := c.largeObjectCreate(opts)
 	if err != nil {
 		return nil, err
 	}
-	return &StaticLargeObjectCreateFile{
+	return withBuffer(opts, &StaticLargeObjectCreateFile{
 		largeObjectCreateFile: *lo,
-	}, nil
+	}), nil
 }
 
 // StaticLargeObjectCreate creates or truncates an existing static

swift_test.go

@@ -1987,6 +1987,7 @@ func TestDLOSegmentation(t *testing.T) {
 		ObjectName:  OBJECT,
 		ContentType: "image/jpeg",
 		ChunkSize:   6,
+		NoBuffer:    true,
 	}
 
 	testSegmentation(t, func() swift.LargeObjectFile {
@@ -2032,6 +2033,57 @@ func TestDLOSegmentation(t *testing.T) {
 	})
 }
 
+func TestDLOSegmentationBuffered(t *testing.T) {
+	opts := swift.LargeObjectOpts{
+		Container:   CONTAINER,
+		ObjectName:  OBJECT,
+		ContentType: "image/jpeg",
+		ChunkSize:   6,
+	}
+
+	testSegmentation(t, func() swift.LargeObjectFile {
+		out, err := c.DynamicLargeObjectCreate(&opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return out
+	}, []segmentTest{
+		{
+			writes:        []string{"0", "1", "2", "3", "4", "5", "6", "7", "8"},
+			expectedSegs:  []string{"012345", "678"},
+			expectedValue: "012345678",
+		},
+		{
+			writes:        []string{"012345", "012345"},
+			expectedSegs:  []string{"012345", "012345"},
+			expectedValue: "012345012345",
+		},
+		{
+			writes:        []string{"0123456", "0123456"},
+			expectedSegs:  []string{"012345", "6", "012345", "6"},
+			expectedValue: "01234560123456",
+		},
+		{
+			writes:        []string{"0123456", "0123456"},
+			seeks:         []int{-4, 0},
+			expectedSegs:  []string{"012012", "3456"},
+			expectedValue: "0120123456",
+		},
+		{
+			writes:        []string{"0123456", "0123456", "abcde"},
+			seeks:         []int{0, -11, 0},
+			expectedSegs:  []string{"012abc", "d", "e12345", "6"},
+			expectedValue: "012abcde123456",
+		},
+		{
+			writes:        []string{"0123456", "ab"},
+			seeks:         []int{-4, 0},
+			expectedSegs:  []string{"012ab5", "6"},
+			expectedValue: "012ab56",
+		},
+	})
+}
+
 func TestSLOCreate(t *testing.T) {
 	headers := swift.Headers{}
 	err := c.ContainerCreate(SEGMENTS_CONTAINER, headers)
@@ -2241,6 +2293,7 @@ func TestSLOMinChunkSize(t *testing.T) {
 		ContentType:  "image/jpeg",
 		ChunkSize:    6,
 		MinChunkSize: 0,
+		NoBuffer:     true,
 	}
 
 	testSLOSegmentation(t, func() swift.LargeObjectFile {
@@ -2260,6 +2313,7 @@ func TestSLOSegmentation(t *testing.T) {
 		ContentType:  "image/jpeg",
 		ChunkSize:    6,
 		MinChunkSize: 4,
+		NoBuffer:     true,
 	}
 	testSLOSegmentation(t, func() swift.LargeObjectFile {
 		out, err := c.StaticLargeObjectCreate(&opts)
@@ -2270,6 +2324,57 @@ func TestSLOSegmentation(t *testing.T) {
 	})
 }
 
+func TestSLOSegmentationBuffered(t *testing.T) {
+	opts := swift.LargeObjectOpts{
+		Container:    CONTAINER,
+		ObjectName:   OBJECT,
+		ContentType:  "image/jpeg",
+		ChunkSize:    6,
+		MinChunkSize: 4,
+	}
+	testSegmentation(t, func() swift.LargeObjectFile {
+		out, err := c.StaticLargeObjectCreate(&opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return out
+	}, []segmentTest{
+		{
+			writes:        []string{"0", "1", "2", "3", "4", "5", "6", "7", "8"},
+			expectedSegs:  []string{"012345", "678"},
+			expectedValue: "012345678",
+		},
+		{
+			writes:        []string{"012345", "012345"},
+			expectedSegs:  []string{"012345", "012345"},
+			expectedValue: "012345012345",
+		},
+		{
+			writes:        []string{"0123456", "0123456"},
+			expectedSegs:  []string{"012345", "601234", "56"},
+			expectedValue: "01234560123456",
+		},
+		{
+			writes:        []string{"0123456", "0123456"},
+			seeks:         []int{-4, 0},
+			expectedSegs:  []string{"012012", "3456"},
+			expectedValue: "0120123456",
+		},
+		{
+			writes:        []string{"0123456", "0123456", "abcde"},
+			seeks:         []int{0, -11, 0},
+			expectedSegs:  []string{"012abc", "de1234", "56"},
+			expectedValue: "012abcde123456",
+		},
+		{
+			writes:        []string{"0123456", "ab"},
+			seeks:         []int{-4, 0},
+			expectedSegs:  []string{"012ab5", "6"},
+			expectedValue: "012ab56",
+		},
+	})
+}
+
 func testSLOSegmentation(t *testing.T, createObj func() swift.LargeObjectFile) {
 	testCases := []segmentTest{
 		{