wip

Author: nicksanford

cli/data.go

@@ -602,7 +602,7 @@ func filenameForDownload(meta *datapb.BinaryMetadata) string {
 	}
 
 	// Replace reserved characters.
-	fileName = data.CaptureFilePathWithReplacedReservedChars(fileName)
+	fileName = data.FilePathWithReplacedReservedChars(fileName)
 
 	return fileName
 }

data/capture_buffer.go

@@ -15,8 +15,8 @@ type CaptureBufferedWriter interface {
 
 // CaptureBuffer is a persistent queue of SensorData backed by a series of *data.CaptureFile.
 type CaptureBuffer struct {
-	Directory          string
-	MetaData           *v1.DataCaptureMetadata
+	directory          string
+	metaData           *v1.DataCaptureMetadata
 	nextFile           *ProgFile
 	lock               sync.Mutex
 	maxCaptureFileSize int64
@@ -25,8 +25,8 @@ type CaptureBuffer struct {
 // NewCaptureBuffer returns a new Buffer.
 func NewCaptureBuffer(dir string, md *v1.DataCaptureMetadata, maxCaptureFileSize int64) *CaptureBuffer {
 	return &CaptureBuffer{
-		Directory:          dir,
-		MetaData:           md,
+		directory:          dir,
+		metaData:           md,
 		maxCaptureFileSize: maxCaptureFileSize,
 	}
 }
@@ -47,12 +47,13 @@ func isBinary(item *v1.SensorData) bool {
 		return false
 	}
 }
+
 func (b *CaptureBuffer) Write(item *v1.SensorData) error {
 	b.lock.Lock()
 	defer b.lock.Unlock()
 
 	if isBinary(item) {
-		binFile, err := NewProgFile(b.Directory, b.MetaData)
+		binFile, err := NewProgFile(b.directory, b.metaData)
 		if err != nil {
 			return err
 		}
@@ -66,19 +67,19 @@ func (b *CaptureBuffer) Write(item *v1.SensorData) error {
 	}
 
 	if b.nextFile == nil {
-		nextFile, err := NewProgFile(b.Directory, b.MetaData)
+		nextFile, err := NewProgFile(b.directory, b.metaData)
 		if err != nil {
 			return err
 		}
 		b.nextFile = nextFile
 		// We want to special case on "CaptureAllFromCamera" because it is sensor data that contains images
 		// and their corresponding annotations. We want each image and its annotations to be stored in a
 		// separate file.
-	} else if b.nextFile.Size() > b.maxCaptureFileSize || b.MetaData.MethodName == "CaptureAllFromCamera" {
+	} else if b.nextFile.Size() > b.maxCaptureFileSize || b.metaData.MethodName == "CaptureAllFromCamera" {
 		if err := b.nextFile.Close(); err != nil {
 			return err
 		}
-		nextFile, err := NewProgFile(b.Directory, b.MetaData)
+		nextFile, err := NewProgFile(b.directory, b.metaData)
 		if err != nil {
 			return err
 		}
@@ -104,5 +105,5 @@ func (b *CaptureBuffer) Flush() error {
 
 // Path returns the path to the directory containing the backing data capture files.
 func (b *CaptureBuffer) Path() string {
-	return b.Directory
+	return b.directory
 }

data/capture_buffer_test.go

@@ -267,7 +267,7 @@ func TestCaptureBufferReader(t *testing.T) {
 				test.That(t, err, test.ShouldBeNil)
 				defer func() { utils.UncheckedError(f.Close()) }()
 
-				cf, err := ReadCaptureFile(f)
+				cf, err := NewCaptureFile(f)
 				test.That(t, err, test.ShouldBeNil)
 				test.That(t, cf.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
 
@@ -335,7 +335,7 @@ func TestCaptureBufferReader(t *testing.T) {
 				test.That(t, err, test.ShouldBeNil)
 				defer func() { utils.UncheckedError(f2.Close()) }()
 
-				cf2, err := ReadCaptureFile(f2)
+				cf2, err := NewCaptureFile(f2)
 				test.That(t, err, test.ShouldBeNil)
 				test.That(t, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
 
@@ -470,7 +470,7 @@ func TestCaptureBufferReader(t *testing.T) {
 				test.That(t, err, test.ShouldBeNil)
 				defer func() { utils.UncheckedError(f.Close()) }()
 
-				cf, err := ReadCaptureFile(f)
+				cf, err := NewCaptureFile(f)
 				test.That(t, err, test.ShouldBeNil)
 				test.That(t, cf.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
 
@@ -508,7 +508,7 @@ func TestCaptureBufferReader(t *testing.T) {
 				test.That(t, err, test.ShouldBeNil)
 				defer func() { utils.UncheckedError(f2.Close()) }()
 
-				cf2, err := ReadCaptureFile(f2)
+				cf2, err := NewCaptureFile(f2)
 				test.That(t, err, test.ShouldBeNil)
 				test.That(t, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
 
@@ -563,7 +563,7 @@ func TestCaptureBufferReader(t *testing.T) {
 				f3, err := os.Open(filepath.Join(b.Path(), newFileNames[0]))
 				test.That(t, err, test.ShouldBeNil)
 				defer func() { utils.UncheckedError(f3.Close()) }()
-				cf3, err := ReadCaptureFile(f3)
+				cf3, err := NewCaptureFile(f3)
 				test.That(t, err, test.ShouldBeNil)
 				test.That(t, cf3.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
 				sd3, err := cf3.ReadNext()
@@ -576,7 +576,7 @@ func TestCaptureBufferReader(t *testing.T) {
 				f4, err := os.Open(filepath.Join(b.Path(), newFileNames[1]))
 				test.That(t, err, test.ShouldBeNil)
 				defer func() { utils.UncheckedError(f4.Close()) }()
-				cf4, err := ReadCaptureFile(f4)
+				cf4, err := NewCaptureFile(f4)
 				test.That(t, err, test.ShouldBeNil)
 				test.That(t, cf4.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
 				sd4, err := cf4.ReadNext()
@@ -644,7 +644,7 @@ func TestCaptureBufferReader(t *testing.T) {
 		test.That(t, err, test.ShouldBeNil)
 		defer func() { utils.UncheckedError(f.Close()) }()
 
-		cf2, err := ReadCaptureFile(f)
+		cf2, err := NewCaptureFile(f)
 		test.That(t, err, test.ShouldBeNil)
 		test.That(t, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
 
@@ -657,7 +657,75 @@ func TestCaptureBufferReader(t *testing.T) {
 	})
 }
 
-//nolint
+func NickTest(t *testing.T) {
+	tmpDir := t.TempDir()
+	name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
+	method := readImage
+	additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
+	tags := []string{"my", "tags"}
+	methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
+	test.That(t, err, test.ShouldBeNil)
+
+	readImageCaptureMetadata := BuildCaptureMetadata(
+		name.API,
+		name.ShortName(),
+		method,
+		additionalParams,
+		methodParams,
+		tags,
+	)
+
+	test.That(t, readImageCaptureMetadata, test.ShouldResemble, &v1.DataCaptureMetadata{
+		ComponentName:    "my-cam",
+		ComponentType:    "rdk:component:camera",
+		MethodName:       readImage,
+		MethodParameters: methodParams,
+		Tags:             tags,
+		Type:             v1.DataType_DATA_TYPE_BINARY_SENSOR,
+		FileExtension:    ".jpeg",
+	})
+
+	b := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))
+
+	// Path() is the same as the first paramenter passed to NewCaptureBuffer
+	test.That(t, b.Path(), test.ShouldResemble, tmpDir)
+	test.That(t, b.MetaData, test.ShouldResemble, readImageCaptureMetadata)
+
+	now := time.Now()
+	timeRequested := timestamppb.New(now.UTC())
+	timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
+	msg := &v1.SensorData{
+		Metadata: &v1.SensorMetadata{
+			TimeRequested: timeRequested,
+			TimeReceived:  timeReceived,
+		},
+		Data: &v1.SensorData_Binary{
+			Binary: []byte("this is a fake image"),
+		},
+	}
+	test.That(t, b.Write(msg), test.ShouldBeNil)
+	test.That(t, b.Flush(), test.ShouldBeNil)
+	dirEntries, err := os.ReadDir(b.Path())
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, len(dirEntries), test.ShouldEqual, 1)
+	test.That(t, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
+	f, err := os.Open(filepath.Join(b.Path(), dirEntries[0].Name()))
+	test.That(t, err, test.ShouldBeNil)
+	defer func() { test.That(t, f.Close(), test.ShouldBeNil) }()
+
+	cf2, err := NewCaptureFile(f)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
+
+	sd2, err := cf2.ReadNext()
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, sd2, test.ShouldResemble, msg)
+
+	_, err = cf2.ReadNext()
+	test.That(t, err, test.ShouldBeError, io.EOF)
+}
+
+// nolint
 func getCaptureFiles(dir string) (dcFiles, progFiles []string) {
 	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {

data/capture_file.go

@@ -39,18 +39,18 @@ const (
 // length delimited protobuf messages, where the first message is the CaptureMetadata for the file, and ensuing
 // messages contain the captured data.
 type CaptureFile struct {
-	path     string
-	lock     sync.Mutex
-	file     *os.File
-	size     int64
-	metadata *v1.DataCaptureMetadata
-
+	Metadata          *v1.DataCaptureMetadata
+	path              string
+	size              int64
 	initialReadOffset int64
-	readOffset        int64
+
+	lock       sync.Mutex
+	file       *os.File
+	readOffset int64
 }
 
-// ReadCaptureFile creates a File struct from a passed os.File previously constructed using NewFile.
-func ReadCaptureFile(f *os.File) (*CaptureFile, error) {
+// NewCaptureFile creates a File struct from a passed os.File previously constructed using NewFile.
+func NewCaptureFile(f *os.File) (*CaptureFile, error) {
 	if !IsDataCaptureFile(f) {
 		return nil, errors.Errorf("%s is not a data capture file", f.Name())
 	}
@@ -69,7 +69,7 @@ func ReadCaptureFile(f *os.File) (*CaptureFile, error) {
 		path:              f.Name(),
 		file:              f,
 		size:              finfo.Size(),
-		metadata:          md,
+		Metadata:          md,
 		initialReadOffset: int64(initOffset),
 		readOffset:        int64(initOffset),
 	}
@@ -79,7 +79,7 @@ func ReadCaptureFile(f *os.File) (*CaptureFile, error) {
 
 // ReadMetadata reads and returns the metadata in f.
 func (f *CaptureFile) ReadMetadata() *v1.DataCaptureMetadata {
-	return f.metadata
+	return f.Metadata
 }
 
 // ReadNext returns the next SensorData reading.
@@ -109,8 +109,6 @@ func (f *CaptureFile) Reset() {
 
 // Size returns the size of the file.
 func (f *CaptureFile) Size() int64 {
-	f.lock.Lock()
-	defer f.lock.Unlock()
 	return f.size
 }
 
@@ -139,22 +137,6 @@ func IsDataCaptureFile(f *os.File) bool {
 	return filepath.Ext(f.Name()) == CompletedCaptureFileExt
 }
 
-// Create a filename based on the current time.
-func getFileTimestampName() string {
-	// RFC3339Nano is a standard time format e.g. 2006-01-02T15:04:05Z07:00.
-	return time.Now().Format(time.RFC3339Nano)
-}
-
-// TODO DATA-246: Implement this in some more robust, programmatic way.
-func getDataType(methodName string) v1.DataType {
-	switch methodName {
-	case nextPointCloud, readImage, pointCloudMap, GetImages:
-		return v1.DataType_DATA_TYPE_BINARY_SENSOR
-	default:
-		return v1.DataType_DATA_TYPE_TABULAR_SENSOR
-	}
-}
-
 // GetFileExt gets the file extension for a capture file.
 func GetFileExt(dataType v1.DataType, methodName string, parameters map[string]string) string {
 	defaultFileExt := ""
@@ -188,22 +170,44 @@ func GetFileExt(dataType v1.DataType, methodName string, parameters map[string]s
 	return defaultFileExt
 }
 
-// SensorDataFromCaptureFilePath returns all readings in the file at filePath.
-// NOTE: (Nick S) At time of writing this is only used in tests.
-func SensorDataFromCaptureFilePath(filePath string) ([]*v1.SensorData, error) {
-	//nolint:gosec
-	f, err := os.Open(filePath)
-	if err != nil {
-		return nil, err
-	}
-	dcFile, err := ReadCaptureFile(f)
-	if err != nil {
-		return nil, err
-	}
+// FilePathWithReplacedReservedChars returns the filepath with substitutions
+// for reserved characters.
+func FilePathWithReplacedReservedChars(filepath string) string {
+	return strings.ReplaceAll(filepath, filePathReservedChars, "_")
+}
 
-	return SensorDataFromCaptureFile(dcFile)
+// Create a filename based on the current time.
+func getFileTimestampName() string {
+	// RFC3339Nano is a standard time format e.g. 2006-01-02T15:04:05Z07:00.
+	return time.Now().Format(time.RFC3339Nano)
+}
+
+// TODO DATA-246: Implement this in some more robust, programmatic way.
+func getDataType(methodName string) v1.DataType {
+	switch methodName {
+	case nextPointCloud, readImage, pointCloudMap, GetImages:
+		return v1.DataType_DATA_TYPE_BINARY_SENSOR
+	default:
+		return v1.DataType_DATA_TYPE_TABULAR_SENSOR
+	}
 }
 
+// SensorDataFromCaptureFilePath returns all readings in the file at filePath.
+// NOTE: (Nick S) At time of writing this is only used in tests.
+//func SensorDataFromCaptureFilePath(filePath string) ([]*v1.SensorData, error) {
+//	//nolint:gosec
+//	f, err := os.Open(filePath)
+//	if err != nil {
+//		return nil, err
+//	}
+//	dcFile, err := NewCaptureFile(f)
+//	if err != nil {
+//		return nil, err
+//	}
+
+//	return SensorDataFromCaptureFile(dcFile)
+//}
+
 // SensorDataFromCaptureFile returns all readings in f.
 func SensorDataFromCaptureFile(f *CaptureFile) ([]*v1.SensorData, error) {
 	f.Reset()
@@ -222,9 +226,3 @@ func SensorDataFromCaptureFile(f *CaptureFile) ([]*v1.SensorData, error) {
 	}
 	return ret, nil
 }
-
-// CaptureFilePathWithReplacedReservedChars returns the filepath with substitutions
-// for reserved characters.
-func CaptureFilePathWithReplacedReservedChars(filepath string) string {
-	return strings.ReplaceAll(filepath, filePathReservedChars, "_")
-}

data/capture_file_test.go

@@ -170,7 +170,7 @@ func TestReadCorruptedFile(t *testing.T) {
 	test.That(t, f.Close(), test.ShouldBeNil)
 
 	// Should still be able to successfully read all the successfully written data.
-	sd, err := SensorDataFromCaptureFilePath(captureFilePath(f.GetPath()))
+	sd, err := SensorDataFromCaptureFilePath(captureFilePath(f.path))
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, len(sd), test.ShouldEqual, numReadings)
 }