Skip to content

Commit 35d1284

Browse files
nicksanfordnicksanford
committed
wip
1 parent aa185b7 commit 35d1284

File tree

2 files changed

+181
-121
lines changed

2 files changed

+181
-121
lines changed

‎data/capture_buffer_test.go

+165-111
Original file line numberDiff line numberDiff line change
@@ -658,34 +658,34 @@ func TestCaptureBufferReader(t *testing.T) {
658658
})
659659
}
660660

661-
func BenchmarkBinaryReader(b *testing.B) {
661+
func BenchmarkChunked(b *testing.B) {
662662
type testCase struct {
663663
name string
664664
data []byte
665665
}
666-
eightKBFilled := make([]byte, 1024*8, 1024*8)
666+
eightKBFilled := make([]byte, 1024*8)
667667
for i := range eightKBFilled {
668668
eightKBFilled[i] = uint8(i % 256)
669669
}
670670

671-
oneMbFilled := make([]byte, 1024*1000, 1024*1000)
671+
oneMbFilled := make([]byte, 1024*1000)
672672
for i := range eightKBFilled {
673673
oneMbFilled[i] = uint8(i % 256)
674674
}
675675

676-
eightMbFilled := make([]byte, 1024*1000*8, 1024*1000*8)
676+
eightMbFilled := make([]byte, 1024*1000*8)
677677
for i := range eightMbFilled {
678678
eightMbFilled[i] = uint8(i % 256)
679679
}
680680

681681
tcs := []testCase{
682682
{"empty data", []byte{}},
683683
{"small data", []byte("this is a fake image")},
684-
{"8kb empty", make([]byte, 1024*8, 1024*8)},
684+
{"8kb empty", make([]byte, 1024*8)},
685685
{"8kb filled", eightKBFilled},
686-
{"1mb empty", make([]byte, 1024*1000, 1024*1000)},
686+
{"1mb empty", make([]byte, 1024*1000)},
687687
{"1mb filled", oneMbFilled},
688-
{"8mb empty", make([]byte, 1024*1000*8, 1024*1000*8)},
688+
{"8mb empty", make([]byte, 1024*1000*8)},
689689
{"8mb filled", eightMbFilled},
690690
}
691691

@@ -694,53 +694,167 @@ func BenchmarkBinaryReader(b *testing.B) {
694694
_, err := s.Write(tc.data)
695695
test.That(b, err, test.ShouldBeNil)
696696
expectedHash := s.Sum(nil)
697+
tmpDir := b.TempDir()
698+
name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
699+
additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
700+
methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
701+
test.That(b, err, test.ShouldBeNil)
702+
703+
readImageCaptureMetadata := BuildCaptureMetadata(
704+
name.API,
705+
name.ShortName(),
706+
readImage,
707+
additionalParams,
708+
methodParams,
709+
[]string{"my", "tags"},
710+
)
711+
712+
now := time.Now()
713+
timeRequested := timestamppb.New(now.UTC())
714+
timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
715+
msg := &v1.SensorData{
716+
Metadata: &v1.SensorMetadata{
717+
TimeRequested: timeRequested,
718+
TimeReceived: timeReceived,
719+
},
720+
Data: &v1.SensorData_Binary{
721+
Binary: tc.data,
722+
},
723+
}
724+
725+
buf := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))
726+
727+
// Path() is the same as the first paramenter passed to NewCaptureBuffer
728+
test.That(b, buf.Path(), test.ShouldResemble, tmpDir)
729+
test.That(b, buf.metaData, test.ShouldResemble, readImageCaptureMetadata)
730+
731+
test.That(b, buf.Write(msg), test.ShouldBeNil)
732+
test.That(b, buf.Flush(), test.ShouldBeNil)
733+
dirEntries, err := os.ReadDir(buf.Path())
734+
test.That(b, err, test.ShouldBeNil)
735+
test.That(b, len(dirEntries), test.ShouldEqual, 1)
736+
test.That(b, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
737+
f, err := os.Open(filepath.Join(buf.Path(), dirEntries[0].Name()))
738+
test.That(b, err, test.ShouldBeNil)
739+
b.Cleanup(func() { test.That(b, f.Close(), test.ShouldBeNil) })
740+
697741
b.ResetTimer()
698-
b.Run(tc.name+" read entire binary", func(b *testing.B) {
742+
b.Run("chunked "+tc.name, func(b *testing.B) {
699743
for i := 0; i < b.N; i++ {
700-
tmpDir := b.TempDir()
701-
name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
702-
additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
703-
methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
744+
ret, err := f.Seek(0, io.SeekStart)
704745
test.That(b, err, test.ShouldBeNil)
746+
test.That(b, ret, test.ShouldEqual, 0)
747+
cf2, err := NewCaptureFile(f)
748+
test.That(b, err, test.ShouldBeNil)
749+
test.That(b, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
705750

706-
readImageCaptureMetadata := BuildCaptureMetadata(
707-
name.API,
708-
name.ShortName(),
709-
readImage,
710-
additionalParams,
711-
methodParams,
712-
[]string{"my", "tags"},
713-
)
714-
715-
now := time.Now()
716-
timeRequested := timestamppb.New(now.UTC())
717-
timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
718-
msg := &v1.SensorData{
719-
Metadata: &v1.SensorMetadata{
720-
TimeRequested: timeRequested,
721-
TimeReceived: timeReceived,
722-
},
723-
Data: &v1.SensorData_Binary{
724-
Binary: tc.data,
725-
},
751+
var md v1.SensorMetadata
752+
r, err := cf2.BinaryReader(&md)
753+
test.That(b, err, test.ShouldBeNil)
754+
test.That(b, r, test.ShouldNotBeNil)
755+
test.That(b, &md, test.ShouldResemble, msg.GetMetadata())
756+
data := make([]byte, 4064)
757+
h := sha1.New()
758+
for {
759+
n, err := r.Read(data)
760+
if errors.Is(err, io.EOF) {
761+
break
762+
}
763+
test.That(b, err, test.ShouldBeNil)
764+
_, err = h.Write(data[:n])
765+
test.That(b, err, test.ShouldBeNil)
726766
}
767+
actualHash := h.Sum(nil)
768+
test.That(b, actualHash, test.ShouldResemble, expectedHash)
769+
}
770+
})
771+
}
772+
}
727773

728-
buf := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))
774+
func BenchmarkNonChunked(b *testing.B) {
775+
type testCase struct {
776+
name string
777+
data []byte
778+
}
779+
eightKBFilled := make([]byte, 1024*8)
780+
for i := range eightKBFilled {
781+
eightKBFilled[i] = uint8(i % 256)
782+
}
729783

730-
// Path() is the same as the first paramenter passed to NewCaptureBuffer
731-
test.That(b, buf.Path(), test.ShouldResemble, tmpDir)
732-
test.That(b, buf.metaData, test.ShouldResemble, readImageCaptureMetadata)
784+
oneMbFilled := make([]byte, 1024*1000)
785+
for i := range eightKBFilled {
786+
oneMbFilled[i] = uint8(i % 256)
787+
}
733788

734-
test.That(b, buf.Write(msg), test.ShouldBeNil)
735-
test.That(b, buf.Flush(), test.ShouldBeNil)
736-
dirEntries, err := os.ReadDir(buf.Path())
737-
test.That(b, err, test.ShouldBeNil)
738-
test.That(b, len(dirEntries), test.ShouldEqual, 1)
739-
test.That(b, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
740-
f, err := os.Open(filepath.Join(buf.Path(), dirEntries[0].Name()))
741-
test.That(b, err, test.ShouldBeNil)
742-
defer func() { test.That(b, f.Close(), test.ShouldBeNil) }()
789+
eightMbFilled := make([]byte, 1024*1000*8)
790+
for i := range eightMbFilled {
791+
eightMbFilled[i] = uint8(i % 256)
792+
}
793+
794+
tcs := []testCase{
795+
{"empty data", []byte{}},
796+
{"small data", []byte("this is a fake image")},
797+
{"8kb empty", make([]byte, 1024*8)},
798+
{"8kb filled", eightKBFilled},
799+
{"1mb empty", make([]byte, 1024*1000)},
800+
{"1mb filled", oneMbFilled},
801+
{"8mb empty", make([]byte, 1024*1000*8)},
802+
{"8mb filled", eightMbFilled},
803+
}
804+
805+
for _, tc := range tcs {
806+
s := sha1.New()
807+
_, err := s.Write(tc.data)
808+
test.That(b, err, test.ShouldBeNil)
809+
expectedHash := s.Sum(nil)
810+
tmpDir := b.TempDir()
811+
name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
812+
additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
813+
methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
814+
test.That(b, err, test.ShouldBeNil)
815+
816+
readImageCaptureMetadata := BuildCaptureMetadata(
817+
name.API,
818+
name.ShortName(),
819+
readImage,
820+
additionalParams,
821+
methodParams,
822+
[]string{"my", "tags"},
823+
)
824+
825+
now := time.Now()
826+
timeRequested := timestamppb.New(now.UTC())
827+
timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
828+
msg := &v1.SensorData{
829+
Metadata: &v1.SensorMetadata{
830+
TimeRequested: timeRequested,
831+
TimeReceived: timeReceived,
832+
},
833+
Data: &v1.SensorData_Binary{
834+
Binary: tc.data,
835+
},
836+
}
743837

838+
buf := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))
839+
840+
test.That(b, buf.Path(), test.ShouldResemble, tmpDir)
841+
test.That(b, buf.metaData, test.ShouldResemble, readImageCaptureMetadata)
842+
843+
test.That(b, buf.Write(msg), test.ShouldBeNil)
844+
test.That(b, buf.Flush(), test.ShouldBeNil)
845+
dirEntries, err := os.ReadDir(buf.Path())
846+
test.That(b, err, test.ShouldBeNil)
847+
test.That(b, len(dirEntries), test.ShouldEqual, 1)
848+
test.That(b, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
849+
f, err := os.Open(filepath.Join(buf.Path(), dirEntries[0].Name()))
850+
test.That(b, err, test.ShouldBeNil)
851+
b.Cleanup(func() { test.That(b, f.Close(), test.ShouldBeNil) })
852+
b.ResetTimer()
853+
b.Run("non chunked "+tc.name, func(b *testing.B) {
854+
for i := 0; i < b.N; i++ {
855+
ret, err := f.Seek(0, io.SeekStart)
856+
test.That(b, err, test.ShouldBeNil)
857+
test.That(b, ret, test.ShouldEqual, 0)
744858
cf2, err := NewCaptureFile(f)
745859
test.That(b, err, test.ShouldBeNil)
746860
test.That(b, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
@@ -755,86 +869,26 @@ func BenchmarkBinaryReader(b *testing.B) {
755869
test.That(b, actualHash, test.ShouldResemble, expectedHash)
756870
}
757871
})
758-
// b.Run(tc.name+" chunked", func(b *testing.B) {
759-
// for i := 0; i < b.N; i++ {
760-
// tmpDir := b.TempDir()
761-
// name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
762-
// additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
763-
// methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
764-
// test.That(b, err, test.ShouldBeNil)
765-
766-
// readImageCaptureMetadata := BuildCaptureMetadata(
767-
// name.API,
768-
// name.ShortName(),
769-
// readImage,
770-
// additionalParams,
771-
// methodParams,
772-
// []string{"my", "tags"},
773-
// )
774-
775-
// now := time.Now()
776-
// timeRequested := timestamppb.New(now.UTC())
777-
// timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
778-
// msg := &v1.SensorData{
779-
// Metadata: &v1.SensorMetadata{
780-
// TimeRequested: timeRequested,
781-
// TimeReceived: timeReceived,
782-
// },
783-
// Data: &v1.SensorData_Binary{
784-
// Binary: tc.data,
785-
// },
786-
// }
787-
788-
// buf := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))
789-
790-
// // Path() is the same as the first paramenter passed to NewCaptureBuffer
791-
// test.That(b, buf.Path(), test.ShouldResemble, tmpDir)
792-
// test.That(b, buf.metaData, test.ShouldResemble, readImageCaptureMetadata)
793-
794-
// test.That(b, buf.Write(msg), test.ShouldBeNil)
795-
// test.That(b, buf.Flush(), test.ShouldBeNil)
796-
// dirEntries, err := os.ReadDir(buf.Path())
797-
// test.That(b, err, test.ShouldBeNil)
798-
// test.That(b, len(dirEntries), test.ShouldEqual, 1)
799-
// test.That(b, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
800-
// f, err := os.Open(filepath.Join(buf.Path(), dirEntries[0].Name()))
801-
// test.That(b, err, test.ShouldBeNil)
802-
// defer func() { test.That(b, f.Close(), test.ShouldBeNil) }()
803-
804-
// cf2, err := NewCaptureFile(f)
805-
// test.That(b, err, test.ShouldBeNil)
806-
// test.That(b, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)
807-
808-
// var md v1.SensorMetadata
809-
// r, err := cf2.BinaryReader(&md)
810-
// test.That(b, err, test.ShouldBeNil)
811-
// test.That(b, r, test.ShouldNotBeNil)
812-
// test.That(b, &md, test.ShouldResemble, msg.GetMetadata())
813-
// data, err := io.ReadAll(r)
814-
// test.That(b, err, test.ShouldBeNil)
815-
// test.That(b, data, test.ShouldResemble, msg.GetBinary())
816-
// }
817-
// })
818872
}
819873
}
820874

821875
func FuzzBinaryReader(f *testing.F) {
822-
eightKBFilled := make([]byte, 1024*8, 1024*8)
876+
eightKBFilled := make([]byte, 1024*8)
823877
for i := range eightKBFilled {
824878
eightKBFilled[i] = uint8(i % 256)
825879
}
826880

827-
eightMbFilled := make([]byte, 1024*1000*8, 1024*1000*8)
881+
eightMbFilled := make([]byte, 1024*1000*8)
828882
for i := range eightMbFilled {
829883
eightMbFilled[i] = uint8(i % 256)
830884
}
831885

832886
tcs := [][]byte{
833-
[]byte{},
887+
{},
834888
[]byte("this is a fake image"),
835-
make([]byte, 1024*8, 1024*8),
889+
make([]byte, 1024*8),
836890
eightKBFilled,
837-
make([]byte, 1024*1000*8, 1024*1000*8),
891+
make([]byte, 1024*1000*8),
838892
}
839893

840894
for _, tc := range tcs {
@@ -900,7 +954,7 @@ func FuzzBinaryReader(f *testing.F) {
900954
})
901955
}
902956

903-
// nolint
957+
//nolint
904958
func getCaptureFiles(dir string) (dcFiles, progFiles []string) {
905959
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
906960
if err != nil {

0 commit comments

Comments
 (0)