Skip to content

Commit 2973153

Browse files
lostlevelslostlevels
committed
Create new OriginId index and remove type from indexes that don't need
it.
1 parent 164a4bc commit 2973153

File tree

3 files changed

+71
-30
lines changed

3 files changed

+71
-30
lines changed

‎data/deduplicator/deduplicator/data_set_delete_origin.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package deduplicator
22

33
import (
44
"context"
5+
"slices"
6+
"strings"
57

68
"github.com/tidepool-org/platform/data"
79
dataStore "github.com/tidepool-org/platform/data/store"
@@ -79,6 +81,15 @@ func (d *DataSetDeleteOrigin) AddData(ctx context.Context, repository dataStore.
7981
if err := repository.DeleteDataSetData(ctx, dataSet, selectors); err != nil {
8082
return err
8183
}
84+
// Even though doing an unordered bulkwrite in AddData there is some benefit in sorting this for our origin.id index
85+
slices.SortFunc(dataSetData, func(a, b data.Datum) int {
86+
originA, originB := a.GetOrigin(), b.GetOrigin()
87+
if originA != nil && originA.ID != nil && originB != nil && originB.ID != nil {
88+
return strings.Compare(pointer.ToString(originA.ID), pointer.ToString(originB.ID))
89+
}
90+
timeA, timeB := a.GetTime(), b.GetTime()
91+
return pointer.ToTime(timeA).Compare(pointer.ToTime(timeB))
92+
})
8293
if err := d.Base.AddData(ctx, repository, dataSet, dataSetData); err != nil {
8394
return err
8495
}

‎data/store/mongo/mongo_datum.go‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -92,16 +92,16 @@ func (d *DatumRepository) EnsureIndexes() error {
9292
},
9393
}),
9494
},
95-
{
96-
Keys: bson.D{
97-
{Key: "origin.id", Value: 1},
98-
{Key: "type", Value: 1},
99-
{Key: "deletedTime", Value: -1},
100-
{Key: "_active", Value: 1},
101-
},
102-
Options: options.Index().
103-
SetName("OriginId"),
104-
},
95+
// {
96+
// Keys: bson.D{
97+
// {Key: "origin.id", Value: 1},
98+
// {Key: "type", Value: 1},
99+
// {Key: "deletedTime", Value: -1},
100+
// {Key: "_active", Value: 1},
101+
// },
102+
// Options: options.Index().
103+
// SetName("OriginId"),
104+
// },
105105
{
106106
Keys: bson.D{
107107
{Key: "_userId", Value: 1},
@@ -125,12 +125,26 @@ func (d *DatumRepository) EnsureIndexes() error {
125125
Options: options.Index().
126126
SetName("UploadId"),
127127
},
128+
// {
129+
// Keys: bson.D{
130+
// {Key: "_userId", Value: 1},
131+
// {Key: "deviceId", Value: 1},
132+
// {Key: "type", Value: 1},
133+
// {Key: "_active", Value: 1},
134+
// {Key: "_deduplicator.hash", Value: 1},
135+
// },
136+
// Options: options.Index().
137+
// SetPartialFilterExpression(bson.D{
138+
// {Key: "_active", Value: true},
139+
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
140+
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
141+
// }).
142+
// SetName("DeduplicatorHash"),
143+
// },
128144
{
129145
Keys: bson.D{
130146
{Key: "_userId", Value: 1},
131147
{Key: "deviceId", Value: 1},
132-
{Key: "type", Value: 1},
133-
{Key: "_active", Value: 1},
134148
{Key: "_deduplicator.hash", Value: 1},
135149
},
136150
Options: options.Index().
@@ -139,7 +153,7 @@ func (d *DatumRepository) EnsureIndexes() error {
139153
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
140154
{Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
141155
}).
142-
SetName("DeduplicatorHash"),
156+
SetName("DeduplicatorHashNoType"),
143157
},
144158
})
145159
}
@@ -233,7 +247,7 @@ func (d *DatumRepository) ArchiveDataSetData(ctx context.Context, dataSet *uploa
233247
if err := validateDataSet(dataSet); err != nil {
234248
return err
235249
}
236-
selector, _, err := validateAndTranslateSelectors(selectors)
250+
selector, hasOriginID, err := validateAndTranslateSelectors(selectors)
237251
if err != nil {
238252
return err
239253
}
@@ -256,7 +270,11 @@ func (d *DatumRepository) ArchiveDataSetData(ctx context.Context, dataSet *uploa
256270
"archivedDatasetId": 1,
257271
"modifiedUserId": 1,
258272
}
259-
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset))
273+
opts := options.Update()
274+
if hasOriginID {
275+
opts.SetHint("UserIdOriginId")
276+
}
277+
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
260278
if err != nil {
261279
logger.WithError(err).Error("Unable to archive data set data")
262280
return fmt.Errorf("unable to archive data set data: %w", err)
@@ -299,7 +317,7 @@ func (d *DatumRepository) DeleteDataSetData(ctx context.Context, dataSet *upload
299317
}
300318
opts := options.Update()
301319
if hasOriginID {
302-
opts.SetHint("OriginId")
320+
opts.SetHint("UserIdOriginId")
303321
}
304322
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
305323
if err != nil {
@@ -332,7 +350,7 @@ func (d *DatumRepository) DestroyDeletedDataSetData(ctx context.Context, dataSet
332350
selector["deletedTime"] = bson.M{"$exists": true}
333351
opts := options.Delete()
334352
if hasOriginID {
335-
opts.SetHint("OriginId")
353+
opts.SetHint("UserIdOriginId")
336354
}
337355
changeInfo, err := d.DeleteMany(ctx, selector, opts)
338356
if err != nil {
@@ -410,7 +428,7 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co
410428
"modifiedTime": timestamp,
411429
}
412430
unset := bson.M{}
413-
opts := options.Update().SetHint("DeduplicatorHash")
431+
opts := options.Update().SetHint("DeduplicatorHashNoType")
414432
updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
415433
}
416434

‎data/store/mongo/mongo_test.go‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -547,9 +547,8 @@ var _ = Describe("Mongo", func() {
547547
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_id")),
548548
}),
549549
MatchFields(IgnoreExtras, Fields{
550-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "-time")),
551-
"Background": Equal(true),
552-
"Name": Equal("UserIdTypeWeighted_v2"),
550+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "-time")),
551+
"Name": Equal("UserIdTypeWeighted_v2"),
553552
}),
554553
MatchFields(IgnoreExtras, Fields{
555554
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "time", "modifiedTime")),
@@ -565,20 +564,33 @@ var _ = Describe("Mongo", func() {
565564
{Key: "time", Value: bson.D{{Key: "$gt", Value: primitive.NewDateTimeFromTime(lowerTimeIndex)}}},
566565
}),
567566
}),
567+
// MatchFields(IgnoreExtras, Fields{
568+
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
569+
// "Name": Equal("OriginId"),
570+
// }),
568571
MatchFields(IgnoreExtras, Fields{
569-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
570-
"Background": Equal(true),
571-
"Name": Equal("OriginId"),
572+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "origin.id", "-deletedTime", "_active")),
573+
"Name": Equal("UserIdOriginId"),
574+
"PartialFilterExpression": Equal(bson.D{
575+
{Key: "origin.id", Value: bson.D{{Key: "$exists", Value: true}}},
576+
}),
572577
}),
573578
MatchFields(IgnoreExtras, Fields{
574-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
575-
"Background": Equal(true),
576-
"Name": Equal("UploadId"),
579+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
580+
"Name": Equal("UploadId"),
577581
}),
582+
// MatchFields(IgnoreExtras, Fields{
583+
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
584+
// "Name": Equal("DeduplicatorHash"),
585+
// "PartialFilterExpression": Equal(bson.D{
586+
// {Key: "_active", Value: true},
587+
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
588+
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
589+
// }),
590+
// }),
578591
MatchFields(IgnoreExtras, Fields{
579-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
580-
"Background": Equal(true),
581-
"Name": Equal("DeduplicatorHash"),
592+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "_deduplicator.hash")),
593+
"Name": Equal("DeduplicatorHashNoType"),
582594
"PartialFilterExpression": Equal(bson.D{
583595
{Key: "_active", Value: true},
584596
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},

0 commit comments

Comments
 (0)