Skip to content

Commit 58960d8

Browse files
jorteljortel
authored
✨ Generate Primary Keys. (#635)
Generate primary keys instead of GORM. This fixes the issue of GORM reusing the highest key after the model with that ID is deleted. When the PK is 0, GORM assigns the next (highest) ID. This approach is to assign the ID ahead of time using a pool managed by tackle. --------- Signed-off-by: Jeff Ortel <[email protected]>
1 parent 1a83a4c commit 58960d8

File tree

15 files changed

+257
-14
lines changed

15 files changed

+257
-14
lines changed

‎api/migrationwave.go

+1
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ func (h MigrationWaveHandler) Create(ctx *gin.Context) {
118118
_ = ctx.Error(err)
119119
return
120120
}
121+
121122
r.With(m)
122123

123124
h.Respond(ctx, http.StatusCreated, r)

‎cmd/main.go

+5
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
crd "github.com/konveyor/tackle2-hub/k8s/api"
1818
"github.com/konveyor/tackle2-hub/metrics"
1919
"github.com/konveyor/tackle2-hub/migration"
20+
"github.com/konveyor/tackle2-hub/model"
2021
"github.com/konveyor/tackle2-hub/reaper"
2122
"github.com/konveyor/tackle2-hub/seed"
2223
"github.com/konveyor/tackle2-hub/settings"
@@ -53,6 +54,10 @@ func Setup() (db *gorm.DB, err error) {
5354
if err != nil {
5455
return
5556
}
57+
err = database.PK.Load(db, model.ALL)
58+
if err != nil {
59+
return
60+
}
5661
return
5762
}
5863

‎database/db_test.go

+53
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,56 @@ func TestConcurrent(t *testing.T) {
6969
fmt.Printf("Done %d\n", id)
7070
}
7171
}
72+
73+
func TestKeyGen(t *testing.T) {
74+
pid := os.Getpid()
75+
Settings.DB.Path = fmt.Sprintf("/tmp/keygen-%d.db", pid)
76+
defer func() {
77+
_ = os.Remove(Settings.DB.Path)
78+
}()
79+
db, err := Open(true)
80+
if err != nil {
81+
panic(err)
82+
}
83+
// ids 1-7 created.
84+
N = 8
85+
for n := 1; n < N; n++ {
86+
m := &model.Setting{Key: fmt.Sprintf("key-%d", n), Value: n}
87+
err := db.Create(m).Error
88+
if err != nil {
89+
panic(err)
90+
}
91+
fmt.Printf("CREATED: %d/%d\n", m.ID, n)
92+
if uint(n) != m.ID {
93+
t.Errorf("id:%d but expected: %d", m.ID, n)
94+
return
95+
}
96+
}
97+
// delete ids=2,4,7.
98+
err = db.Delete(&model.Setting{}, []uint{2, 4, 7}).Error
99+
if err != nil {
100+
panic(err)
101+
}
102+
103+
var count int64
104+
err = db.Model(&model.Setting{}).Where([]uint{2, 4, 7}).Count(&count).Error
105+
if err != nil {
106+
panic(err)
107+
}
108+
if count > 0 {
109+
t.Errorf("DELETED ids: 2,4,7 found.")
110+
return
111+
}
112+
// id=8 (next) created.
113+
next := N
114+
m := &model.Setting{Key: fmt.Sprintf("key-%d", next), Value: next}
115+
err = db.Create(m).Error
116+
if err != nil {
117+
panic(err)
118+
}
119+
fmt.Printf("CREATED: %d/%d (next)\n", m.ID, next)
120+
if uint(N) != m.ID {
121+
t.Errorf("id:%d but expected: %d", m.ID, next)
122+
return
123+
}
124+
}

‎database/pk.go

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package database
2+
3+
import (
4+
"errors"
5+
"reflect"
6+
"strings"
7+
"sync"
8+
9+
"github.com/konveyor/tackle2-hub/model"
10+
"gorm.io/gorm"
11+
"gorm.io/gorm/clause"
12+
"gorm.io/gorm/logger"
13+
)
14+
15+
// PK singleton pk sequence.
16+
var PK PkSequence
17+
18+
// PkSequence provides a primary key sequence.
19+
type PkSequence struct {
20+
mutex sync.Mutex
21+
}
22+
23+
// Load highest key for all models.
24+
func (r *PkSequence) Load(db *gorm.DB, models []any) (err error) {
25+
r.mutex.Lock()
26+
defer r.mutex.Unlock()
27+
for _, m := range models {
28+
mt := reflect.TypeOf(m)
29+
if mt.Kind() == reflect.Ptr {
30+
mt = mt.Elem()
31+
}
32+
kind := strings.ToUpper(mt.Name())
33+
db = r.session(db)
34+
q := db.Table(kind)
35+
q = q.Select("MAX(ID) id")
36+
cursor, err := q.Rows()
37+
if err != nil || !cursor.Next() {
38+
// not a table with id.
39+
// discarded.
40+
continue
41+
}
42+
id := int64(0)
43+
err = cursor.Scan(&id)
44+
_ = cursor.Close()
45+
if err != nil {
46+
r.add(db, kind, uint(0))
47+
} else {
48+
r.add(db, kind, uint(id))
49+
}
50+
}
51+
return
52+
}
53+
54+
// Next returns the next primary key.
55+
func (r *PkSequence) Next(db *gorm.DB) (id uint) {
56+
r.mutex.Lock()
57+
defer r.mutex.Unlock()
58+
kind := strings.ToUpper(db.Statement.Table)
59+
m := &model.PK{}
60+
db = r.session(db)
61+
err := db.First(m, "Kind", kind).Error
62+
if err != nil {
63+
return
64+
}
65+
m.LastID++
66+
id = m.LastID
67+
err = db.Save(m).Error
68+
if err != nil {
69+
panic(err)
70+
}
71+
return
72+
}
73+
74+
// session returns a new DB with a new session.
75+
func (r *PkSequence) session(in *gorm.DB) (out *gorm.DB) {
76+
out = &gorm.DB{
77+
Config: in.Config,
78+
}
79+
out.Config.Logger.LogMode(logger.Warn)
80+
out.Statement = &gorm.Statement{
81+
DB: out,
82+
ConnPool: in.Statement.ConnPool,
83+
Context: in.Statement.Context,
84+
Clauses: map[string]clause.Clause{},
85+
Vars: make([]interface{}, 0, 8),
86+
}
87+
return
88+
}
89+
90+
// add the last (higher) id for the kind.
91+
func (r *PkSequence) add(db *gorm.DB, kind string, id uint) {
92+
m := &model.PK{Kind: kind}
93+
db = r.session(db)
94+
err := db.First(m).Error
95+
if err != nil {
96+
if !errors.Is(err, gorm.ErrRecordNotFound) {
97+
panic(err)
98+
}
99+
}
100+
if m.LastID > id {
101+
return
102+
}
103+
m.LastID = id
104+
db = r.session(db)
105+
err = db.Save(m).Error
106+
if err != nil {
107+
panic(err)
108+
}
109+
}
110+
111+
// assignPk assigns PK as needed.
112+
func assignPk(db *gorm.DB) {
113+
statement := db.Statement
114+
schema := statement.Schema
115+
if schema == nil {
116+
return
117+
}
118+
switch statement.ReflectValue.Kind() {
119+
case reflect.Slice,
120+
reflect.Array:
121+
for i := 0; i < statement.ReflectValue.Len(); i++ {
122+
for _, f := range schema.Fields {
123+
if f.Name != "ID" {
124+
continue
125+
}
126+
_, isZero := f.ValueOf(
127+
statement.Context,
128+
statement.ReflectValue.Index(i))
129+
if isZero {
130+
id := PK.Next(db)
131+
_ = f.Set(
132+
statement.Context,
133+
statement.ReflectValue.Index(i),
134+
id)
135+
136+
}
137+
break
138+
}
139+
}
140+
case reflect.Struct:
141+
for _, f := range schema.Fields {
142+
if f.Name != "ID" {
143+
continue
144+
}
145+
_, isZero := f.ValueOf(
146+
statement.Context,
147+
statement.ReflectValue)
148+
if isZero {
149+
id := PK.Next(db)
150+
_ = f.Set(
151+
statement.Context,
152+
statement.ReflectValue,
153+
id)
154+
}
155+
break
156+
}
157+
default:
158+
log.Info("[WARN] assignPk: unknown kind.")
159+
}
160+
}

‎database/pkg.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,16 @@ func Open(enforceFKs bool) (db *gorm.DB, err error) {
5151
err = liberr.Wrap(err)
5252
return
5353
}
54-
err = db.AutoMigrate(model.Setting{})
54+
err = db.AutoMigrate(model.PK{}, model.Setting{})
55+
if err != nil {
56+
err = liberr.Wrap(err)
57+
return
58+
}
59+
err = PK.Load(db, []any{model.Setting{}})
60+
if err != nil {
61+
return
62+
}
63+
err = db.Callback().Create().Before("gorm:before_create").Register("assign-pk", assignPk)
5564
if err != nil {
5665
err = liberr.Wrap(err)
5766
return

‎migration/v14/model/core.go

+7
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@ type Model struct {
2020
UpdateUser string
2121
}
2222

23+
// PK sequence.
24+
type PK struct {
25+
Kind string `gorm:"<-:create;primaryKey"`
26+
LastID uint
27+
}
28+
29+
// Setting hub settings.
2330
type Setting struct {
2431
Model
2532
Key string `gorm:"<-:create;uniqueIndex"`

‎migration/v14/model/pkg.go

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func All() []any {
3232
ImportTag{},
3333
JobFunction{},
3434
MigrationWave{},
35+
PK{},
3536
Proxy{},
3637
Review{},
3738
Setting{},

‎model/pkg.go

+3
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
// Field (data) types.
99
type JSON = model.JSON
1010

11+
var ALL = model.All()
12+
1113
// Models
1214
type Model = model.Model
1315
type Application = model.Application
@@ -29,6 +31,7 @@ type ImportSummary = model.ImportSummary
2931
type ImportTag = model.ImportTag
3032
type JobFunction = model.JobFunction
3133
type MigrationWave = model.MigrationWave
34+
type PK = model.PK
3235
type Proxy = model.Proxy
3336
type Questionnaire = model.Questionnaire
3437
type Review = model.Review

‎test/api/migrationwave/api_test.go

+6
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ func TestMigrationWaveCRUD(t *testing.T) {
1818
}
1919
assert.Must(t, Application.Create(&expectedApp))
2020
createdApps = append(createdApps, expectedApp)
21+
r.Applications[0].ID = expectedApp.ID
2122
}
2223

2324
createdStakeholders := []api.Stakeholder{}
@@ -28,6 +29,7 @@ func TestMigrationWaveCRUD(t *testing.T) {
2829
}
2930
assert.Must(t, Stakeholder.Create(&expectedStakeholder))
3031
createdStakeholders = append(createdStakeholders, expectedStakeholder)
32+
r.Stakeholders[0].ID = expectedStakeholder.ID
3133
}
3234

3335
createdStakeholderGroups := []api.StakeholderGroup{}
@@ -38,6 +40,7 @@ func TestMigrationWaveCRUD(t *testing.T) {
3840
}
3941
assert.Must(t, StakeholderGroup.Create(&expectedStakeholderGroup))
4042
createdStakeholderGroups = append(createdStakeholderGroups, expectedStakeholderGroup)
43+
r.StakeholderGroups[0].ID = expectedStakeholderGroup.ID
4144
}
4245

4346
assert.Must(t, MigrationWave.Create(&r))
@@ -102,6 +105,7 @@ func TestMigrationWaveList(t *testing.T) {
102105
}
103106
assert.Must(t, Application.Create(&expectedApp))
104107
createdApps = append(createdApps, expectedApp)
108+
r.Applications[0].ID = expectedApp.ID
105109
}
106110

107111
for _, stakeholder := range r.Stakeholders {
@@ -111,6 +115,7 @@ func TestMigrationWaveList(t *testing.T) {
111115
}
112116
assert.Must(t, Stakeholder.Create(&expectedStakeholder))
113117
createdStakeholders = append(createdStakeholders, expectedStakeholder)
118+
r.Stakeholders[0].ID = expectedStakeholder.ID
114119
}
115120

116121
for _, stakeholderGroup := range r.StakeholderGroups {
@@ -120,6 +125,7 @@ func TestMigrationWaveList(t *testing.T) {
120125
}
121126
assert.Must(t, StakeholderGroup.Create(&expectedStakeholderGroup))
122127
createdStakeholderGroups = append(createdStakeholderGroups, expectedStakeholderGroup)
128+
r.StakeholderGroups[0].ID = expectedStakeholderGroup.ID
123129
}
124130
assert.Must(t, MigrationWave.Create(&r))
125131
createdMigrationWaves = append(createdMigrationWaves, r)

‎test/api/migrationwave/samples.go

-3
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,16 @@ var Samples = []api.MigrationWave{
1313
EndDate: time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local).Add(30 * time.Minute),
1414
Applications: []api.Ref{
1515
{
16-
ID: 1,
1716
Name: "Sample Application",
1817
},
1918
},
2019
Stakeholders: []api.Ref{
2120
{
22-
ID: 1,
2321
Name: "Sample Stakeholders",
2422
},
2523
},
2624
StakeholderGroups: []api.Ref{
2725
{
28-
ID: 1,
2926
Name: "Sample Stakeholders Groups",
3027
},
3128
},

‎test/api/review/api_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ func TestReviewList(t *testing.T) {
143143

144144
// Delete related reviews and applications.
145145
for _, review := range createdReviews {
146-
assert.Must(t, Application.Delete(review.ID))
146+
assert.Must(t, Application.Delete(review.Application.ID))
147147
assert.Must(t, Review.Delete(review.ID))
148148
}
149149
}

‎test/api/review/samples.go

-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ var Samples = []api.Review{
1212
WorkPriority: 1,
1313
Comments: "nil",
1414
Application: &api.Ref{
15-
ID: 1,
1615
Name: "Sample Review 1",
1716
},
1817
},
@@ -23,7 +22,6 @@ var Samples = []api.Review{
2322
WorkPriority: 2,
2423
Comments: "nil",
2524
Application: &api.Ref{
26-
ID: 2,
2725
Name: "Sample Review 2",
2826
},
2927
},

0 commit comments

Comments
 (0)