Skip to content

Commit cce361c

Browse files
dabasovdabasov
authored
Merge pull request #1383 from 0chain/sprint-1.12
Sprint 1.12
2 parents 06be785 + 1d81b05 commit cce361c

20 files changed

+259
-101
lines changed

‎.github/workflows/build-&-publish-docker-image.yml

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ jobs:
4545
- name: Setup go
4646
uses: actions/setup-go@v4
4747
with:
48-
go-version: ^1.20 # The Go version to download (if necessary) and use.
48+
go-version: ^1.21 # The Go version to download (if necessary) and use.
4949

5050
- name: Clone blobber
5151
uses: actions/checkout@v3
@@ -71,7 +71,7 @@ jobs:
7171
docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
7272
7373
- name: Build Base image
74-
if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
74+
# if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
7575
run: |
7676
SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
7777
@@ -84,14 +84,17 @@ jobs:
8484
- name: Build blobber
8585
run: |
8686
SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
87-
export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base"
87+
export DOCKER_IMAGE_BASE="$BLOBBER_BUILD_BASE_REGISTRY:$TAG"
8888
export DOCKER_IMAGE_SWAGGER="${BLOBBER_REGISTRY}:swagger_test"
89-
export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
90-
export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}"
89+
# export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
90+
export DOCKER_BUILD="build --push"
91+
export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG}"
9192
export CONTEXT_NAME="$RUNNER_NAME" && (docker context inspect "$CONTEXT_NAME" >/dev/null 2>&1 || docker context create "$CONTEXT_NAME")
9293
docker buildx inspect "blobber-$RUNNER_NAME" || docker buildx create --name "blobber-$RUNNER_NAME" --driver-opt network=host --buildkitd-flags '--allow-insecure-entitlement security.insecure' "$CONTEXT_NAME"
9394
docker buildx use "blobber-$RUNNER_NAME"
9495
./docker.local/bin/build.blobber.sh
96+
docker tag ${BLOBBER_REGISTRY}:${TAG} ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}
97+
docker push ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}
9598
9699
validator:
97100
timeout-minutes: 30
@@ -118,7 +121,7 @@ jobs:
118121
- name: Setup go
119122
uses: actions/setup-go@v4
120123
with:
121-
go-version: ^1.20 # The Go version to download (if necessary) and use.
124+
go-version: ^1.21 # The Go version to download (if necessary) and use.
122125

123126
- name: Clone blobber
124127
uses: actions/checkout@v3
@@ -145,7 +148,7 @@ jobs:
145148
docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
146149
147150
- name: Build Base image
148-
if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
151+
# if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
149152
run: |
150153
SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
151154
@@ -158,14 +161,16 @@ jobs:
158161
- name: Build validator
159162
run: |
160163
SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
161-
export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base"
162-
export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
163-
export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}"
164+
export DOCKER_IMAGE_BASE="$BLOBBER_BUILD_BASE_REGISTRY:$TAG"
165+
# export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
166+
export DOCKER_BUILD="build --push"
167+
export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG}"
164168
export CONTEXT_NAME="$RUNNER_NAME" && (docker context inspect "$CONTEXT_NAME" >/dev/null 2>&1 || docker context create "$CONTEXT_NAME")
165169
docker buildx inspect "validator-$RUNNER_NAME" || docker buildx create --name "validator-$RUNNER_NAME" --driver-opt network=host --buildkitd-flags '--allow-insecure-entitlement security.insecure' "$CONTEXT_NAME"
166170
docker buildx use "validator-$RUNNER_NAME"
167171
./docker.local/bin/build.validator.sh
168-
172+
docker tag ${VALIDATOR_REGISTRY}:${TAG} ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}
173+
docker push ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}
169174
170175
system-tests:
171176
if: github.event_name != 'workflow_dispatch'

‎.github/workflows/build-for-conductor-testing.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
- name: Setup go
4343
uses: actions/setup-go@v3
4444
with:
45-
go-version: ^1.20 # The Go version to download (if necessary) and use.
45+
go-version: ^1.21 # The Go version to download (if necessary) and use.
4646

4747
- name: Clone blobber
4848
uses: actions/checkout@v1

‎code/go/0chain.net/blobbercore/allocation/copyfilechange.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,11 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, rootRef *reference.Re
6868
}
6969
}
7070

71+
if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir {
72+
return nil, common.NewErrorf("max_objects_in_dir_reached",
73+
"maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir)
74+
}
75+
7176
if !found {
7277
newRef := reference.NewDirectoryRef()
7378
newRef.AllocationID = rf.AllocationID

‎code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,7 @@ func TestBlobberCore_CopyFile(t *testing.T) {
375375
tc.setupDbMock()
376376

377377
config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc
378+
config.Configuration.MaxObjectsInDir = 1000
378379

379380
ctx := datastore.GetStore().CreateTransaction(context.TODO())
380381

‎code/go/0chain.net/blobbercore/allocation/file_changer_upload.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference
5959
found = true
6060
}
6161
}
62+
63+
if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir {
64+
return nil, common.NewErrorf("max_objects_in_dir_reached",
65+
"maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir)
66+
}
67+
6268
if !found {
6369
newRef := reference.NewDirectoryRef()
6470
newRef.AllocationID = dirRef.AllocationID

‎code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) {
9696
tc.setupDbMock()
9797

9898
config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc
99+
config.Configuration.MaxObjectsInDir = 1000
99100

100101
ctx := datastore.GetStore().CreateTransaction(context.TODO())
101102

‎code/go/0chain.net/blobbercore/allocation/movefilechange.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"strings"
99
"sync"
1010

11+
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
1112
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
1213
"github.com/0chain/blobber/code/go/0chain.net/core/common"
1314
)
@@ -91,6 +92,11 @@ func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Re
9192
}
9293
}
9394

95+
if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir {
96+
return nil, common.NewErrorf("max_objects_in_dir_reached",
97+
"maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir)
98+
}
99+
94100
if !found {
95101
newRef := reference.NewDirectoryRef()
96102
newRef.AllocationID = rf.AllocationID

‎code/go/0chain.net/blobbercore/allocation/movefilechange_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,7 @@ func TestBlobberCore_MoveFile(t *testing.T) {
472472
tc.setupDbMock()
473473

474474
config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc
475+
config.Configuration.MaxObjectsInDir = 1000
475476

476477
ctx := datastore.GetStore().CreateTransaction(context.TODO())
477478

‎code/go/0chain.net/blobbercore/config/config.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ func SetupDefaultConfig() {
3030
viper.SetDefault("rate_limiters.block_limit_monthly", 31250000)
3131
viper.SetDefault("rate_limiters.upload_limit_monthly", 31250000)
3232
viper.SetDefault("rate_limiters.commit_limit_monthly", 30000)
33+
viper.SetDefault("rate_limiters.commit_limit_daily", 1600)
34+
viper.SetDefault("rate_limiters.commit_zero_limit_daily", 400)
3335

3436
viper.SetDefault("healthcheck.frequency", "60s")
3537

@@ -47,6 +49,7 @@ func SetupDefaultConfig() {
4749
viper.SetDefault("finalize_allocations_interval", time.Duration(-1))
4850

4951
viper.SetDefault("max_dirs_files", 50000)
52+
viper.SetDefault("max_objects_dir", 1000)
5053
}
5154

5255
/*SetupConfig - setup the configuration system */
@@ -109,6 +112,8 @@ type Config struct {
109112
BlockLimitMonthly int64
110113
UploadLimitMonthly int64
111114
CommitLimitMonthly int64
115+
CommitLimitDaily int64
116+
CommitZeroLimitDaily int64
112117
ChallengeCleanupGap int64
113118

114119
HealthCheckWorkerFreq time.Duration
@@ -124,6 +129,7 @@ type Config struct {
124129
FinalizeAllocationsInterval time.Duration
125130

126131
MaxAllocationDirFiles int
132+
MaxObjectsInDir int
127133

128134
// DelegateWallet for pool owner.
129135
DelegateWallet string `json:"delegate_wallet"`
@@ -256,6 +262,8 @@ func ReadConfig(deploymentMode int) {
256262
Configuration.MaxAllocationDirFiles =
257263
viper.GetInt("max_dirs_files")
258264

265+
Configuration.MaxObjectsInDir = viper.GetInt("max_objects_dir")
266+
259267
Configuration.DelegateWallet = viper.GetString("delegate_wallet")
260268
if w := Configuration.DelegateWallet; len(w) != 64 {
261269
log.Fatal("invalid delegate wallet:", w)
@@ -279,6 +287,8 @@ func ReadConfig(deploymentMode int) {
279287
Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly")
280288
Configuration.UploadLimitMonthly = viper.GetInt64("rate_limiters.upload_limit_monthly")
281289
Configuration.CommitLimitMonthly = viper.GetInt64("rate_limiters.commit_limit_monthly")
290+
Configuration.CommitLimitDaily = viper.GetInt64("rate_limiters.commit_limit_daily")
291+
Configuration.CommitZeroLimitDaily = viper.GetInt64("rate_limiters.commit_zero_limit_daily")
282292
}
283293

284294
// StorageSCConfiguration will include all the required sc configs to operate blobber

‎code/go/0chain.net/blobbercore/handler/client_quota.go

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,10 @@ const (
2525
type ClientStats struct {
2626
ClientID string `gorm:"column:client_id;size:64;primaryKey" json:"client_id"`
2727
CreatedAt common.Timestamp `gorm:"created_at;primaryKey" json:"created"`
28-
TotalUpload uint64 `gorm:"column:total_upload" json:"total_upload"`
29-
TotalDownload uint64 `gorm:"column:total_download" json:"total_download"`
30-
TotalWM uint64 `gorm:"column:total_write_marker" json:"total_write_marker"`
28+
TotalUpload int64 `gorm:"column:total_upload" json:"total_upload"`
29+
TotalDownload int64 `gorm:"column:total_download" json:"total_download"`
30+
TotalWM int64 `gorm:"column:total_write_marker" json:"total_write_marker"`
31+
TotalZeroWM int64 `gorm:"-" json:"total_zero_write_marker"`
3132
}
3233

3334
func (ClientStats) TableName() string {
@@ -39,7 +40,7 @@ func (cs *ClientStats) BeforeCreate(tx *gorm.DB) error {
3940
return nil
4041
}
4142

42-
func GetUploadedData(clientID string) uint64 {
43+
func GetUploadedData(clientID string) int64 {
4344
mpLock.RLock()
4445
defer mpLock.RUnlock()
4546
cs := clientMap[clientID]
@@ -57,28 +58,10 @@ func AddUploadedData(clientID string, data int64) {
5758
cs = &ClientStats{ClientID: clientID}
5859
clientMap[clientID] = cs
5960
}
60-
cs.TotalUpload += uint64(data)
61+
cs.TotalUpload += data
6162
}
6263

63-
func GetDownloadedData(clientID string) uint64 {
64-
mpLock.RLock()
65-
defer mpLock.RUnlock()
66-
cs := clientMap[clientID]
67-
return cs.TotalDownload
68-
}
69-
70-
func AddDownloadedData(clientID string, data int64) {
71-
mpLock.Lock()
72-
defer mpLock.Unlock()
73-
cs := clientMap[clientID]
74-
if cs == nil {
75-
cs = &ClientStats{ClientID: clientID}
76-
clientMap[clientID] = cs
77-
}
78-
cs.TotalDownload += uint64(data)
79-
}
80-
81-
func GetWriteMarkerCount(clientID string) uint64 {
64+
func GetWriteMarkerCount(clientID string) int64 {
8265
mpLock.RLock()
8366
defer mpLock.RUnlock()
8467
cs := clientMap[clientID]
@@ -88,15 +71,27 @@ func GetWriteMarkerCount(clientID string) uint64 {
8871
return cs.TotalWM
8972
}
9073

91-
func AddWriteMarkerCount(clientID string, count int64) {
74+
func AddWriteMarkerCount(clientID string, zeroSizeWM bool) {
9275
mpLock.Lock()
9376
defer mpLock.Unlock()
9477
cs := clientMap[clientID]
9578
if cs == nil {
9679
cs = &ClientStats{ClientID: clientID}
9780
clientMap[clientID] = cs
9881
}
99-
cs.TotalWM += uint64(count)
82+
cs.TotalWM++
83+
if zeroSizeWM {
84+
cs.TotalZeroWM++
85+
}
86+
if cs.TotalZeroWM > config.Configuration.CommitZeroLimitDaily || cs.TotalWM > config.Configuration.CommitLimitDaily {
87+
SetBlacklist(clientID)
88+
}
89+
}
90+
91+
func SetBlacklist(clientID string) {
92+
blMap.Lock()
93+
blackListMap[clientID] = true
94+
blMap.Unlock()
10095
}
10196

10297
func CheckBlacklist(clientID string) bool {
@@ -109,10 +104,13 @@ func CheckBlacklist(clientID string) bool {
109104
func saveClientStats() {
110105
dbStats := make([]*ClientStats, 0, len(clientMap))
111106
mpLock.Lock()
107+
now := common.Now()
112108
for _, cs := range clientMap {
109+
cs.CreatedAt = now
110+
cs.TotalDownload = getDailyBlocks(cs.ClientID)
113111
dbStats = append(dbStats, cs)
114-
delete(clientMap, cs.ClientID)
115112
}
113+
clear(clientMap)
116114
mpLock.Unlock()
117115
_ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
118116
if len(dbStats) > 0 {
@@ -129,7 +127,7 @@ func saveClientStats() {
129127
})
130128
if err == nil {
131129
blMap.Lock()
132-
blackListMap = make(map[string]bool)
130+
clear(blackListMap)
133131
for _, clientID := range blackList {
134132
blackListMap[clientID] = true
135133
}
@@ -138,17 +136,19 @@ func saveClientStats() {
138136
}
139137

140138
func startBlackListWorker(ctx context.Context) {
141-
BlackListWorkerTime := 6 * time.Hour
139+
BlackListWorkerTime := 24 * time.Hour
142140
if config.Development() {
143141
BlackListWorkerTime = 10 * time.Second
144142
}
143+
saveClientStats()
145144

146145
for {
147146
select {
148147
case <-ctx.Done():
149148
return
150149
case <-time.After(BlackListWorkerTime):
151150
saveClientStats()
151+
cleanupDownloadLimit()
152152
}
153153
}
154154
}

0 commit comments

Comments
 (0)