Skip to content

Commit 7866fbd

Browse files
LexLuthrLexLuthr
committed
add indexer remove all
1 parent 85004d5 commit 7866fbd

File tree

18 files changed

+209
-8
lines changed

18 files changed

+209
-8
lines changed

‎api/api.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type Boost interface {
2626
Net
2727

2828
// MethodGroup: Boost
29+
BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) //perm:admin
2930
BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
3031
BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error) //perm:admin
3132
BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin

‎api/proxy_gen.go

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎build/openrpc/boost.json.gz

63 Bytes
Binary file not shown.

‎cmd/boostd/index.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
2222
indexProvAnnounceLatestHttp,
2323
indexProvAnnounceDealRemovalAd,
2424
indexProvAnnounceDeal,
25+
indexProvRemoveAllCmd,
2526
},
2627
}
2728

@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
303304
return nil
304305
},
305306
}
307+
308+
var indexProvRemoveAllCmd = &cli.Command{
309+
Name: "remove-all",
310+
Usage: "Announce all removal ad for all contextIDs",
311+
Flags: []cli.Flag{},
312+
Action: func(cctx *cli.Context) error {
313+
ctx := lcli.ReqContext(cctx)
314+
315+
// get boost api
316+
napi, closer, err := bcli.GetBoostAPI(cctx)
317+
if err != nil {
318+
return err
319+
}
320+
defer closer()
321+
322+
// announce markets and boost deals
323+
cids, err := napi.BoostIndexerRemoveAll(ctx)
324+
if err != nil {
325+
return err
326+
}
327+
for _, c := range cids {
328+
fmt.Println("Published the removal ad with CID", c.String())
329+
}
330+
return nil
331+
},
332+
}

‎documentation/en/api-v1-methods.md

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
* [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
2121
* [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal)
2222
* [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
23+
* [BoostIndexerRemoveAll](#boostindexerremoveall)
2324
* [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid)
2425
* [BoostOfflineDealWithData](#boostofflinedealwithdata)
2526
* [I](#i)
@@ -374,7 +375,7 @@ Response:
374375
```
375376

376377
### BoostIndexerAnnounceAllDeals
377-
There are not yet any comments for this method.
378+
378379

379380
Perms: admin
380381

@@ -513,6 +514,20 @@ Response:
513514
]
514515
```
515516

517+
### BoostIndexerRemoveAll
518+
There are not yet any comments for this method.
519+
520+
Perms: admin
521+
522+
Inputs: `null`
523+
524+
Response:
525+
```json
526+
[
527+
null
528+
]
529+
```
530+
516531
### BoostLegacyDealByProposalCid
517532

518533

‎gql/resolver_ipni.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,3 +229,7 @@ func (r *resolver) IpniDistanceFromLatestAd(ctx context.Context, args struct {
229229

230230
return count, nil
231231
}
232+
233+
func (r *resolver) IpniRemovedAllAdsStatus(ctx context.Context) (bool, error) {
234+
return r.idxProvWrapper.RemoveAllStatus(ctx), nil
235+
}

‎gql/schema.graphql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,9 @@ type RootQuery {
652652

653653
"""Get the latest IPNI advertisemen"""
654654
ipniDistanceFromLatestAd(LatestAdcid: String!, Adcid: String!): Int!
655+
656+
"""Get the IPNI Remove All Status"""
657+
ipniRemovedAllAdsStatus: Boolean!
655658
}
656659

657660
type RootMutation {

‎indexprovider/wrapper.go

Lines changed: 86 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"fmt"
88
"net/url"
99
"os"
10+
"time"
1011

1112
"github.com/filecoin-project/boost/lib/legacy"
1213
"github.com/filecoin-project/boost/storagemarket/types/legacytypes"
@@ -18,11 +19,11 @@ import (
1819
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
1920
chainTypes "github.com/filecoin-project/lotus/chain/types"
2021
"github.com/google/uuid"
21-
cbor "github.com/ipfs/go-ipld-cbor"
22-
"go.uber.org/fx"
23-
2422
"github.com/ipfs/go-datastore"
23+
cbor "github.com/ipfs/go-ipld-cbor"
2524
"github.com/ipld/go-ipld-prime"
25+
"github.com/ipni/go-libipni/ingest/schema"
26+
"go.uber.org/fx"
2627

2728
"github.com/filecoin-project/boost/db"
2829
bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types"
@@ -71,6 +72,7 @@ type Wrapper struct {
7172
bitswapEnabled bool
7273
httpEnabled bool
7374
stop context.CancelFunc
75+
removeAllAds bool
7476
}
7577

7678
func NewWrapper(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, directDealsDB *db.DirectDealsDB, dealsDB *db.DealsDB,
@@ -127,7 +129,11 @@ func (w *Wrapper) Start(_ context.Context) {
127129

128130
log.Info("starting index provider")
129131

130-
go w.checkForUpdates(runCtx)
132+
if w.cfg.CurioMigration.Enable {
133+
go w.tryAnnounceRemoveAll(runCtx)
134+
} else {
135+
go w.checkForUpdates(runCtx)
136+
}
131137
}
132138

133139
func (w *Wrapper) checkForUpdates(ctx context.Context) {
@@ -867,3 +873,79 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u
867873
}
868874
return annCid, err
869875
}
876+
877+
func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) {
878+
var allAds []*schema.Advertisement
879+
_, ad, err := w.prov.GetLatestAdv(ctx)
880+
if err != nil {
881+
return nil, err
882+
}
883+
allAds = append(allAds, ad)
884+
885+
prev, err := cid.Parse(ad.PreviousID.String())
886+
if err != nil {
887+
return nil, err
888+
}
889+
890+
for prev != cid.Undef {
891+
ad, err := w.prov.GetAdv(ctx, prev)
892+
if err != nil {
893+
return nil, err
894+
}
895+
896+
prev, err = cid.Parse(ad.PreviousID.String())
897+
if err != nil {
898+
return nil, err
899+
}
900+
}
901+
902+
var entryAds []*schema.Advertisement
903+
904+
for _, ad := range allAds {
905+
if !ad.IsRm {
906+
entryAds = append(entryAds, ad)
907+
}
908+
}
909+
910+
var newAds []cid.Cid
911+
912+
for _, ad := range entryAds {
913+
a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID)
914+
if err != nil {
915+
if !errors.Is(err, provider.ErrContextIDNotFound) {
916+
return nil, fmt.Errorf("failed to publish the removal ad: %w", err)
917+
}
918+
}
919+
newAds = append(newAds, a)
920+
}
921+
922+
return newAds, nil
923+
924+
}
925+
926+
func (w *Wrapper) tryAnnounceRemoveAll(ctx context.Context) {
927+
ticker := time.NewTicker(time.Minute)
928+
929+
for {
930+
select {
931+
case <-ticker.C:
932+
out, err := w.AnnounceRemoveAll(ctx)
933+
if err != nil {
934+
log.Errorw("error while announcing remove all", "err", err)
935+
continue
936+
}
937+
if len(out) > 0 {
938+
continue
939+
}
940+
log.Debugw("Cleaned up all the IPNI ads")
941+
w.removeAllAds = true
942+
return
943+
case <-ctx.Done():
944+
return
945+
}
946+
}
947+
}
948+
949+
func (w *Wrapper) RemoveAllStatus(ctx context.Context) bool {
950+
return w.removeAllAds
951+
}

‎node/config/def.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,9 @@ func DefaultBoost() *Boost {
170170
MaxDealsPerPublishMsg: 8,
171171
MaxPublishDealsFee: types.MustParseFIL("0.05"),
172172
},
173+
CurioMigration: CurioMigration{
174+
Enable: false,
175+
},
173176
}
174177
return cfg
175178
}

‎node/config/doc_gen.go

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)