add indexer remove all

Author: LexLuthr

api/api.go

@@ -26,6 +26,7 @@ type Boost interface {
 	Net
 
 	// MethodGroup: Boost
+	BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error)                                                                               //perm:admin
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
 	BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error)                                           //perm:admin
 	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin

api/proxy_gen.go

@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
 		indexProvAnnounceLatestHttp,
 		indexProvAnnounceDealRemovalAd,
 		indexProvAnnounceDeal,
+		indexProvRemoveAllCmd,
 	},
 }
 
@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
 		return nil
 	},
 }
+
+var indexProvRemoveAllCmd = &cli.Command{
+	Name:  "remove-all",
+	Usage: "Announce all removal ad for all contextIDs",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		// get boost api
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		// announce markets and boost deals
+		cids, err := napi.BoostIndexerRemoveAll(ctx)
+		if err != nil {
+			return err
+		}
+		for _, c := range cids {
+			fmt.Println("Published the removal ad with CID", c.String())
+		}
+		return nil
+	},
+}

build/openrpc/boost.json.gz

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/filecoin-project/boost/node/repo"
+	lotus_repo "github.com/filecoin-project/lotus/node/repo"
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
+)
+
+var removeIPNICmd = &cli.Command{
+	Name:        "remove-ipni",
+	Description: "Sends remove ad for all the currently published Ads to IPNI",
+	Usage:       "migrate-curio remove-ipni",
+	Before:      before,
+	Action: func(cctx *cli.Context) error {
+		ctx := cctx.Context
+
+		repoDir, err := homedir.Expand(cctx.String(FlagBoostRepo))
+		if err != nil {
+			return err
+		}
+
+		r, err := lotus_repo.NewFS(repoDir)
+		if err != nil {
+			return err
+		}
+		ok, err := r.Exists()
+		if err != nil {
+			return err
+		}
+		if !ok {
+			return fmt.Errorf("repo at '%s' is not initialized", cctx.String(FlagBoostRepo))
+		}
+
+		lr, err := r.Lock(repo.Boost)
+		if err != nil {
+			return err
+		}
+
+		mds, err := lr.Datastore(ctx, "/metadata")
+		if err != nil {
+			return err
+		}
+
+		return migrate(cctx, repoDir)
+	},
+}

cmd/boostd/index.go

@@ -20,6 +20,7 @@
   * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
   * [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal)
   * [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
+  * [BoostIndexerRemoveAll](#boostindexerremoveall)
   * [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid)
   * [BoostOfflineDealWithData](#boostofflinedealwithdata)
 * [I](#i)
@@ -374,7 +375,7 @@ Response:
 ```
 
 ### BoostIndexerAnnounceAllDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -513,6 +514,20 @@ Response:
 ]
 ```
 
+### BoostIndexerRemoveAll
+There are not yet any comments for this method.
+
+Perms: admin
+
+Inputs: `null`
+
+Response:
+```json
+[
+  null
+]
+```
+
 ### BoostLegacyDealByProposalCid
 
 

cmd/migrate-curio/ipni.go

@@ -18,11 +18,11 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	chainTypes "github.com/filecoin-project/lotus/chain/types"
 	"github.com/google/uuid"
-	cbor "github.com/ipfs/go-ipld-cbor"
-	"go.uber.org/fx"
-
 	"github.com/ipfs/go-datastore"
+	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/ipld/go-ipld-prime"
+	"github.com/ipni/go-libipni/ingest/schema"
+	"go.uber.org/fx"
 
 	"github.com/filecoin-project/boost/db"
 	bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types"
@@ -126,8 +126,6 @@ func (w *Wrapper) Start(_ context.Context) {
 	}()
 
 	log.Info("starting index provider")
-
-	go w.checkForUpdates(runCtx)
 }
 
 func (w *Wrapper) checkForUpdates(ctx context.Context) {
@@ -867,3 +865,52 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u
 	}
 	return annCid, err
 }
+
+func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+	var allAds []*schema.Advertisement
+	_, ad, err := w.prov.GetLatestAdv(ctx)
+	if err != nil {
+		return nil, err
+	}
+	allAds = append(allAds, ad)
+
+	prev, err := cid.Parse(ad.PreviousID.String())
+	if err != nil {
+		return nil, err
+	}
+
+	for prev != cid.Undef {
+		ad, err := w.prov.GetAdv(ctx, prev)
+		if err != nil {
+			return nil, err
+		}
+
+		prev, err = cid.Parse(ad.PreviousID.String())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var entryAds []*schema.Advertisement
+
+	for _, ad := range allAds {
+		if !ad.IsRm {
+			entryAds = append(entryAds, ad)
+		}
+	}
+
+	var newAds []cid.Cid
+
+	for _, ad := range entryAds {
+		a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID)
+		if err != nil {
+			if !errors.Is(err, provider.ErrContextIDNotFound) {
+				return nil, fmt.Errorf("failed to publish the removal ad: %w", err)
+			}
+		}
+		newAds = append(newAds, a)
+	}
+
+	return newAds, nil
+
+}

documentation/en/api-v1-methods.md

@@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error {
 func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) {
 	return sm.StorageProvider.GetAsk(), nil
 }
+
+func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+	return sm.IndexProvider.AnnounceRemoveAll(ctx)
+}