add indexer remove all

api/api.go

@@ -26,6 +26,7 @@ type Boost interface {
 	Net
 
 	// MethodGroup: Boost
+	BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error)                                                                               //perm:admin
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
 	BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error)                                           //perm:admin
 	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin

cmd/boostd/index.go

@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
 		indexProvAnnounceLatestHttp,
 		indexProvAnnounceDealRemovalAd,
 		indexProvAnnounceDeal,
+		indexProvRemoveAllCmd,
 	},
 }
 
@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
 		return nil
 	},
 }
+
+var indexProvRemoveAllCmd = &cli.Command{
+	Name:  "remove-all",
+	Usage: "Announce all removal ad for all contextIDs",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		// get boost api
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		// announce markets and boost deals
+		cids, err := napi.BoostIndexerRemoveAll(ctx)
+		if err != nil {
+			return err
+		}
+		for _, c := range cids {
+			fmt.Println("Published the removal ad with CID", c.String())
+		}
+		return nil
+	},
+}

documentation/en/api-v1-methods.md

@@ -20,6 +20,7 @@
   * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
   * [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal)
   * [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
+  * [BoostIndexerRemoveAll](#boostindexerremoveall)
   * [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid)
   * [BoostOfflineDealWithData](#boostofflinedealwithdata)
 * [I](#i)
@@ -374,7 +375,7 @@ Response:
 ```
 
 ### BoostIndexerAnnounceAllDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -513,6 +514,20 @@ Response:
 ]
 ```
 
+### BoostIndexerRemoveAll
+There are not yet any comments for this method.
+
+Perms: admin
+
+Inputs: `null`
+
+Response:
+```json
+[
+  null
+]
+```
+
 ### BoostLegacyDealByProposalCid
 
 

gql/resolver_ipni.go

@@ -229,3 +229,7 @@ func (r *resolver) IpniDistanceFromLatestAd(ctx context.Context, args struct {
 
 	return count, nil
 }
+
+func (r *resolver) IpniRemovedAllAdsStatus(ctx context.Context) (bool, error) {
+	return r.idxProvWrapper.RemoveAllStatus(ctx), nil
+}

gql/schema.graphql

@@ -652,6 +652,9 @@ type RootQuery {
 
   """Get the latest IPNI advertisemen"""
   ipniDistanceFromLatestAd(LatestAdcid: String!, Adcid: String!): Int!
+
+  """Get the IPNI Remove All Status"""
+  ipniRemovedAllAdsStatus: Boolean!
 }
 
 type RootMutation {

indexprovider/wrapper.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"time"
 
 	"github.com/filecoin-project/boost/lib/legacy"
 	"github.com/filecoin-project/boost/storagemarket/types/legacytypes"
@@ -18,11 +19,11 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	chainTypes "github.com/filecoin-project/lotus/chain/types"
 	"github.com/google/uuid"
-	cbor "github.com/ipfs/go-ipld-cbor"
-	"go.uber.org/fx"
-
 	"github.com/ipfs/go-datastore"
+	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/ipld/go-ipld-prime"
+	"github.com/ipni/go-libipni/ingest/schema"
+	"go.uber.org/fx"
 
 	"github.com/filecoin-project/boost/db"
 	bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types"
@@ -71,6 +72,7 @@ type Wrapper struct {
 	bitswapEnabled bool
 	httpEnabled    bool
 	stop           context.CancelFunc
+	removeAllAds   bool
 }
 
 func NewWrapper(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, directDealsDB *db.DirectDealsDB, dealsDB *db.DealsDB,
@@ -127,7 +129,11 @@ func (w *Wrapper) Start(_ context.Context) {
 
 	log.Info("starting index provider")
 
-	go w.checkForUpdates(runCtx)
+	if w.cfg.CurioMigration.Enable {
+		go w.tryAnnounceRemoveAll(runCtx)
+	} else {
+		go w.checkForUpdates(runCtx)
+	}
 }
 
 func (w *Wrapper) checkForUpdates(ctx context.Context) {
@@ -867,3 +873,79 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u
 	}
 	return annCid, err
 }
+
+func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+	var allAds []*schema.Advertisement
+	_, ad, err := w.prov.GetLatestAdv(ctx)
+	if err != nil {
+		return nil, err
+	}
+	allAds = append(allAds, ad)
+
+	prev, err := cid.Parse(ad.PreviousID.String())
+	if err != nil {
+		return nil, err
+	}
+
+	for prev != cid.Undef {
+		ad, err := w.prov.GetAdv(ctx, prev)
+		if err != nil {
+			return nil, err
+		}
+
+		prev, err = cid.Parse(ad.PreviousID.String())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var entryAds []*schema.Advertisement
+
+	for _, ad := range allAds {
+		if !ad.IsRm {
+			entryAds = append(entryAds, ad)
+		}
+	}
+
+	var newAds []cid.Cid
+
+	for _, ad := range entryAds {
+		a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID)
+		if err != nil {
+			if !errors.Is(err, provider.ErrContextIDNotFound) {
+				return nil, fmt.Errorf("failed to publish the removal ad: %w", err)
+			}
+		}
+		newAds = append(newAds, a)
+	}
+
+	return newAds, nil
+
+}
+
+func (w *Wrapper) tryAnnounceRemoveAll(ctx context.Context) {
+	ticker := time.NewTicker(time.Minute)
+
+	for {
+		select {
+		case <-ticker.C:
+			out, err := w.AnnounceRemoveAll(ctx)
+			if err != nil {
+				log.Errorw("error while announcing remove all", "err", err)
+				continue
+			}
+			if len(out) > 0 {
+				continue
+			}
+			log.Debugw("Cleaned up all the IPNI ads")
+			w.removeAllAds = true
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (w *Wrapper) RemoveAllStatus(ctx context.Context) bool {
+	return w.removeAllAds
+}

node/config/def.go

@@ -170,6 +170,9 @@ func DefaultBoost() *Boost {
 			MaxDealsPerPublishMsg: 8,
 			MaxPublishDealsFee:    types.MustParseFIL("0.05"),
 		},
+		CurioMigration: CurioMigration{
+			Enable: false,
+		},
 	}
 	return cfg
 }

node/config/types.go

@@ -51,6 +51,7 @@ type Boost struct {
 	HttpDownload        HttpDownloadConfig
 	Retrievals          RetrievalConfig
 	IndexProvider       IndexProviderConfig
+	CurioMigration      CurioMigration
 }
 
 type WalletsConfig struct {
@@ -399,3 +400,8 @@ type DealPublishConfig struct {
 	// The maximum fee to pay when sending the PublishStorageDeals message
 	MaxPublishDealsFee types.FIL
 }
+
+type CurioMigration struct {
+	// Enable limits the Boost functionality to prepare for the migration
+	Enable bool
+}

node/impl/boost.go

@@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error {
 func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) {
 	return sm.StorageProvider.GetAsk(), nil
 }
+
+func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+	return sm.IndexProvider.AnnounceRemoveAll(ctx)
+}

node/modules/directdeals.go

@@ -34,6 +34,7 @@ func NewDirectDealsProvider(provAddr address.Address, cfg *config.Boost) func(lc
 		ddpCfg := storagemarket.DDPConfig{
 			StartEpochSealingBuffer: abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer),
 			RemoteCommp:             cfg.Dealmaking.RemoteCommp,
+			CurioMigration:          cfg.CurioMigration.Enable,
 		}
 
 		prov := storagemarket.NewDirectDealsProvider(ddpCfg, provAddr, fullnodeApi, secb, commpc, commpt, sps, directDealsDB, dl, piecedirectory, ip)

node/modules/storageminer.go

@@ -486,6 +486,7 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func(
 			DealLogDurationDays:         cfg.Dealmaking.DealLogDurationDays,
 			StorageFilter:               cfg.Dealmaking.Filter,
 			SealingPipelineCacheTimeout: time.Duration(cfg.Dealmaking.SealingPipelineCacheTimeout),
+			CurioMigration:              cfg.CurioMigration.Enable,
 		}
 		dl := logs.NewDealLogger(logsDB)
 		tspt := httptransport.New(h, dl, httptransport.NChunksOpt(cfg.HttpDownload.NChunks), httptransport.AllowPrivateIPsOpt(cfg.HttpDownload.AllowPrivateIPs))

react/src/Ipni.js

@@ -6,6 +6,7 @@ import {
     IpniProviderInfoQuery,
     IpniLatestAdQuery,
     IpniDistanceFromLatestAdQuery,
+    IpniRemovedAllAdsStatus,
 } from "./gql";
 import moment from "moment";
 import React, {useEffect, useState} from "react";
@@ -89,6 +90,7 @@ function ProviderIpniInfoRender(props){
             adcid: adCid
         }
     })
+    const publishedRemoveAll = useQuery(IpniRemovedAllAdsStatus)
     return <div className="ipni-prov-info">
         <h3>Provider Indexer Info</h3>
         <div className="subtitle">
@@ -111,13 +113,14 @@ function ProviderIpniInfoRender(props){
                     &nbsp;
                     <span className="aux">({moment(data.LastAdvertisementTime).fromNow()} ago)</span>
                     &nbsp;
-                    {distance.data ? <span className="aux">({distance.data.ipniDistanceFromLatestAd} behind)</span>: ''}
+                    {distance.data ?
+                        <span className="aux">({distance.data.ipniDistanceFromLatestAd} behind)</span> : ''}
                 </td>
             </tr>
             <tr>
                 <th>Latest Advertisement on Boost</th>
                 <td>
-                    {lad ? <Link to={'/ipni/ad/'+lad}>{lad}</Link>: ''}
+                    {lad ? <Link to={'/ipni/ad/' + lad}>{lad}</Link> : ''}
                 </td>
             </tr>
             <tr>
@@ -128,6 +131,12 @@ function ProviderIpniInfoRender(props){
                     <span className="aux">({moment(data.LastErrorTime).fromNow()} ago)</span>
                 </td>
             </tr>
+            <tr>
+                <th>Published Removal Ads for Curio Migration</th>
+                <td>
+                    {publishedRemoveAll.data ? publishedRemoveAll.data: ''}
+                </td>
+            </tr>
             </tbody>
         </table>
     </div>
@@ -137,7 +146,7 @@ function ProviderConfig({configJson}) {
     const cfg = JSON.parse(configJson)
     return <div>
         <h3>Index Provider Config</h3>
-        <ExpandableJSObject v={cfg} topLevel={false} expanded={true} key={'config'} />
+        <ExpandableJSObject v={cfg} topLevel={false} expanded={true} key={'config'}/>
     </div>
 }