diff --git a/api/api.go b/api/api.go index 91f4fd425..6f7a0b943 100644 --- a/api/api.go +++ b/api/api.go @@ -26,6 +26,7 @@ type Boost interface { Net // MethodGroup: Boost + BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) //perm:admin BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error) //perm:admin BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 8f3ce38a0..b98bf78bb 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -60,6 +60,8 @@ type BoostStruct struct { BoostIndexerListMultihashes func(p0 context.Context, p1 []byte) ([]multihash.Multihash, error) `perm:"admin"` + BoostIndexerRemoveAll func(p0 context.Context) ([]cid.Cid, error) `perm:"admin"` + BoostLegacyDealByProposalCid func(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) `perm:"admin"` BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string, p3 bool) (*ProviderDealRejectionInfo, error) `perm:"admin"` @@ -355,6 +357,17 @@ func (s *BoostStub) BoostIndexerListMultihashes(p0 context.Context, p1 []byte) ( return *new([]multihash.Multihash), ErrNotSupported } +func (s *BoostStruct) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) { + if s.Internal.BoostIndexerRemoveAll == nil { + return *new([]cid.Cid), ErrNotSupported + } + return s.Internal.BoostIndexerRemoveAll(p0) +} + +func (s *BoostStub) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) { + return *new([]cid.Cid), ErrNotSupported +} + func (s *BoostStruct) BoostLegacyDealByProposalCid(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) { if s.Internal.BoostLegacyDealByProposalCid == nil { return *new(legacytypes.MinerDeal), ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index ba155d660..e51ae0ab4 100644 Binary files a/build/openrpc/boost.json.gz and b/build/openrpc/boost.json.gz differ diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go index 0e4b0fd69..35dc919dd 100644 --- a/cmd/boostd/index.go +++ b/cmd/boostd/index.go @@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{ indexProvAnnounceLatestHttp, indexProvAnnounceDealRemovalAd, indexProvAnnounceDeal, + indexProvRemoveAllCmd, }, } @@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{ return nil }, } + +var indexProvRemoveAllCmd = &cli.Command{ + Name: "remove-all", + Usage: "Announce all removal ad for all contextIDs", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + + // get boost api + napi, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + // announce markets and boost deals + cids, err := napi.BoostIndexerRemoveAll(ctx) + if err != nil { + return err + } + for _, c := range cids { + fmt.Println("Published the removal ad with CID", c.String()) + } + return nil + }, +} diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index 859bfc871..5dfd09678 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -20,6 +20,7 @@ * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp) * [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal) * [BoostIndexerListMultihashes](#boostindexerlistmultihashes) + * [BoostIndexerRemoveAll](#boostindexerremoveall) * [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid) * [BoostOfflineDealWithData](#boostofflinedealwithdata) * [I](#i) @@ -374,7 +375,7 @@ Response: ``` ### BoostIndexerAnnounceAllDeals -There are not yet any comments for this method. + Perms: admin @@ -513,6 +514,20 @@ Response: ] ``` +### BoostIndexerRemoveAll +There are not yet any comments for this method. + +Perms: admin + +Inputs: `null` + +Response: +```json +[ + null +] +``` + ### BoostLegacyDealByProposalCid diff --git a/gql/resolver_ipni.go b/gql/resolver_ipni.go index 1b23892b4..7391b288a 100644 --- a/gql/resolver_ipni.go +++ b/gql/resolver_ipni.go @@ -229,3 +229,7 @@ func (r *resolver) IpniDistanceFromLatestAd(ctx context.Context, args struct { return count, nil } + +func (r *resolver) IpniRemovedAllAdsStatus(ctx context.Context) (bool, error) { + return r.idxProvWrapper.RemoveAllStatus(ctx), nil +} diff --git a/gql/schema.graphql b/gql/schema.graphql index 5e02f01f3..b9301f7ce 100644 --- a/gql/schema.graphql +++ b/gql/schema.graphql @@ -652,6 +652,9 @@ type RootQuery { """Get the latest IPNI advertisemen""" ipniDistanceFromLatestAd(LatestAdcid: String!, Adcid: String!): Int! + + """Get the IPNI Remove All Status""" + ipniRemovedAllAdsStatus: Boolean! } type RootMutation { diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index 0976ad1a2..ea2d230dd 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -7,6 +7,7 @@ import ( "fmt" "net/url" "os" + "time" "github.com/filecoin-project/boost/lib/legacy" "github.com/filecoin-project/boost/storagemarket/types/legacytypes" @@ -18,11 +19,11 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" chainTypes "github.com/filecoin-project/lotus/chain/types" "github.com/google/uuid" - cbor "github.com/ipfs/go-ipld-cbor" - "go.uber.org/fx" - "github.com/ipfs/go-datastore" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/ipld/go-ipld-prime" + "github.com/ipni/go-libipni/ingest/schema" + "go.uber.org/fx" "github.com/filecoin-project/boost/db" bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types" @@ -71,6 +72,7 @@ type Wrapper struct { bitswapEnabled bool httpEnabled bool stop context.CancelFunc + removeAllAds bool } func NewWrapper(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, directDealsDB *db.DirectDealsDB, dealsDB *db.DealsDB, @@ -127,7 +129,11 @@ func (w *Wrapper) Start(_ context.Context) { log.Info("starting index provider") - go w.checkForUpdates(runCtx) + if w.cfg.CurioMigration.Enable { + go w.tryAnnounceRemoveAll(runCtx) + } else { + go w.checkForUpdates(runCtx) + } } func (w *Wrapper) checkForUpdates(ctx context.Context) { @@ -867,3 +873,79 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u } return annCid, err } + +func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) { + var allAds []*schema.Advertisement + _, ad, err := w.prov.GetLatestAdv(ctx) + if err != nil { + return nil, err + } + allAds = append(allAds, ad) + + prev, err := cid.Parse(ad.PreviousID.String()) + if err != nil { + return nil, err + } + + for prev != cid.Undef { + ad, err := w.prov.GetAdv(ctx, prev) + if err != nil { + return nil, err + } + + prev, err = cid.Parse(ad.PreviousID.String()) + if err != nil { + return nil, err + } + } + + var entryAds []*schema.Advertisement + + for _, ad := range allAds { + if !ad.IsRm { + entryAds = append(entryAds, ad) + } + } + + var newAds []cid.Cid + + for _, ad := range entryAds { + a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID) + if err != nil { + if !errors.Is(err, provider.ErrContextIDNotFound) { + return nil, fmt.Errorf("failed to publish the removal ad: %w", err) + } + } + newAds = append(newAds, a) + } + + return newAds, nil + +} + +func (w *Wrapper) tryAnnounceRemoveAll(ctx context.Context) { + ticker := time.NewTicker(time.Minute) + + for { + select { + case <-ticker.C: + out, err := w.AnnounceRemoveAll(ctx) + if err != nil { + log.Errorw("error while announcing remove all", "err", err) + continue + } + if len(out) > 0 { + continue + } + log.Debugw("Cleaned up all the IPNI ads") + w.removeAllAds = true + return + case <-ctx.Done(): + return + } + } +} + +func (w *Wrapper) RemoveAllStatus(ctx context.Context) bool { + return w.removeAllAds +} diff --git a/node/config/def.go b/node/config/def.go index 7348db15c..ec39b0827 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -170,6 +170,9 @@ func DefaultBoost() *Boost { MaxDealsPerPublishMsg: 8, MaxPublishDealsFee: types.MustParseFIL("0.05"), }, + CurioMigration: CurioMigration{ + Enable: false, + }, } return cfg } diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 2f0745bea..50ca25f3e 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -145,6 +145,12 @@ a file containing the booster-bitswap peer id's private key. Can be left blank w Name: "IndexProvider", Type: "IndexProviderConfig", + Comment: ``, + }, + { + Name: "CurioMigration", + Type: "CurioMigration", + Comment: ``, }, }, @@ -194,6 +200,14 @@ a file containing the booster-bitswap peer id's private key. Can be left blank w Comment: `From address for eth_ state call`, }, }, + "CurioMigration": []DocField{ + { + Name: "Enable", + Type: "bool", + + Comment: `Enable limits the Boost functionality to prepare for the migration`, + }, + }, "DealPublishConfig": []DocField{ { Name: "ManualDealPublish", diff --git a/node/config/types.go b/node/config/types.go index 06e17159a..5b4b5acc3 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -51,6 +51,7 @@ type Boost struct { HttpDownload HttpDownloadConfig Retrievals RetrievalConfig IndexProvider IndexProviderConfig + CurioMigration CurioMigration } type WalletsConfig struct { @@ -399,3 +400,8 @@ type DealPublishConfig struct { // The maximum fee to pay when sending the PublishStorageDeals message MaxPublishDealsFee types.FIL } + +type CurioMigration struct { + // Enable limits the Boost functionality to prepare for the migration + Enable bool +} diff --git a/node/impl/boost.go b/node/impl/boost.go index 645fb0e49..c27be73ff 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error { func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) { return sm.StorageProvider.GetAsk(), nil } + +func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) { + return sm.IndexProvider.AnnounceRemoveAll(ctx) +} diff --git a/node/modules/directdeals.go b/node/modules/directdeals.go index 2094a555d..227f85f8a 100644 --- a/node/modules/directdeals.go +++ b/node/modules/directdeals.go @@ -34,6 +34,7 @@ func NewDirectDealsProvider(provAddr address.Address, cfg *config.Boost) func(lc ddpCfg := storagemarket.DDPConfig{ StartEpochSealingBuffer: abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer), RemoteCommp: cfg.Dealmaking.RemoteCommp, + CurioMigration: cfg.CurioMigration.Enable, } prov := storagemarket.NewDirectDealsProvider(ddpCfg, provAddr, fullnodeApi, secb, commpc, commpt, sps, directDealsDB, dl, piecedirectory, ip) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 80a1f606e..84cf5d922 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -486,6 +486,7 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func( DealLogDurationDays: cfg.Dealmaking.DealLogDurationDays, StorageFilter: cfg.Dealmaking.Filter, SealingPipelineCacheTimeout: time.Duration(cfg.Dealmaking.SealingPipelineCacheTimeout), + CurioMigration: cfg.CurioMigration.Enable, } dl := logs.NewDealLogger(logsDB) tspt := httptransport.New(h, dl, httptransport.NChunksOpt(cfg.HttpDownload.NChunks), httptransport.AllowPrivateIPsOpt(cfg.HttpDownload.AllowPrivateIPs)) diff --git a/react/src/Ipni.js b/react/src/Ipni.js index cd7bc782a..793fdcb26 100644 --- a/react/src/Ipni.js +++ b/react/src/Ipni.js @@ -6,6 +6,7 @@ import { IpniProviderInfoQuery, IpniLatestAdQuery, IpniDistanceFromLatestAdQuery, + IpniRemovedAllAdsStatus, } from "./gql"; import moment from "moment"; import React, {useEffect, useState} from "react"; @@ -89,6 +90,7 @@ function ProviderIpniInfoRender(props){ adcid: adCid } }) + const publishedRemoveAll = useQuery(IpniRemovedAllAdsStatus) return

Provider Indexer Info

@@ -111,13 +113,14 @@ function ProviderIpniInfoRender(props){   ({moment(data.LastAdvertisementTime).fromNow()} ago)   - {distance.data ? ({distance.data.ipniDistanceFromLatestAd} behind): ''} + {distance.data ? + ({distance.data.ipniDistanceFromLatestAd} behind) : ''} Latest Advertisement on Boost - {lad ? {lad}: ''} + {lad ? {lad} : ''} @@ -128,6 +131,12 @@ function ProviderIpniInfoRender(props){ ({moment(data.LastErrorTime).fromNow()} ago) + + Published Removal Ads for Curio Migration + + {publishedRemoveAll.data ? publishedRemoveAll.data: ''} + +
@@ -137,7 +146,7 @@ function ProviderConfig({configJson}) { const cfg = JSON.parse(configJson) return

Index Provider Config

- +
} diff --git a/react/src/gql.js b/react/src/gql.js index 2aa0a7a12..519136620 100644 --- a/react/src/gql.js +++ b/react/src/gql.js @@ -880,6 +880,12 @@ const IpniDistanceFromLatestAdQuery = gql` } `; +const IpniRemovedAllAdsStatus = gql` + query AppIpniRemovedAllAdsStatusQuery{ + ipniRemovedAllAdsStatus + } +`; + export { gqlClient, EpochQuery, @@ -931,4 +937,5 @@ export { StorageAskQuery, PublishPendingDealsMutation, PiecePayloadCidsQuery, + IpniRemovedAllAdsStatus, } diff --git a/storagemarket/direct_deals_provider.go b/storagemarket/direct_deals_provider.go index 1447b30ae..ffecd09f3 100644 --- a/storagemarket/direct_deals_provider.go +++ b/storagemarket/direct_deals_provider.go @@ -44,6 +44,7 @@ type DDPConfig struct { // Minimum start epoch buffer to give time for sealing of sector with deal StartEpochSealingBuffer abi.ChainEpoch Curio bool + CurioMigration bool } type DirectDealsProvider struct { @@ -181,6 +182,9 @@ func (ddp *DirectDealsProvider) Accept(ctx context.Context, entry *types.DirectD } func (ddp *DirectDealsProvider) Import(ctx context.Context, params smtypes.DirectDealParams) (*api.ProviderDealRejectionInfo, error) { + if ddp.config.CurioMigration { + return &api.ProviderDealRejectionInfo{Reason: "Boost is migrating to Curio"}, nil + } piececid := params.PieceCid.String() clientAddr := params.ClientAddr.String() log.Infow("received direct data import", "piececid", piececid, "filepath", params.FilePath, "clientAddr", clientAddr, "allocationId", params.AllocationID) diff --git a/storagemarket/provider.go b/storagemarket/provider.go index 5e09fee1c..b6af08047 100644 --- a/storagemarket/provider.go +++ b/storagemarket/provider.go @@ -84,6 +84,7 @@ type Config struct { SealingPipelineCacheTimeout time.Duration StorageFilter string Curio bool + CurioMigration bool } var log = logging.Logger("boost-provider") @@ -250,6 +251,9 @@ func (p *Provider) GetAsk() *legacytypes.SignedStorageAsk { // an offline deal (the deal must already have been proposed by the client) func (p *Provider) ImportOfflineDealData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (pi *api.ProviderDealRejectionInfo, err error) { p.dealLogger.Infow(dealUuid, "import data for offline deal", "filepath", filePath, "delete after import", delAfterImport) + if p.config.CurioMigration { + return &api.ProviderDealRejectionInfo{Reason: "Boost is migrating to Curio"}, nil + } // db should already have a deal with this uuid as the deal proposal should have been made beforehand ds, err := p.dealsDB.ByID(p.ctx, dealUuid) @@ -293,6 +297,9 @@ func (p *Provider) ImportOfflineDealData(ctx context.Context, dealUuid uuid.UUID // ExecuteDeal is called when the Storage Provider receives a deal proposal // from the network func (p *Provider) ExecuteDeal(ctx context.Context, dp *types.DealParams, clientPeer peer.ID) (*api.ProviderDealRejectionInfo, error) { + if p.config.CurioMigration { + return &api.ProviderDealRejectionInfo{Reason: "Boost is migrating to Curio"}, nil + } ctx, span := tracing.Tracer.Start(ctx, "Provider.ExecuteLibp2pDeal") defer span.End()