From 7866fbd7c4c2ffbe13497deada026adbfc176473 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 2 Oct 2024 21:40:01 +0400 Subject: [PATCH] add indexer remove all --- api/api.go | 1 + api/proxy_gen.go | 13 ++++ build/openrpc/boost.json.gz | Bin 3229 -> 3292 bytes cmd/boostd/index.go | 27 ++++++++ documentation/en/api-v1-methods.md | 17 ++++- gql/resolver_ipni.go | 4 ++ gql/schema.graphql | 3 + indexprovider/wrapper.go | 90 +++++++++++++++++++++++-- node/config/def.go | 3 + node/config/doc_gen.go | 14 ++++ node/config/types.go | 6 ++ node/impl/boost.go | 4 ++ node/modules/directdeals.go | 1 + node/modules/storageminer.go | 1 + react/src/Ipni.js | 15 ++++- react/src/gql.js | 7 ++ storagemarket/direct_deals_provider.go | 4 ++ storagemarket/provider.go | 7 ++ 18 files changed, 209 insertions(+), 8 deletions(-) diff --git a/api/api.go b/api/api.go index 91f4fd425..6f7a0b943 100644 --- a/api/api.go +++ b/api/api.go @@ -26,6 +26,7 @@ type Boost interface { Net // MethodGroup: Boost + BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) //perm:admin BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error) //perm:admin BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 8f3ce38a0..b98bf78bb 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -60,6 +60,8 @@ type BoostStruct struct { BoostIndexerListMultihashes func(p0 context.Context, p1 []byte) ([]multihash.Multihash, error) `perm:"admin"` + BoostIndexerRemoveAll func(p0 context.Context) ([]cid.Cid, error) `perm:"admin"` + BoostLegacyDealByProposalCid func(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) `perm:"admin"` BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string, p3 bool) (*ProviderDealRejectionInfo, error) `perm:"admin"` @@ -355,6 +357,17 @@ func (s *BoostStub) BoostIndexerListMultihashes(p0 context.Context, p1 []byte) ( return *new([]multihash.Multihash), ErrNotSupported } +func (s *BoostStruct) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) { + if s.Internal.BoostIndexerRemoveAll == nil { + return *new([]cid.Cid), ErrNotSupported + } + return s.Internal.BoostIndexerRemoveAll(p0) +} + +func (s *BoostStub) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) { + return *new([]cid.Cid), ErrNotSupported +} + func (s *BoostStruct) BoostLegacyDealByProposalCid(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) { if s.Internal.BoostLegacyDealByProposalCid == nil { return *new(legacytypes.MinerDeal), ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index ba155d660c236bdf109eb97a5ab34dcce3787a36..e51ae0ab4c6d0ba3f84331a9d563759eb00621a9 100644 GIT binary patch literal 3292 zcmV<23?uU&iwFP!00000|Li?$Q`usvJuDK zdn~E60eA5M=bl7ogG1CC^s#|&L3ori+{Nuir_sby13Mtc-12bu@*Xqds8t`S4JN0o zabR=v#*~{7I>zht_(qZ`}Xace`O<|nP|&HZ-ULvrSMI;jpmkaxlLqpCgDyt z4#WQ&$Za8$T8%?$ZKJKrtKmd~fj|f@w$c4p_Epjm@)~_6K`>&^SfAR+)UF3|)eB5Z<8?l{1theli=Ii?;8iQ2#hc8MTPNEBZv;tnS}F863a!|3Zd z;=8R3p(8TQVWK{mV{l0Nx&zuWiTV~WNk^?LL|fm-)EpzqENT)7Y7fV^B{kKz zJ1#MAz?$tMCoXIw!YoA4shTw=-dNEM?ug^sph&}f+rBcenu@!Z*pq^?YkV~|uz-Jj zD!{^BDSS{zbp@zRE!-Uu+k@$qn&kPmRAh=M9CAp8GPB_vBBwCIe=)oH_xZutWWS8z z|KGpItKz2_Q)dqu^N5?C+Ib$&PgIWmArjzl3CO1G1{>HHHtyoFl&;s^-MtQ`G)(U7 zjv`s!aRvV`m~wZhq;z*^^Wm;TJPBf#xOCSQ{BH7}YhaCQ{(mR!=02WIjRJ*mPXEnP zvAj}H%Zn8QWwNwZP!RQ+p}Dd;h_K983AKzrMH#0oYp=qw0`fB78EQ!SIT%)SA!s%w zg@3EvNueLjc-AVQ^|nX_920L<@joVBvFHyux68@?G4b?rA5wEA*$+{y<^Flez2cb+ zU82EayRLRhVWZXoVb5ER+dt>Xv08itBulR4G8`|Cz$Ae45s z0zr$Y7lOrIJZiSuojq*e_|)mK6EXxlJoC9ELVj?$IaXj>&6a^bFl&C`TyjCKVIZgp z0}#=dxS4`}$ZeZQ5DJsvix828;b}a6POljWMd2P_1U%6;@DY&&ce4S|bZX$f_uFV> zQwGUNfiT|<*Q5_W01%35xYa)D{L1+spZ>l3_t(Gwx`pFErTb~W=ZP=B9GHLH9Upu= zZy$Rf_@|3Iae8w5>z{Ijjo`W!)HBXyO#40?a-UgB!3~I1hYzg@x+S)+_~=3q=8Y7U z1`h9gudOIICX^}U6pWwt6bB6ap-sJUib>AFlsq{8I|KhV z8hIdNb~yodEywYv!!ejQE?0hQx0U|5jA=gzkwyOuWFSDqmmC}T^T9-dcLpZVTP4Hq z4E&LJQq7`pn{aB6G43-HE_i?NxgU-yQ74uasKrn)KuD^@Ffw^=YUn-#;yzL@Gvm5Q zHI0soRz#nt^Mt83zZa}fJ`Pbdw1|WqNgYU5MJ~v6fqS)hYxP3~;%Ee|!s-|?2P_)= zV@MehlMl?~mO4Tg4~Yl+dkWRe;X(l@CSJ0Z50%rQ)~GBR<;V&fSmeO8IQHIRyvX={e!nmsogo$C)(r{A7e`mGN6>yW<=`RkCs z4*BblzYh89kiQQ3>yW<=`RkDXMTY!8HJ63_RDiki@9VS0d4YW#JS8*3>sFC|I-oiO zbsoxDpN)8Nq(6hU!Qei9mIsgL^XO6~ru-5_10`e1o^5lJ1Qb5P-Dd0)DD@t?{-KQa zENd>*D-+-4hc!oFgM{8lg6KQy8e-Ww1Bc(j7v=%6!8+vbB>v|7hPuifkDc??Jo~xB zmU6>Wr?2V&B$x3c4W)+GI*V4#-&|@t6Ar3wjomQq?6oA0dzVz^+4`II9z9b>i$9CC)aGQf;SLx=zZ7*^2`^&@pt();!m-THhi z*LLuABJ|m}vjJ2zm?~n4xJ@~e&zskN?v&@X6@yC+l)NxAzlLQSZAGO;g|Ve(za{Mr z;)<@y@uvP*4<32eWj-$N20Y%%{#))Y*zUTk#~b6?K_UUFK7l`MkUoQSy97 z{oKZep4)h$k^Q^9YLR`Iyj|rPs(fI@k5R1=)JJB?fQ@?NpZ>ok8V zukub55v%rK&4r^)ny;$dLXeN3*JijsNGiRSyVoLjt-S80xjc6)rMoHFQ?T+JjmxW$ zzUvC3+T+m#GCI(M`x!BDO8szfIiD!Kn(v~v$jm7QU$^U7twvsp8foWsNYzOSToZHE za@Sl~4;HT^TResqNuQ^VR-UdL*6QvxtGmv!MzWDepHNTENCInA`78j+_m_f~J)o`2 zD`$2Svvp;)-oj=}UoN59W@QDKJ>(?K=H}Z1tu>z$YrX`^%n9PaN)p$$IVRav&AQC3dhdtPDJ*EG!Ewl<%DF zmWa7_pXwr?f6h8ydAg3YPD^STB15RWS!D`V>nc)h6=_mjmTHdBv#gIkv%J($T&IzY z1Q)P+Md^&3E3OZVmAiZG==!1(RBYf=ki;UABqQfG@x}{X9CYQEr{HCkkLU|r)Ah28Ehy%=uDxgAGx9xn(M_tpwRcOXO@=mn--ayj4O5U2bJT% z)u^6&&_F4CC7prXHVQAnKyC*q6|`PsAPp9qYZ0VBqB@c~h&n<2GP{R(nO(xTZadpb zseBE}LJ7*MKL;k*bbipQE$ikDFSU6?=jWW}4W~rhfcylq=iOA(&gqrlx6afr{pd> z7Ukb&sYex?dy+VAl5V_nsX{7FHo`xjd6ZbH`mlLLAr6mPVMUB;84qus8l_gu(X`}> z)xtq)u8<&!Zl;tQo!oMn=?ffI4KZy@@w`%3FU&5y%W1Uo8&fXn0Woj-OteKq&v0yodW8e`(Lo$96R9{z4~v&77wY2iAd4Z7V4K_{ha9=Tk`n za)1>6TAnKnIRMucjC?$wnvB#xj&JnG@z*Bpt(*t&23FJ|rmB2L%c}LHn=M;kZj=4% zZgU&pk9N+3VgoBI{yPF<#|pA4t7IV**nw2?aw#)q0k6+jT4duyWL^IBG&TIwN@Ldg aVGl- literal 3229 zcmV;O3}W*iiwFP!00000|Li?$Q`4yMq$u4yB0^J*Bo+|bU z(St2{Wtoy?^5374FR>HL@g+c^c*;AOsiN=j7_paeLGRunM(wcQ7b!sc zf}vLozD1C>4fI}!Ph6h)uFHi4OZA+qUs2~g&|7X>;kNyS&A+g(U%z4ld%oikF~fa) zJ_P}Y2tbTWGy{nUn<0}s4lwDV2^UCCsfR+Mwy=R+B8U?b#TSaWHu+Vsh2>Lx_DFYF7z%4M**0(7&r-(9(nnZ${!|`oNP4(@L zOU!Gq7PH8i3)_e=3lVgpdQFKpRdj<};_(lFn)FAc1^;{FBpq@e5yU(O9I;2)m~ zuy9`rAJkG^0cuhU_b0^mV1B7OdA=QJS3#GS$(`Lv zB+EOl;Qs|v?v9m|?v8Cf-gSs4LF^Kj?z)2C&i-=+taZiz@3`CE!}GaOpb*aJzga4l zR|;x)v0|W1mevXiqFyt!Ypa6@%Y2p4$oNx~amupxDr_qtFY}$ThNPc@W2I5%G#ef6TdEPWF$8rz=g-5#ZM5~ua}+JT)=2Oh zYJpG#&m<}E^y4Y?GR>%6NupB-Ke#k67GKVdZ2uJg+tK`r|0&m!N{FVeW2Q1z>C(Us zxgCZ;J8ul^P?q`>_a{P-*#@$hzxx@y*LE!?~EyNpL?K0L|wH9(un`CN^b| ztP}|I-Ed5L@go4CsD?Y;!``o)|MBtP+kb!k`>z`~`ct|e-wZtQ`Imk3kK3dD4`+x`ek)FnZ+`t#POuSdw}N`gxr}K)WMl3#ODVV!k!tawH9o0b(oPj3VjJF5JN3e(!Y^<)(x(g`9%%dQWk{$RFF(o2HoL3{1&`<-axXZ=!U{`V)e>#|gdF^uLxAr^fkL#HBy%1USPeBF(M10Ayfj{lfBzUJ_0=-c({MNu9 zh$mGq3O5O-4jAJ;GvS;MN1ujas}gl$S%F#%1p|bnN(>{D=cbzOG9d0E^|COot5nnI zxM)Q5dA>}TYW)Yn2<77tMMH~7*pbwMWK`sW%vZQqi?>!kR3MH<&?=0M5p%$z!5?GF zh?u=+Cb!fQ`glw{c(bQa%`7eyfMVh$WBFJ)9ct`Gt1TRqnJ%lQszOM#Wf^XXG;h^O z5?fHy7h_Qau<7ft3iU^rz@m;v48As#Q&8)n zoDJE87f1RFX!jW0r`Ph}@q8X#s>GC^gJ_^+OgXS^Zjyk)hq&L4eFCN4L)Sl)(ScgFiB{W@f`%%JU6$?dDF+_h<>Tosm0BdzON zM&t*k39d{qJO$!~sw6FQRRv`MR7co&ihPs#V;IQSZ`T<}-*IMj?PEN)&^+FkVT>od zN4uC>!{eF7hc}3tj9`;et&&pJB|0A=kNLo^Lz6gb5@$`~>=`A_?jgzb-F^3hEY$tF z%S{faar0lKoBvaLjhjDY7Tkh}MQ32!Y6q(x99siP69R09?tHE{p94wn+gEh!^RayF z6XiOI`{di%cq?j5)x1Q^seI4p&6+=V%Clz8SX2WgFDlJ1dD%u=Q6*AgD5{xn&20#A z0a)emRKKhT4^*3~pdTTGZK{Hrs-UJSs7Y=#$&Dtt@g$QQO;u1+71UG({dgIrm0et2UdLQYJ;FYn&XjT`(#DUk*lXGi;c>A!78uU?v+=0 z$BKwmuWZeQqs_6`Rc;~3N6>2v+&5NdjoiItx$ERzhMLQBM^d`?C3^~1o}+Pb8Pa!C zVN`QGnm|SidT>7@CQhjb2N%nU(#z#b*H)Q1#o(J>lGdn^m!d|xc^9hcBn7UBS+_bi z7uJKtE6Eg(p+(Z`X`+>Q*2Sf> z*onosvRZFtvbFD`&`eWX0cIEZ2tL$zcqfAXoedgFYS7j9D&zm{e^V$g`F;0)?e41A zSksYwq9VVy?hdTPZ12xDy8|l+Qn7bq2Lr?Uq7%x(;-W&S&zWwG*lXvhF82BBY`mVQ zYmRl=OUn=$Lgn=;Q}))>qG@Z30rl(B7UZlsJ}i0c z?{%Z?iwfDWflojZi%62RoY};it{ffL6>*(_=aDN-wOc@qQ}93pkJ5u}IE6_-;^+~y zzDzD%b+>x%eTnP3dj>uw--5Smq^jfwH-y?`Y{LgGGUzT>&jke;l(TahNK{VETa50( zlrRSNFenTdGAb#Ne1q*g1<&Qo+Jqz2QhJLLfY4(I3htu9iG#uErl%3od|kC>A!jS{ ztKIZvGtaY*ZJD|1+*44tq@zS3&5ATu#U{;8njKmw0qu8b)uQVVD#g4z3-WI?XzB!; zI>D!2C%ED(RNj-63sJpbY*JA)>faGkk$0A@cBv7y>K~UKjS^*JtCqABYo$D;a;zA5 zVO47NmOPIN8(P+Yq?{RzqD}cJFYZ3B-s^SuaYcovN*(+`jsE}aO z`%drEYfg;+s3*pIZ*ne%Iw9g3=mdQt+klSoqlz&^K9#w4aN#eLk zy7AVf3aL2R2>*QIQDUj;k=2Sq94@uOh#1u}9^NuFN{v{eX~_}w!a-`*NRUK3Q_77_ zcDYRV6%MP0m^P+(Ua6}e%r5+#bCbw#Ou3}{#Ju(^)>WoQ=3W*U)llYM8j=RG=_Mg6 z2<1P?d$@~1)?O6&*p6m%OUZHXZO$zvBWvHMwiT3reBfe{^Qj~OIY0`Zmgh=C4!}(X zBOlMFCL_(m@q2wZ{=%evubp#pab!jJOjMQcXjrw2uolBMm)m6i2MJ|f8h>CjmpH>>P(I>}y-90>?U(Wv* P00960pbD7=FOUEL_B30P diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go index 0e4b0fd69..35dc919dd 100644 --- a/cmd/boostd/index.go +++ b/cmd/boostd/index.go @@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{ indexProvAnnounceLatestHttp, indexProvAnnounceDealRemovalAd, indexProvAnnounceDeal, + indexProvRemoveAllCmd, }, } @@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{ return nil }, } + +var indexProvRemoveAllCmd = &cli.Command{ + Name: "remove-all", + Usage: "Announce all removal ad for all contextIDs", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + + // get boost api + napi, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + // announce markets and boost deals + cids, err := napi.BoostIndexerRemoveAll(ctx) + if err != nil { + return err + } + for _, c := range cids { + fmt.Println("Published the removal ad with CID", c.String()) + } + return nil + }, +} diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index 859bfc871..5dfd09678 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -20,6 +20,7 @@ * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp) * [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal) * [BoostIndexerListMultihashes](#boostindexerlistmultihashes) + * [BoostIndexerRemoveAll](#boostindexerremoveall) * [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid) * [BoostOfflineDealWithData](#boostofflinedealwithdata) * [I](#i) @@ -374,7 +375,7 @@ Response: ``` ### BoostIndexerAnnounceAllDeals -There are not yet any comments for this method. + Perms: admin @@ -513,6 +514,20 @@ Response: ] ``` +### BoostIndexerRemoveAll +There are not yet any comments for this method. + +Perms: admin + +Inputs: `null` + +Response: +```json +[ + null +] +``` + ### BoostLegacyDealByProposalCid diff --git a/gql/resolver_ipni.go b/gql/resolver_ipni.go index 1b23892b4..7391b288a 100644 --- a/gql/resolver_ipni.go +++ b/gql/resolver_ipni.go @@ -229,3 +229,7 @@ func (r *resolver) IpniDistanceFromLatestAd(ctx context.Context, args struct { return count, nil } + +func (r *resolver) IpniRemovedAllAdsStatus(ctx context.Context) (bool, error) { + return r.idxProvWrapper.RemoveAllStatus(ctx), nil +} diff --git a/gql/schema.graphql b/gql/schema.graphql index 5e02f01f3..b9301f7ce 100644 --- a/gql/schema.graphql +++ b/gql/schema.graphql @@ -652,6 +652,9 @@ type RootQuery { """Get the latest IPNI advertisemen""" ipniDistanceFromLatestAd(LatestAdcid: String!, Adcid: String!): Int! + + """Get the IPNI Remove All Status""" + ipniRemovedAllAdsStatus: Boolean! } type RootMutation { diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index 0976ad1a2..ea2d230dd 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -7,6 +7,7 @@ import ( "fmt" "net/url" "os" + "time" "github.com/filecoin-project/boost/lib/legacy" "github.com/filecoin-project/boost/storagemarket/types/legacytypes" @@ -18,11 +19,11 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" chainTypes "github.com/filecoin-project/lotus/chain/types" "github.com/google/uuid" - cbor "github.com/ipfs/go-ipld-cbor" - "go.uber.org/fx" - "github.com/ipfs/go-datastore" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/ipld/go-ipld-prime" + "github.com/ipni/go-libipni/ingest/schema" + "go.uber.org/fx" "github.com/filecoin-project/boost/db" bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types" @@ -71,6 +72,7 @@ type Wrapper struct { bitswapEnabled bool httpEnabled bool stop context.CancelFunc + removeAllAds bool } func NewWrapper(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, directDealsDB *db.DirectDealsDB, dealsDB *db.DealsDB, @@ -127,7 +129,11 @@ func (w *Wrapper) Start(_ context.Context) { log.Info("starting index provider") - go w.checkForUpdates(runCtx) + if w.cfg.CurioMigration.Enable { + go w.tryAnnounceRemoveAll(runCtx) + } else { + go w.checkForUpdates(runCtx) + } } func (w *Wrapper) checkForUpdates(ctx context.Context) { @@ -867,3 +873,79 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u } return annCid, err } + +func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) { + var allAds []*schema.Advertisement + _, ad, err := w.prov.GetLatestAdv(ctx) + if err != nil { + return nil, err + } + allAds = append(allAds, ad) + + prev, err := cid.Parse(ad.PreviousID.String()) + if err != nil { + return nil, err + } + + for prev != cid.Undef { + ad, err := w.prov.GetAdv(ctx, prev) + if err != nil { + return nil, err + } + + prev, err = cid.Parse(ad.PreviousID.String()) + if err != nil { + return nil, err + } + } + + var entryAds []*schema.Advertisement + + for _, ad := range allAds { + if !ad.IsRm { + entryAds = append(entryAds, ad) + } + } + + var newAds []cid.Cid + + for _, ad := range entryAds { + a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID) + if err != nil { + if !errors.Is(err, provider.ErrContextIDNotFound) { + return nil, fmt.Errorf("failed to publish the removal ad: %w", err) + } + } + newAds = append(newAds, a) + } + + return newAds, nil + +} + +func (w *Wrapper) tryAnnounceRemoveAll(ctx context.Context) { + ticker := time.NewTicker(time.Minute) + + for { + select { + case <-ticker.C: + out, err := w.AnnounceRemoveAll(ctx) + if err != nil { + log.Errorw("error while announcing remove all", "err", err) + continue + } + if len(out) > 0 { + continue + } + log.Debugw("Cleaned up all the IPNI ads") + w.removeAllAds = true + return + case <-ctx.Done(): + return + } + } +} + +func (w *Wrapper) RemoveAllStatus(ctx context.Context) bool { + return w.removeAllAds +} diff --git a/node/config/def.go b/node/config/def.go index 7348db15c..ec39b0827 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -170,6 +170,9 @@ func DefaultBoost() *Boost { MaxDealsPerPublishMsg: 8, MaxPublishDealsFee: types.MustParseFIL("0.05"), }, + CurioMigration: CurioMigration{ + Enable: false, + }, } return cfg } diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 2f0745bea..50ca25f3e 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -145,6 +145,12 @@ a file containing the booster-bitswap peer id's private key. Can be left blank w Name: "IndexProvider", Type: "IndexProviderConfig", + Comment: ``, + }, + { + Name: "CurioMigration", + Type: "CurioMigration", + Comment: ``, }, }, @@ -194,6 +200,14 @@ a file containing the booster-bitswap peer id's private key. Can be left blank w Comment: `From address for eth_ state call`, }, }, + "CurioMigration": []DocField{ + { + Name: "Enable", + Type: "bool", + + Comment: `Enable limits the Boost functionality to prepare for the migration`, + }, + }, "DealPublishConfig": []DocField{ { Name: "ManualDealPublish", diff --git a/node/config/types.go b/node/config/types.go index 06e17159a..5b4b5acc3 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -51,6 +51,7 @@ type Boost struct { HttpDownload HttpDownloadConfig Retrievals RetrievalConfig IndexProvider IndexProviderConfig + CurioMigration CurioMigration } type WalletsConfig struct { @@ -399,3 +400,8 @@ type DealPublishConfig struct { // The maximum fee to pay when sending the PublishStorageDeals message MaxPublishDealsFee types.FIL } + +type CurioMigration struct { + // Enable limits the Boost functionality to prepare for the migration + Enable bool +} diff --git a/node/impl/boost.go b/node/impl/boost.go index 645fb0e49..c27be73ff 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error { func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) { return sm.StorageProvider.GetAsk(), nil } + +func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) { + return sm.IndexProvider.AnnounceRemoveAll(ctx) +} diff --git a/node/modules/directdeals.go b/node/modules/directdeals.go index 2094a555d..227f85f8a 100644 --- a/node/modules/directdeals.go +++ b/node/modules/directdeals.go @@ -34,6 +34,7 @@ func NewDirectDealsProvider(provAddr address.Address, cfg *config.Boost) func(lc ddpCfg := storagemarket.DDPConfig{ StartEpochSealingBuffer: abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer), RemoteCommp: cfg.Dealmaking.RemoteCommp, + CurioMigration: cfg.CurioMigration.Enable, } prov := storagemarket.NewDirectDealsProvider(ddpCfg, provAddr, fullnodeApi, secb, commpc, commpt, sps, directDealsDB, dl, piecedirectory, ip) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 80a1f606e..84cf5d922 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -486,6 +486,7 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func( DealLogDurationDays: cfg.Dealmaking.DealLogDurationDays, StorageFilter: cfg.Dealmaking.Filter, SealingPipelineCacheTimeout: time.Duration(cfg.Dealmaking.SealingPipelineCacheTimeout), + CurioMigration: cfg.CurioMigration.Enable, } dl := logs.NewDealLogger(logsDB) tspt := httptransport.New(h, dl, httptransport.NChunksOpt(cfg.HttpDownload.NChunks), httptransport.AllowPrivateIPsOpt(cfg.HttpDownload.AllowPrivateIPs)) diff --git a/react/src/Ipni.js b/react/src/Ipni.js index cd7bc782a..793fdcb26 100644 --- a/react/src/Ipni.js +++ b/react/src/Ipni.js @@ -6,6 +6,7 @@ import { IpniProviderInfoQuery, IpniLatestAdQuery, IpniDistanceFromLatestAdQuery, + IpniRemovedAllAdsStatus, } from "./gql"; import moment from "moment"; import React, {useEffect, useState} from "react"; @@ -89,6 +90,7 @@ function ProviderIpniInfoRender(props){ adcid: adCid } }) + const publishedRemoveAll = useQuery(IpniRemovedAllAdsStatus) return

Provider Indexer Info

@@ -111,13 +113,14 @@ function ProviderIpniInfoRender(props){   ({moment(data.LastAdvertisementTime).fromNow()} ago)   - {distance.data ? ({distance.data.ipniDistanceFromLatestAd} behind): ''} + {distance.data ? + ({distance.data.ipniDistanceFromLatestAd} behind) : ''} Latest Advertisement on Boost - {lad ? {lad}: ''} + {lad ? {lad} : ''} @@ -128,6 +131,12 @@ function ProviderIpniInfoRender(props){ ({moment(data.LastErrorTime).fromNow()} ago) + + Published Removal Ads for Curio Migration + + {publishedRemoveAll.data ? publishedRemoveAll.data: ''} + +
@@ -137,7 +146,7 @@ function ProviderConfig({configJson}) { const cfg = JSON.parse(configJson) return

Index Provider Config

- +
} diff --git a/react/src/gql.js b/react/src/gql.js index 2aa0a7a12..519136620 100644 --- a/react/src/gql.js +++ b/react/src/gql.js @@ -880,6 +880,12 @@ const IpniDistanceFromLatestAdQuery = gql` } `; +const IpniRemovedAllAdsStatus = gql` + query AppIpniRemovedAllAdsStatusQuery{ + ipniRemovedAllAdsStatus + } +`; + export { gqlClient, EpochQuery, @@ -931,4 +937,5 @@ export { StorageAskQuery, PublishPendingDealsMutation, PiecePayloadCidsQuery, + IpniRemovedAllAdsStatus, } diff --git a/storagemarket/direct_deals_provider.go b/storagemarket/direct_deals_provider.go index 1447b30ae..ffecd09f3 100644 --- a/storagemarket/direct_deals_provider.go +++ b/storagemarket/direct_deals_provider.go @@ -44,6 +44,7 @@ type DDPConfig struct { // Minimum start epoch buffer to give time for sealing of sector with deal StartEpochSealingBuffer abi.ChainEpoch Curio bool + CurioMigration bool } type DirectDealsProvider struct { @@ -181,6 +182,9 @@ func (ddp *DirectDealsProvider) Accept(ctx context.Context, entry *types.DirectD } func (ddp *DirectDealsProvider) Import(ctx context.Context, params smtypes.DirectDealParams) (*api.ProviderDealRejectionInfo, error) { + if ddp.config.CurioMigration { + return &api.ProviderDealRejectionInfo{Reason: "Boost is migrating to Curio"}, nil + } piececid := params.PieceCid.String() clientAddr := params.ClientAddr.String() log.Infow("received direct data import", "piececid", piececid, "filepath", params.FilePath, "clientAddr", clientAddr, "allocationId", params.AllocationID) diff --git a/storagemarket/provider.go b/storagemarket/provider.go index 5e09fee1c..b6af08047 100644 --- a/storagemarket/provider.go +++ b/storagemarket/provider.go @@ -84,6 +84,7 @@ type Config struct { SealingPipelineCacheTimeout time.Duration StorageFilter string Curio bool + CurioMigration bool } var log = logging.Logger("boost-provider") @@ -250,6 +251,9 @@ func (p *Provider) GetAsk() *legacytypes.SignedStorageAsk { // an offline deal (the deal must already have been proposed by the client) func (p *Provider) ImportOfflineDealData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (pi *api.ProviderDealRejectionInfo, err error) { p.dealLogger.Infow(dealUuid, "import data for offline deal", "filepath", filePath, "delete after import", delAfterImport) + if p.config.CurioMigration { + return &api.ProviderDealRejectionInfo{Reason: "Boost is migrating to Curio"}, nil + } // db should already have a deal with this uuid as the deal proposal should have been made beforehand ds, err := p.dealsDB.ByID(p.ctx, dealUuid) @@ -293,6 +297,9 @@ func (p *Provider) ImportOfflineDealData(ctx context.Context, dealUuid uuid.UUID // ExecuteDeal is called when the Storage Provider receives a deal proposal // from the network func (p *Provider) ExecuteDeal(ctx context.Context, dp *types.DealParams, clientPeer peer.ID) (*api.ProviderDealRejectionInfo, error) { + if p.config.CurioMigration { + return &api.ProviderDealRejectionInfo{Reason: "Boost is migrating to Curio"}, nil + } ctx, span := tracing.Tracer.Start(ctx, "Provider.ExecuteLibp2pDeal") defer span.End()