initial draft of market 2.0

Author: LexLuthr

cmd/curio/tasks/tasks.go

@@ -244,16 +244,19 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan
 	}
 
 	{
+		var sdeps cuhttp.ServiceDeps
 		// Market tasks
 		var dm *storage_market.CurioStorageDealMarket
 		if cfg.Subsystems.EnableDealMarket {
 			// Main market poller should run on all nodes
-			dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, si, full, as)
+			dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, must.One(dependencies.EthClient.Val()), si, full, as)
 			err := dm.StartMarket(ctx)
 			if err != nil {
 				return nil, err
 			}
 
+			sdeps.DealMarket = dm
+
 			if cfg.Subsystems.EnableCommP {
 				commpTask := storage_market.NewCommpTask(dm, db, must.One(slrLazy.Val()), full, cfg.Subsystems.CommPMaxTasks)
 				activeTasks = append(activeTasks, commpTask)
@@ -275,7 +278,6 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan
 		if err != nil {
 			return nil, err
 		}
-		var sdeps cuhttp.ServiceDeps
 
 		if cfg.Subsystems.EnablePDP {
 			es := getSenderEth()
@@ -298,7 +300,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan
 		activeTasks = append(activeTasks, ipniTask, indexingTask)
 
 		if cfg.HTTP.Enable {
-			err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps, dm)
+			err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps)
 			if err != nil {
 				return nil, xerrors.Errorf("failed to start the HTTP server: %w", err)
 			}

cuhttp/server.go

@@ -120,10 +120,11 @@ func isWebSocketUpgrade(r *http.Request) bool {
 }
 
 type ServiceDeps struct {
-	EthSender *message.SenderETH
+	EthSender  *message.SenderETH
+	DealMarket *storage_market.CurioStorageDealMarket
 }
 
-func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps, dm *storage_market.CurioStorageDealMarket) error {
+func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error {
 	cfg := d.Cfg.HTTP
 
 	// Setup the Chi router for more complex routing (if needed in the future)
@@ -165,7 +166,9 @@ func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps, dm *sto
 		fmt.Fprintf(w, "Service is up and running")
 	})
 
-	chiRouter, err = attachRouters(ctx, chiRouter, d, sd, dm)
+	// TODO: Attach a info page here with details about all the service and endpoints
+
+	chiRouter, err = attachRouters(ctx, chiRouter, d, sd)
 	if err != nil {
 		return xerrors.Errorf("failed to attach routers: %w", err)
 	}
@@ -259,7 +262,7 @@ func (c cache) Delete(ctx context.Context, key string) error {
 
 var _ autocert.Cache = cache{}
 
-func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDeps, dm *storage_market.CurioStorageDealMarket) (*chi.Mux, error) {
+func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDeps) (*chi.Mux, error) {
 	// Attach retrievals
 	rp := retrieval.NewRetrievalProvider(ctx, d.DB, d.IndexStore, d.CachedPieceReader)
 	retrieval.Router(r, rp)
@@ -283,7 +286,7 @@ func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDep
 	}
 
 	// Attach the market handler
-	dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, dm)
+	dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, sd.DealMarket)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create new market handler: %w", err)
 	}

deps/config/doc_gen.go

@@ -110,6 +110,10 @@ func DefaultCurioConfig() *CurioConfig {
 					ExpectedSnapSealDuration:  2 * time.Hour,
 					CIDGravityTokens:          []string{},
 				},
+				MK20: MK20Config{
+					ExpectedPoRepSealDuration: 8 * time.Hour,
+					ExpectedSnapSealDuration:  2 * time.Hour,
+				},
 				IPNI: IPNIConfig{
 					ServiceURL:         []string{"https://cid.contact"},
 					DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"},
@@ -692,6 +696,9 @@ type StorageMarketConfig struct {
 	// MK12 encompasses all configuration related to deal protocol mk1.2.0 and mk1.2.1 (i.e. Boost deals)
 	MK12 MK12Config
 
+	// MK20 encompasses all configuration related to deal protocol mk2.0 i.e. market 2.0
+	MK20 MK20Config
+
 	// IPNI configuration for ipni-provider
 	IPNI IPNIConfig
 
@@ -851,3 +858,32 @@ type MK12CollateralConfig struct {
 	// Accepts a decimal string (e.g., "123.45" or "123 fil") with optional "fil" or "attofil" suffix. (Default: "20 FIL")
 	CollateralHighThreshold types.FIL
 }
+
+type MK20Config struct {
+	// ExpectedPoRepSealDuration is the expected time it would take to seal the deal sector
+	// This will be used to fail the deals which cannot be sealed on time.
+	// Time duration string (e.g., "1h2m3s") in TOML format. (Default: "8h0m0s")
+	ExpectedPoRepSealDuration time.Duration
+
+	// ExpectedSnapSealDuration is the expected time it would take to snap the deal sector
+	// This will be used to fail the deals which cannot be sealed on time.
+	// Time duration string (e.g., "1h2m3s") in TOML format. (Default: "2h0m0s")
+	ExpectedSnapSealDuration time.Duration
+
+	// SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain
+	// Warning: If this check is skipped and there is a commP mismatch, all deals in the
+	// sector will need to be sent again (Default: false)
+	SkipCommP bool
+
+	// DisabledMiners is a list of miner addresses that should be excluded from online deal making protocols
+	DisabledMiners []string
+
+	// MaxConcurrentDealSizeGiB is a sum of all size of all deals which are waiting to be added to a sector
+	// When the cumulative size of all deals in process reaches this number, new deals will be rejected.
+	// (Default: 0 = unlimited)
+	MaxConcurrentDealSizeGiB int64
+
+	// DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list
+	// If True then all deals coming from unknown clients will be rejected. (Default: false)
+	DenyUnknownClients bool
+}

deps/config/types.go

@@ -637,6 +637,45 @@ description: The default curio configuration
       # type: bool
       #DefaultCIDGravityAccept = false
 
+    # MK20 encompasses all configuration related to deal protocol mk2.0 i.e. market 2.0
+    #
+    # type: MK20Config
+    [Market.StorageMarketConfig.MK20]
+
+      # ExpectedPoRepSealDuration is the expected time it would take to seal the deal sector
+      # This will be used to fail the deals which cannot be sealed on time.
+      # Time duration string (e.g., "1h2m3s") in TOML format. (Default: "8h0m0s")
+      #
+      # type: time.Duration
+      #ExpectedPoRepSealDuration = "8h0m0s"
+
+      # ExpectedSnapSealDuration is the expected time it would take to snap the deal sector
+      # This will be used to fail the deals which cannot be sealed on time.
+      # Time duration string (e.g., "1h2m3s") in TOML format. (Default: "2h0m0s")
+      #
+      # type: time.Duration
+      #ExpectedSnapSealDuration = "2h0m0s"
+
+      # SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain
+      # Warning: If this check is skipped and there is a commP mismatch, all deals in the
+      # sector will need to be sent again (Default: false)
+      #
+      # type: bool
+      #SkipCommP = false
+
+      # MaxConcurrentDealSizeGiB is a sum of all size of all deals which are waiting to be added to a sector
+      # When the cumulative size of all deals in process reaches this number, new deals will be rejected.
+      # (Default: 0 = unlimited)
+      #
+      # type: int64
+      #MaxConcurrentDealSizeGiB = 0
+
+      # DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list
+      # If True then all deals coming from unknown clients will be rejected. (Default: false)
+      #
+      # type: bool
+      #DenyUnknownClients = false
+
     # IPNI configuration for ipni-provider
     #
     # type: IPNIConfig

documentation/en/configuration/default-curio-configuration.md

@@ -25,6 +25,7 @@ require (
 	github.com/filecoin-project/go-commp-utils v0.1.4
 	github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8
 	github.com/filecoin-project/go-commp-utils/v2 v2.1.0
+	github.com/filecoin-project/go-data-segment v0.0.1
 	github.com/filecoin-project/go-f3 v0.8.3
 	github.com/filecoin-project/go-fil-commcid v0.2.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.2.0
@@ -84,6 +85,7 @@ require (
 	github.com/multiformats/go-multicodec v0.9.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/multiformats/go-varint v0.0.7
+	github.com/oklog/ulid v1.3.1
 	github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.20.5

go.mod

@@ -329,6 +329,8 @@ github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0=
 github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI=
+github.com/filecoin-project/go-data-segment v0.0.1 h1:1wmDxOG4ubWQm3ZC1XI5nCon5qgSq7Ra3Rb6Dbu10Gs=
+github.com/filecoin-project/go-data-segment v0.0.1/go.mod h1:H0/NKbsRxmRFBcLibmABv+yFNHdmtl5AyplYLnb0Zv4=
 github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 h1:v+zJS5B6pA3ptWZS4t8tbt1Hz9qENnN4nVr1w99aSWc=
 github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7/go.mod h1:V3Y4KbttaCwyg1gwkP7iai8CbQx4mZUGjd3h9GZWLKE=
 github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w=
@@ -1119,6 +1121,8 @@ github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOW
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
 github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
+github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

go.sum

@@ -0,0 +1,93 @@
+CREATE TABLE ddo_contracts (
+    address TEXT NOT NULL PRIMARY KEY,
+    abi TEXT NOT NULL
+);
+
+CREATE TABLE market_mk20_deal (
+    created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()),
+    id TEXT PRIMARY KEY,
+    piece_cid TEXT NOT NULL,
+    size BIGINT NOT NULL,
+
+    format JSONB NOT NULL,
+    source_http JSONB NOT NULL DEFAULT 'null',
+    source_aggregate JSONB NOT NULL DEFAULT 'null',
+    source_offline JSONB NOT NULL DEFAULT 'null',
+
+    ddov1 JSONB NOT NULL DEFAULT 'null',
+    market_deal_id TEXT DEFAULT NULL
+);
+
+CREATE TABLE market_mk20_pipeline (
+    created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()),
+    id TEXT NOT NULL,
+    sp_id BIGINT NOT NULL,
+    contract TEXT NOT NULL,
+    client TEXT NOT NULL,
+    piece_cid TEXT NOT NULL,
+    piece_size BIGINT NOT NULL,
+    raw_size BIGINT NOT NULL,
+    offline BOOLEAN NOT NULL,
+    url TEXT DEFAULT NULL,
+    indexing BOOLEAN NOT NULL,
+    announce BOOLEAN NOT NULL,
+    allocation_id BIGINT DEFAULT NULL,
+    duration BIGINT NOT NULL,
+    piece_aggregation INT DEFAULT 0,
+
+    started BOOLEAN DEFAULT FALSE,
+
+    downloaded BOOLEAN DEFAULT FALSE,
+
+    commp_task_id BIGINT DEFAULT NULL,
+    after_commp BOOLEAN DEFAULT FALSE,
+
+    deal_aggregation INT DEFAULT 0,
+    aggr_index BIGINT DEFAULT 0,
+    agg_task_id BIGINT DEFAULT NULL,
+    aggregated BOOLEAN DEFAULT FALSE,
+
+    sector BIGINT DEFAULT NULL,
+    reg_seal_proof INT DEFAULT NULL,
+    sector_offset BIGINT DEFAULT NULL, -- padded offset
+
+    sealed BOOLEAN DEFAULT FALSE,
+
+    indexing_created_at TIMESTAMPTZ DEFAULT NULL,
+    indexing_task_id BIGINT DEFAULT NULL,
+    indexed BOOLEAN DEFAULT FALSE,
+
+    complete BOOLEAN NOT NULL DEFAULT FALSE,
+
+    PRIMARY KEY (id, aggr_index)
+);
+
+CREATE TABLE market_mk20_pipeline_waiting (
+    id TEXT NOT NULL PRIMARY KEY
+);
+
+CREATE TABLE market_mk20_download_pipeline (
+    id TEXT NOT NULL,
+    piece_cid TEXT NOT NULL,
+    piece_size BIGINT NOT NULL,
+    ref_ids BIGINT[] NOT NULL,
+    PRIMARY KEY (id, piece_cid, piece_size)
+);
+
+CREATE TABLE market_mk20_offline_urls (
+    id TEXT NOT NULL,
+    piece_cid TEXT NOT NULL,
+    piece_size BIGINT NOT NULL,
+    url TEXT NOT NULL,
+    headers jsonb NOT NULL DEFAULT '{}',
+    raw_size BIGINT NOT NULL,
+    CONSTRAINT market_mk20_offline_urls_id_fk FOREIGN KEY (id)
+      REFERENCES market_mk20_deal_pipeline (id)
+      ON DELETE CASCADE,
+    CONSTRAINT market_mk20_offline_urls_id_unique UNIQUE (id)
+);
+
+
+
+
+