From ffc2f8655589656faca7d5518aa9c8d49e8a89e0 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 21 Jan 2021 17:01:11 -0800 Subject: [PATCH 1/3] feat(graphsync): define protocol as IPLD schema, encoded in CBOR --- block-layer/graphsync/graphsync.md | 113 +++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 29 deletions(-) diff --git a/block-layer/graphsync/graphsync.md b/block-layer/graphsync/graphsync.md index 0e040987..4cbf5ec3 100644 --- a/block-layer/graphsync/graphsync.md +++ b/block-layer/graphsync/graphsync.md @@ -59,35 +59,90 @@ type GraphSyncNet interface { ## Network Messages -```protobuf -message GraphsyncMessage { - - message Request { - int32 id = 1; // unique id set on the requester side - bytes root = 2; // a CID for the root node in the query - bytes selector = 3; // ipld selector to retrieve - map extensions = 4; // side channel information - int32 priority = 5; // the priority (normalized). default to 1 - bool cancel = 6; // whether this cancels a request - bool update = 7; // whether this is an update to an in progress request - } - - message Response { - int32 id = 1; // the request id - int32 status = 2; // a status code. - map extensions = 3; // side channel information - } - - message Block { - bytes prefix = 1; // CID prefix (cid version, multicodec and multihash prefix (type + length) - bytes data = 2; - } - - // the actual data included in this message - bool completeRequestList = 1; // This request list includes *all* requests, replacing outstanding requests. - repeated Request requests = 2; // The list of requests. - repeated Response responses = 3; // The list of responses. - repeated Block data = 4; // Blocks related to the responses +Graphsync network messages are encoded in DAG-CBOR. They have the following schema + +```ipldsch +type GraphSyncExtensions {string:Any} +type GraphSyncRequestID int +type GraphSyncPriority int + +type GraphSyncMetadatum struct { + Link &Any + BlockPresent bool +} +type GraphSyncMetadata [GraphsyncMetadatum] + +type GraphSyncResponseCode enum { + # RequestAcknowledged means the request was received and is being worked on. + | RequestAcknowledged ("10") + # AdditionalPeers means additional peers were found that may be able + # to satisfy the request and contained in the extra block of the response. + | AdditionalPeers ("11") + # NotEnoughGas means fulfilling this request requires payment. + | NotEnoughGas ("12") + # OtherProtocol means a different type of response than GraphSync is + # contained in extra. + | OtherProtocol ("13") + # PartialResponse may include blocks and metadata about the in progress response + # in extra. + | PartialResponse ("14") + # RequestPaused indicates a request is paused and will not send any more data + # until unpaused + | RequestPaused ("15") + + # Success Response Codes (request terminated) + + # RequestCompletedFull means the entire fulfillment of the GraphSync request + # was sent back. + | RequestCompletedFull ("20") + # RequestCompletedPartial means the response is completed, and part of the + # GraphSync request was sent back, but not the complete request. + | RequestCompletedPartial ("21") + + # Error Response Codes (request terminated) + + # RequestRejected means the node did not accept the incoming request. + | RequestRejected ("30") + # RequestFailedBusy means the node is too busy, try again later. Backoff may + # be contained in extra. + | RequestFailedBusy ("31") + # RequestFailedUnknown means the request failed for an unspecified reason. May + # contain data about why in extra. + | RequestFailedUnknown ("32") + # RequestFailedLegal means the request failed for legal reasons. + | RequestFailedLegal ("33") + # RequestFailedContentNotFound means the respondent does not have the content. + | RequestFailedContentNotFound ("34") + # RequestCancelled means the responder was processing the request but decided to top, for whatever reason + | RequestCancelled ("35") +} representation int + +type GraphSyncRequest struct { + Id GraphSyncRequestID # unique id set on the requester side + Root &Any # a CID for the root node in the query + Selector Selector # see https://github.com/ipld/specs/blob/master/selectors/selectors.md + Extensions GraphSyncExtensions # side channel information + Priority GraphsyncPriority # the priority (normalized). default to 1 + Cancel bool # whether this cancels a request + Update bool # whether this is an update to an in progress request +} + +type GraphSyncResponse struct { + ID GraphSyncRequestID # the request id we are responding to + Status GraphSyncResponseStatusCode # a status code. + Metadata GraphSyncMetadata # metadata about response + Extensions GraphSyncExtensions # side channel information +} + +type GraphSyncBlock struct { + Prefix bytes # CID prefix (cid version, multicodec and multihash prefix (type + length) + Data bytes +} + +type GraphSyncMessage struct { + Requests [GraphSyncRequest] + Responses [GraphSyncResponse] + Blocks [GraphSyncBlock] } ``` From 7313610c812a0e19d1f3e71bd93064bf0e6a5baa Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 21 Jan 2021 17:05:52 -0800 Subject: [PATCH 2/3] feat(graphsync): include legacy protocol --- block-layer/graphsync/graphsync.md | 45 +++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/block-layer/graphsync/graphsync.md b/block-layer/graphsync/graphsync.md index 4cbf5ec3..77a8fb4a 100644 --- a/block-layer/graphsync/graphsync.md +++ b/block-layer/graphsync/graphsync.md @@ -59,6 +59,8 @@ type GraphSyncNet interface { ## Network Messages +### Protocol Version 1.1.0 + Graphsync network messages are encoded in DAG-CBOR. They have the following schema ```ipldsch @@ -128,10 +130,10 @@ type GraphSyncRequest struct { } type GraphSyncResponse struct { - ID GraphSyncRequestID # the request id we are responding to - Status GraphSyncResponseStatusCode # a status code. - Metadata GraphSyncMetadata # metadata about response - Extensions GraphSyncExtensions # side channel information + ID GraphSyncRequestID # the request id we are responding to + Status GraphSyncResponseStatusCode # a status code. + Metadata GraphSyncMetadata # metadata about response + Extensions GraphSyncExtensions # side channel information } type GraphSyncBlock struct { @@ -146,6 +148,41 @@ type GraphSyncMessage struct { } ``` +### Legacy Protocol Version 1.0.0 + +An earlier version of graphsync encoded messages using protobufs + +```protobuf +message GraphsyncMessage { + + message Request { + int32 id = 1; // unique id set on the requester side + bytes root = 2; // a CID for the root node in the query + bytes selector = 3; // ipld selector to retrieve + map extensions = 4; // side channel information + int32 priority = 5; // the priority (normalized). default to 1 + bool cancel = 6; // whether this cancels a request + bool update = 7; // whether this is an update to an in progress request + } + + message Response { + int32 id = 1; // the request id + int32 status = 2; // a status code. + map extensions = 3; // side channel information + } + + message Block { + bytes prefix = 1; // CID prefix (cid version, multicodec and multihash prefix (type + length) + bytes data = 2; + } + + // the actual data included in this message + bool completeRequestList = 1; // This request list includes *all* requests, replacing outstanding requests. + repeated Request requests = 2; // The list of requests. + repeated Response responses = 3; // The list of responses. + repeated Block data = 4; // Blocks related to the responses +} +``` ### Extensions From 23eea5a49aba92bd71e6bca92c8646734472d602 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 22 Jan 2021 14:45:53 -0800 Subject: [PATCH 3/3] feat(graphsync): cleanup schema --- block-layer/graphsync/graphsync.md | 141 ++++++++++------------ block-layer/graphsync/known_extensions.md | 2 + 2 files changed, 64 insertions(+), 79 deletions(-) diff --git a/block-layer/graphsync/graphsync.md b/block-layer/graphsync/graphsync.md index 77a8fb4a..160edebf 100644 --- a/block-layer/graphsync/graphsync.md +++ b/block-layer/graphsync/graphsync.md @@ -64,88 +64,69 @@ type GraphSyncNet interface { Graphsync network messages are encoded in DAG-CBOR. They have the following schema ```ipldsch -type GraphSyncExtensions {string:Any} +type GraphSyncExtensions {String:Any} type GraphSyncRequestID int type GraphSyncPriority int type GraphSyncMetadatum struct { - Link &Any - BlockPresent bool -} + link &Any + blockPresent Bool +} representation tuple + type GraphSyncMetadata [GraphsyncMetadatum] type GraphSyncResponseCode enum { - # RequestAcknowledged means the request was received and is being worked on. - | RequestAcknowledged ("10") - # AdditionalPeers means additional peers were found that may be able - # to satisfy the request and contained in the extra block of the response. - | AdditionalPeers ("11") - # NotEnoughGas means fulfilling this request requires payment. - | NotEnoughGas ("12") - # OtherProtocol means a different type of response than GraphSync is - # contained in extra. - | OtherProtocol ("13") - # PartialResponse may include blocks and metadata about the in progress response - # in extra. - | PartialResponse ("14") - # RequestPaused indicates a request is paused and will not send any more data - # until unpaused - | RequestPaused ("15") - - # Success Response Codes (request terminated) - - # RequestCompletedFull means the entire fulfillment of the GraphSync request - # was sent back. - | RequestCompletedFull ("20") - # RequestCompletedPartial means the response is completed, and part of the - # GraphSync request was sent back, but not the complete request. - | RequestCompletedPartial ("21") - - # Error Response Codes (request terminated) - - # RequestRejected means the node did not accept the incoming request. - | RequestRejected ("30") - # RequestFailedBusy means the node is too busy, try again later. Backoff may - # be contained in extra. - | RequestFailedBusy ("31") - # RequestFailedUnknown means the request failed for an unspecified reason. May - # contain data about why in extra. - | RequestFailedUnknown ("32") - # RequestFailedLegal means the request failed for legal reasons. - | RequestFailedLegal ("33") - # RequestFailedContentNotFound means the respondent does not have the content. - | RequestFailedContentNotFound ("34") - # RequestCancelled means the responder was processing the request but decided to top, for whatever reason - | RequestCancelled ("35") + # Informational Codes (request in progress) + + | RequestAcknowledged ("10") + | AdditionalPeers ("11") + | NotEnoughGas ("12") + | OtherProtocol ("13") + | PartialResponse ("14") + | RequestPaused ("15") + + # Success Response Codes (request terminated) + + | RequestCompletedFull ("20") + | RequestCompletedPartial ("21") + + # Error Response Codes (request terminated) + + | RequestRejected ("30") + | RequestFailedBusy ("31") + | RequestFailedUnknown ("32") + | RequestFailedLegal ("33") + | RequestFailedContentNotFound ("34") + | RequestCancelled ("35") } representation int type GraphSyncRequest struct { - Id GraphSyncRequestID # unique id set on the requester side - Root &Any # a CID for the root node in the query - Selector Selector # see https://github.com/ipld/specs/blob/master/selectors/selectors.md - Extensions GraphSyncExtensions # side channel information - Priority GraphsyncPriority # the priority (normalized). default to 1 - Cancel bool # whether this cancels a request - Update bool # whether this is an update to an in progress request -} + id GraphSyncRequestID (rename "ID") # unique id set on the requester side + root &Any (rename "Root") # a CID for the root node in the query + selector Selector (rename "Sel") # see https://github.com/ipld/specs/blob/master/selectors/selectors.md + extensions GraphSyncExtensions (rename "Ext") # side channel information + priority GraphsyncPriority (rename "Pri") # the priority (normalized). default to 1 + cancel Bool (rename "Canc") # whether this cancels a request + update Bool (rename "Updt") # whether this is an update to an in progress request +} representation map type GraphSyncResponse struct { - ID GraphSyncRequestID # the request id we are responding to - Status GraphSyncResponseStatusCode # a status code. - Metadata GraphSyncMetadata # metadata about response - Extensions GraphSyncExtensions # side channel information -} + id GraphSyncRequestID (rename "ID") # the request id we are responding to + status GraphSyncResponseStatusCode (rename "Stat") # a status code. + metadata GraphSyncMetadata (rename "Meta") # metadata about response + extensions GraphSyncExtensions (rename "Ext") # side channel information +} representation map type GraphSyncBlock struct { - Prefix bytes # CID prefix (cid version, multicodec and multihash prefix (type + length) - Data bytes -} + prefix Bytes (rename "Pre") # CID prefix (cid version, multicodec and multihash prefix (type + length) + data Bytes (rename "Data") +} representation map type GraphSyncMessage struct { - Requests [GraphSyncRequest] - Responses [GraphSyncResponse] - Blocks [GraphSyncBlock] -} + requests [GraphSyncRequest] (rename "Reqs") + responses [GraphSyncResponse] (rename "Rsps") + blocks [GraphSyncBlock] (rename "Blks") +} representation map ``` ### Legacy Protocol Version 1.0.0 @@ -204,23 +185,25 @@ The update mechanism in conjunction with the paused response code can also be us ``` # info - partial -10 Request Acknowledged. Working on it. -11 Additional Peers. PeerIDs in extra. -12 Not enough vespene gas ($) -13 Other Protocol - info in extra. -14 Partial Response w/ metadata, may include blocks -15 Request Paused, pending update, see extensions for info +10 Request acknowledged, indicates request was received and is being worked on +11 Additional peers, indicates additional peers were found that may be able to satisfy the request and contained in the extensions block of the response +12 Not enough gas, fulfilling this request requires payment +13 Other protocol, indicates a different type of response than GraphSync is contained in extensions +14 Partial response, includes blocks and metadata about the in progress response +15 Request paused, indicates a request is paused and will not send any more data until unpaused # success - terminal -20 Request Completed, full content. -21 Request Completed, partial content. + +20 Request completed, entire fulfillment of the GraphSync request was sent back. +21 Request completed, partial fulfillment of the GraphSync request was sent back, but not the complete request. # error - terminal -30 Request Rejected. NOT working on it. -31 Request failed, busy, try again later (getting dosed. backoff in extra). -32 Request failed, for unknown reason. Extra may have more info. -33 Request failed, for legal reasons. -34 Request failed, content not found. +30 Request rejected, indicates the node did not accept the incoming request. +31 Request failed, indiciates the node is too busy, try again later. Backoff may be contained in extensions. +32 Request failed for unknown reasons, may contain data about why in extensions. +33 Request failed for legal reasons. +34 Request failed because respondent does not have the content. +35 Request cancelled, indicates the responder was processing the request but decided to stop. Extensions may contain information about reason for cancellation ``` ## Example Use Cases diff --git a/block-layer/graphsync/known_extensions.md b/block-layer/graphsync/known_extensions.md index adf62dcb..5e3f2074 100644 --- a/block-layer/graphsync/known_extensions.md +++ b/block-layer/graphsync/known_extensions.md @@ -25,6 +25,8 @@ The responder node will execute the selector query as it would normally. However ### Response Metadata +***Legacy Graphsync Protocol 1.0.0 Only*** + Extension Name: `graphsync/response-metadata` What it does: