diff --git a/SPEC.md b/SPEC.md index 795b0ff..da45461 100644 --- a/SPEC.md +++ b/SPEC.md @@ -138,7 +138,7 @@ Single-value codes (sent as first/only byte): | 8 | future time | Timestamp too far in future. | | 9 | time travel | Timestamp before parent's timestamp. | | 10 | duplicate | Already received for all recipients. | -| 11 | accept add to | Add-to accepted; parent already stored; no _add to_ recipients on this host. Stop. | +| 11 | accept add to | Add-to accepted; parent already stored; no _add to_ recipients on this host (including notification-only participant domains). Stop. | | 64 | continue | Header accepted; send data. | | 65 | skip data | Add-to accepted; parent already stored; _add to_ recipients on this host. Skip data, per-recipient codes follow. | @@ -176,7 +176,11 @@ One message per connection. Two TCP connections used: Connection 1 (message tran ### 10.2 Sending (Host A perspective) -Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. For each unique recipient domain: +Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. + +When _has add to_ is NOT set: perform the steps below for each unique recipient domain. + +When _has add to_ IS set: perform the steps below for each unique participant domain — the domains of _from_ and of every address in _to_ and _add to_. _from_'s domain is omitted when _from_ is the _add to from_ (the adder is the original sender, whose host is Host A). Domains having no address in this message's _to_ or _add to_ are **notification-only**: the exchange completes at the single response code in step 5 (code 11 on success, or code 6 when the domain's host does not hold the parent) and never reaches step 6. 1. Resolve recipient domain IPs via ``fmsg.``. Connect to first responsive IP (Connection 1). Retry with backoff if unreachable. 2. Register the message header hash and Host B's IP in an outgoing record (for matching challenges). @@ -201,10 +205,11 @@ Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. For each 3. Validate (all must pass, else respond code 1 invalid and close): - _to_ has ≥ 1 distinct address. - If _has add to_: _add to from_ exists and is in _from_ or _to_; _add to_ has ≥ 1 distinct address. - - ≥ 1 recipient in _to_ or _add to_ belongs to Host B's domain. + - If _has add to_ not set: ≥ 1 recipient in _to_ belongs to Host B's domain. If _has add to_ set: ≥ 1 participant (_from_, _to_, _add to from_ or _add to_) belongs to Host B's domain. - Common type IDs (message and attachment) are mapped. + - _expanded size_ fields are present iff the corresponding zlib-deflate flag is set. 4. DNS-verify sender IP: resolve `fmsg.`, check Connection 1 source IP is in result set. Fail → TERMINATE. -5. If _size_ + attachment sizes > MAX_SIZE → respond code 4, close. +5. If _size_ + attachment sizes > MAX_SIZE, or total expanded size > MAX_EXPANDED_SIZE → respond code 4, close. Total expanded size uses _expanded size_ for compressed parts and _size_ for uncompressed parts. 6. Compute DELTA = now − _time_: - DELTA > MAX_MESSAGE_AGE → respond code 7, close. - DELTA < −MAX_TIME_SKEW → respond code 8, close. @@ -218,17 +223,18 @@ Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. For each - pid MUST also be set. Fail → respond code 1, close. - Check if parent stored (§11): - **Stored**: check time travel (code 9 if fail). - - **Not stored**: treat as full message delivery. + - **Not stored**: if ≥ 1 recipient in _to_ or _add to_ belongs to Host B's domain, treat as full message delivery. Otherwise (Host B hosts only non-recipient participants) respond code 6 (parent not found), close. 8. Optionally issue a CHALLENGE on Connection 2 (see §10.5). ### 10.4 Receiving — ACCEPT Response, Data Download and Per-Recipient Response 1. If _add to_ set and parent verified stored in step 7: + - If Host B has already recorded this exact add-to batch (§11) → respond code 10 (duplicate), close. - If any _add to_ recipient belongs to Host B's domain → respond 65 (skip data). - - Otherwise → record add-to fields, respond 11 (accept add to), close. + - Otherwise → record the add-to batch (_add to from_, _add to_, _time_) per §11, respond 11 (accept add to), close. This is the path notification-only participant domains take. 2. If challenge was completed, use the message hash from the challenge response to check for duplicates across all recipients on Host B. If duplicate for all → respond code 10, close. 3. Otherwise → respond 64 (continue). -4. If code 65 was sent, skip to step 6 (data already stored). Otherwise download data + attachments (exactly declared sizes). +4. If code 65 was sent, skip to step 6 (data already stored). Otherwise download data + attachments (exactly declared on-wire sizes). For each zlib-deflate part, decompress and verify output byte length exactly equals _expanded size_; failure or mismatch means invalid → TERMINATE. 5. If challenge was completed, verify computed message hash matches the challenge response hash. For code 65, compute from received header + stored data. Mismatch → TERMINATE. 6. For each recipient on Host B's domain (in _to_ order, then _add to_ order), send one response byte: - Already received → 103 (or 105). @@ -278,10 +284,14 @@ An add-to message is a duplicate of the original message with these differences: - _time_ = new timestamp. - _topic_ is NOT present (pid is set). +An add-to message MUST be sent to every participant domain per §10.2, so all participants of the message being added to — including the original sender, when not themselves the _add to from_ — learn of the added recipients, not only the domains hosting the new recipients. This is required because a subsequent reply may reference this add-to message via _pid_, and a host can only accept a reply whose parent it holds. + +Add-to batches do not chain: recipients are always added to the original message; an add-to message's _pid_ MUST NOT reference another add-to message. A message therefore has 0 or more add-to batches, each independently referencing it. + ## 13. Security Requirements -- Enforce MAX_SIZE before downloading data. -- Enforce MAX_EXPANDED_SIZE: reject when declared expanded size exceeds limit. +- Enforce MAX_SIZE and MAX_EXPANDED_SIZE before downloading data. +- For zlib-deflate parts, bound decompression and require output length to exactly match _expanded size_; mismatch means invalid → TERMINATE. - Enforce per-connection and per-IP rate limits. - Apply idle/slow-connection timeouts. - Verify sender IP via DNS BEFORE issuing any challenge. diff --git a/cmd/fmsgd/host.go b/cmd/fmsgd/host.go index 0235dbe..3cb5808 100644 --- a/cmd/fmsgd/host.go +++ b/cmd/fmsgd/host.go @@ -522,12 +522,28 @@ func verifySenderIP(c net.Conn, senderDomain string) error { return fmt.Errorf("DNS verification failed") } +// handleAddToParentNotStored resolves an add-to whose parent this host does +// not hold (SPEC §10.3 step 7): with a local recipient the add-to is treated +// as a full message delivery; without one nothing can be stored for anyone, +// so respond code 6 (parent not found) and close. +func handleAddToParentNotStored(c net.Conn, h *FMsgHeader, hasLocalRecipient bool) (*FMsgHeader, error) { + if hasLocalRecipient { + h.InitialResponseCode = AcceptCodeContinue + return h, nil + } + if err := sendCode(c, RejectCodeParentNotFound); err != nil { + return h, err + } + return h, fmt.Errorf("add-to: parent not stored and no recipients for domain %s", Domain) +} + func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { if len(h.AddTo) == 0 { return h, nil } addToHasOurDomain := hasDomainRecipient(h.AddTo, Domain) + hasLocalRecipient := addToHasOurDomain || hasDomainRecipient(h.To, Domain) parentID, err := lookupMsgIdByHash(h.Pid) if err != nil { @@ -535,8 +551,7 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { } if parentID == 0 { - h.InitialResponseCode = AcceptCodeContinue - return h, nil + return handleAddToParentNotStored(c, h, hasLocalRecipient) } parentMsg, err := getMsgByID(parentID) @@ -544,8 +559,7 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { return h, err } if parentMsg == nil || !isMessageRetrievable(parentMsg) { - h.InitialResponseCode = AcceptCodeContinue - return h, nil + return handleAddToParentNotStored(c, h, hasLocalRecipient) } if parentMsg.Timestamp-FutureTimeDelta > h.Timestamp { @@ -555,6 +569,18 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { return h, fmt.Errorf("add-to: time travel detected (parent time %f, current %f)", parentMsg.Timestamp, h.Timestamp) } + // A batch this host already recorded is a duplicate (SPEC §10.4 step 1). + recorded, err := addToBatchRecorded(parentID, h.AddTo) + if err != nil { + return h, err + } + if recorded { + if err := sendCode(c, RejectCodeDuplicate); err != nil { + return h, err + } + return h, fmt.Errorf("add-to: batch already recorded for msg %d", parentID) + } + if addToHasOurDomain { h.InitialResponseCode = AcceptCodeSkipData return h, nil @@ -1022,10 +1048,17 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { log.Printf("INFO: <-- MSG\n%s", h) if !hasDomainRecipient(h.To, Domain) && !hasDomainRecipient(h.AddTo, Domain) { - if err := sendCode(c, RejectCodeInvalid); err != nil { - return h, r, err + // An add-to message is delivered to every participant domain, not only + // recipient domains (SPEC §10.2/§10.3): a domain hosting no recipient + // is being notified that recipients were added. add to from is always + // in from or to, so from is the only participant left to check. + participantOnly := len(h.AddTo) > 0 && strings.EqualFold(h.From.Domain, Domain) + if !participantOnly { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return h, r, err + } + return h, r, fmt.Errorf("no participants for domain %s", Domain) } - return h, r, fmt.Errorf("no recipients for domain %s", Domain) } if err := verifySenderIP(c, determineSenderDomain(h)); err != nil { diff --git a/cmd/fmsgd/host_test.go b/cmd/fmsgd/host_test.go index 48c54be..018e394 100644 --- a/cmd/fmsgd/host_test.go +++ b/cmd/fmsgd/host_test.go @@ -620,3 +620,30 @@ func TestReadAttachmentHeadersRejectsExpandedSizeExceedsMax(t *testing.T) { t.Fatalf("expected reject code %d, got %v", RejectCodeTooBig, got) } } + +func TestHandleAddToParentNotStoredWithLocalRecipient(t *testing.T) { + c := &testConn{} + h := &FMsgHeader{} + got, err := handleAddToParentNotStored(c, h, true) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.InitialResponseCode != AcceptCodeContinue { + t.Fatalf("InitialResponseCode = %d, want %d (continue)", got.InitialResponseCode, AcceptCodeContinue) + } + if c.Buffer.Len() != 0 { + t.Fatalf("unexpected bytes written: %v", c.Buffer.Bytes()) + } +} + +func TestHandleAddToParentNotStoredParticipantOnly(t *testing.T) { + c := &testConn{} + h := &FMsgHeader{} + _, err := handleAddToParentNotStored(c, h, false) + if err == nil { + t.Fatal("expected error for participant-only add-to without stored parent") + } + if got := c.Buffer.Bytes(); len(got) != 1 || got[0] != RejectCodeParentNotFound { + t.Fatalf("wrote %v, want single code %d (parent not found)", got, RejectCodeParentNotFound) + } +} diff --git a/cmd/fmsgd/sender.go b/cmd/fmsgd/sender.go index f68c8af..5c46598 100644 --- a/cmd/fmsgd/sender.go +++ b/cmd/fmsgd/sender.go @@ -57,7 +57,9 @@ func findPendingTargets() ([]pendingTarget, error) { now := timeutil.TimestampNow().Float64() - // query both msg_to and msg_add_to for pending targets + // query both msg_to and msg_add_to for pending targets; add-to rows age + // against their batch, not the message — recipients may be added long + // after the message was sent rows, err := db.Query(` SELECT mt.msg_id, mt.addr FROM msg_to mt @@ -71,11 +73,12 @@ func findPendingTargets() ([]pendingTarget, error) { SELECT mat.msg_id, mat.addr FROM msg_add_to mat INNER JOIN msg m ON m.id = mat.msg_id + INNER JOIN msg_add_to_batch b ON b.id = mat.batch_id WHERE mat.time_delivered IS NULL AND m.time_sent IS NOT NULL AND (mat.response_code IS NULL OR mat.response_code = ANY($4)) AND (mat.time_last_attempt IS NULL OR ($1 - mat.time_last_attempt) > LEAST($2 * POWER(2.0, GREATEST(mat.attempt_count - 1, 0)::float), $3)) - AND ($1 - m.time_sent) < $3 + AND ($1 - b.time_added) < $3 `, now, RetryInterval, RetryMaxAge, pq.Array(retryableResponseCodes)) if err != nil { return nil, err @@ -88,6 +91,16 @@ func findPendingTargets() ([]pendingTarget, error) { } seen := make(map[key]bool) var targets []pendingTarget + addTarget := func(msgID int64, domain string) { + if strings.EqualFold(domain, Domain) { + return // local domain — no remote delivery needed + } + k := key{msgID, strings.ToLower(domain)} + if !seen[k] { + seen[k] = true + targets = append(targets, pendingTarget{MsgID: msgID, Domain: domain}) + } + } for rows.Next() { var msgID int64 @@ -99,17 +112,39 @@ func findPendingTargets() ([]pendingTarget, error) { if lastAt == -1 { continue } - domain := addr[lastAt+1:] - if strings.EqualFold(domain, Domain) { - continue // local domain — no remote delivery needed - } - k := key{msgID, domain} - if !seen[k] { - seen[k] = true - targets = append(targets, pendingTarget{MsgID: msgID, Domain: domain}) + addTarget(msgID, addr[lastAt+1:]) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // Participant domains pending add-to notification (SPEC §10.2): domains + // hosting no recipient of a batch still receive the batch's add-to message + // so all participants learn recipients were added. + nrows, err := db.Query(` + SELECT b.msg_id, n.domain + FROM msg_add_to_notify n + INNER JOIN msg_add_to_batch b ON b.id = n.batch_id + INNER JOIN msg m ON m.id = b.msg_id + WHERE n.time_notified IS NULL + AND m.time_sent IS NOT NULL + AND (n.response_code IS NULL OR n.response_code = ANY($4)) + AND (n.time_last_attempt IS NULL OR ($1 - n.time_last_attempt) > LEAST($2 * POWER(2.0, GREATEST(n.attempt_count - 1, 0)::float), $3)) + AND ($1 - b.time_added) < $3 + `, now, RetryInterval, RetryMaxAge, pq.Array(retryableResponseCodes)) + if err != nil { + return nil, err + } + defer nrows.Close() + for nrows.Next() { + var msgID int64 + var domain string + if err := nrows.Scan(&msgID, &domain); err != nil { + return nil, err } + addTarget(msgID, domain) } - return targets, rows.Err() + return targets, nrows.Err() } // sendMsgData transmits the message body then all attachment payloads on conn. @@ -176,13 +211,70 @@ func commitOrLog(tx *sql.Tx, committed *bool, msgID int64) { } } -// recordRetryableFailure marks one delivery unit's locked recipients for retry. -func recordRetryableFailure(tx *sql.Tx, committed *bool, table string, locked []string, msgID int64) { +// recordRetryableFailure marks one delivery unit's locked recipients and +// locked notify row (if any) for retry. +func recordRetryableFailure(tx *sql.Tx, committed *bool, table string, locked []string, msgID int64, notifyID int64) { now := timeutil.TimestampNow().Float64() updateLocked(tx, table, locked, msgID, now, localResponseCodeNoResponse, false) + updateNotify(tx, notifyID, now, localResponseCodeNoResponse, false) commitOrLog(tx, committed, msgID) } +// lockPendingNotify locks (FOR UPDATE SKIP LOCKED) the pending +// msg_add_to_notify row for one batch and domain, returning its id — 0 when +// none is pending or another sender holds it. A notify row is a participant +// domain owed the batch's add-to message even though it hosts none of the +// batch's recipients (SPEC §10.2). +func lockPendingNotify(tx *sql.Tx, batchID int64, domain string, now float64) (int64, error) { + if batchID == 0 { + return 0, nil + } + var id int64 + err := tx.QueryRow(` + SELECT n.id + FROM msg_add_to_notify n + INNER JOIN msg_add_to_batch b ON b.id = n.batch_id + INNER JOIN msg m ON m.id = b.msg_id + WHERE n.batch_id = $1 + AND lower(n.domain) = lower($2) + AND n.time_notified IS NULL + AND m.time_sent IS NOT NULL + AND (n.response_code IS NULL OR n.response_code = ANY($6)) + AND (n.time_last_attempt IS NULL OR ($3 - n.time_last_attempt) > LEAST($4 * POWER(2.0, GREATEST(n.attempt_count - 1, 0)::float), $5)) + AND ($3 - b.time_added) < $5 + FOR UPDATE OF n SKIP LOCKED + `, batchID, domain, now, RetryInterval, RetryMaxAge, pq.Array(retryableResponseCodes)).Scan(&id) + if err == sql.ErrNoRows { + return 0, nil + } + return id, err +} + +// updateNotify records a notification outcome for one msg_add_to_notify row. +// notifyID 0 is a no-op so callers need not branch on whether the unit +// carried a notify row. +func updateNotify(tx *sql.Tx, notifyID int64, now float64, code int, notified bool) { + if notifyID == 0 { + return + } + var err error + if notified { + _, err = tx.Exec(` + UPDATE msg_add_to_notify SET time_notified = $1, response_code = $2 + WHERE id = $3 + `, now, code, notifyID) + } else { + _, err = tx.Exec(` + UPDATE msg_add_to_notify SET time_last_attempt = $1, response_code = $2, + attempt_count = attempt_count + 1 + WHERE id = $3 + `, now, code, notifyID) + } + if err != nil { + log.Printf("ERROR: sender: update notify row %d: %s", notifyID, err) + } +} + // deflatePart is one compressed payload (message body or attachment). type deflatePart struct { path string @@ -268,16 +360,25 @@ func (d deflateState) removeTempFiles() { // addresses. For msg_add_to it locks only rows in batchID, so each add-to batch // is delivered as an independent unit. func lockPendingRecipients(tx *sql.Tx, table string, msgID int64, domain string, batchID int64, now float64) ([]string, error) { + // msg_add_to rows age against their batch, not the message — recipients + // may be added long after the message was sent. + ageRef := "m.time_sent" + joinBatch := "" + if table == "msg_add_to" { + ageRef = "b.time_added" + joinBatch = "INNER JOIN msg_add_to_batch b ON b.id = r.batch_id" + } q := fmt.Sprintf(` SELECT r.addr FROM %s r INNER JOIN msg m ON m.id = r.msg_id + %s WHERE r.msg_id = $1 AND r.time_delivered IS NULL AND m.time_sent IS NOT NULL AND (r.response_code IS NULL OR r.response_code = ANY($5)) AND (r.time_last_attempt IS NULL OR ($2 - r.time_last_attempt) > LEAST($3 * POWER(2.0, GREATEST(r.attempt_count - 1, 0)::float), $4)) - AND ($2 - m.time_sent) < $4`, table) + AND ($2 - %s) < $4`, table, joinBatch, ageRef) args := []interface{}{msgID, now, RetryInterval, RetryMaxAge, pq.Array(retryableResponseCodes)} if table == "msg_add_to" { q += " AND r.batch_id = $6" @@ -366,6 +467,9 @@ func deliverMessage(target pendingTarget) { // Deliver each add-to batch as its own add-to message (one sender each). for _, b := range batches { + if len(b.Recipients) == 0 { + continue // degenerate batch; an add-to message needs ≥ 1 recipient + } h := m.addToHeader(b, sharedHash) d.applyTo(h) deliverUnit(db, target, h, "msg_add_to", b.ID) @@ -433,14 +537,24 @@ func deliverUnit(db *sql.DB, target pendingTarget, h *FMsgHeader, table string, log.Printf("ERROR: sender: lock %s rows for msg %d: %s", table, target.MsgID, err) return } - if len(locked) == 0 { + // An add-to unit may be owed to this domain as a participant notification + // even when it hosts none of the batch's recipients (SPEC §10.2). + var notifyID int64 + if table == "msg_add_to" { + notifyID, err = lockPendingNotify(tx, batchID, target.Domain, now) + if err != nil { + log.Printf("ERROR: sender: lock notify row for batch %d: %s", batchID, err) + return + } + } + if len(locked) == 0 && notifyID == 0 { return // nothing pending for this unit (or locked by another sender) } deferRetry := true defer func() { if deferRetry && !committed { - recordRetryableFailure(tx, &committed, table, locked, target.MsgID) + recordRetryableFailure(tx, &committed, table, locked, target.MsgID, notifyID) } }() @@ -543,6 +657,7 @@ func deliverUnit(db *sql.DB, target pendingTarget, h *FMsgHeader, table string, } log.Printf("INFO: sender: msg %d add-to accepted by %s (code 11)", target.MsgID, target.Domain) updateLocked(tx, table, locked, target.MsgID, now, int(initCode[0]), true) + updateNotify(tx, notifyID, now, int(initCode[0]), true) deferRetry = false commitOrLog(tx, &committed, target.MsgID) return @@ -552,6 +667,7 @@ func deliverUnit(db *sql.DB, target pendingTarget, h *FMsgHeader, table string, log.Printf("WARN: sender: msg %d rejected by %s: %s (%d)", target.MsgID, target.Domain, responseCodeName(initCode[0]), initCode[0]) updateLocked(tx, table, locked, target.MsgID, now, int(initCode[0]), false) + updateNotify(tx, notifyID, now, int(initCode[0]), false) deferRetry = false commitOrLog(tx, &committed, target.MsgID) } else { @@ -586,6 +702,10 @@ func deliverUnit(db *sql.DB, target pendingTarget, h *FMsgHeader, table string, updateRecipient(tx, table, dr.addr, target.MsgID, now, int(c), delivered) } + // The exchange completed, so a locked notify row is satisfied too: the + // domain was informed of the batch through this delivery. + updateNotify(tx, notifyID, now, int(initCode[0]), true) + deferRetry = false commitOrLog(tx, &committed, target.MsgID) } diff --git a/cmd/fmsgd/store.go b/cmd/fmsgd/store.go index c23dbea..af35c0e 100644 --- a/cmd/fmsgd/store.go +++ b/cmd/fmsgd/store.go @@ -29,7 +29,7 @@ func testDb() error { log.Printf("INFO: Database connected: %s@%s:%s/%s", user, host, port, dbName) // verify required tables exist - for _, table := range []string{"msg", "msg_to", "msg_add_to", "msg_add_to_batch", "msg_attachment"} { + for _, table := range []string{"msg", "msg_to", "msg_add_to", "msg_add_to_batch", "msg_add_to_notify", "msg_attachment"} { var exists bool err = db.QueryRow(`SELECT EXISTS ( SELECT FROM information_schema.tables @@ -244,6 +244,37 @@ func existingMsgIDForAddTo(tx *sql.Tx, msg *FMsgHeader, msgHash []byte) (int64, return id, err } +// addToBatchRecorded reports whether an incoming add-to batch carries nothing +// this host has not already recorded against stored message msgID: every +// address is unique per message across batches (msg_add_to unique (msg_id, +// addr)), so when every address in the incoming batch is already attached the +// delivery is a re-send of a recorded batch and is a duplicate (code 10, +// SPEC §10.4 step 1). +func addToBatchRecorded(msgID int64, addTo []FMsgAddress) (bool, error) { + if len(addTo) == 0 { + return false, nil + } + db, err := sql.Open("postgres", "") + if err != nil { + return false, err + } + defer db.Close() + + for i := range addTo { + var exists bool + err = db.QueryRow(`SELECT EXISTS ( + SELECT 1 FROM msg_add_to WHERE msg_id = $1 AND lower(addr) = $2 + )`, msgID, strings.ToLower(addTo[i].ToString())).Scan(&exists) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + } + return true, nil +} + // insertAddToBatch records one add-to delivery as a batch (its sender and the // time this host recorded it) and returns the new batch id. Recipients carried // by the delivery are linked to this batch so readers can reconstruct who added @@ -264,14 +295,22 @@ values ($1, $2, $3) returning id`, msgID, addToFrom, now).Scan(&batchID) func attachAddToRecipients(tx *sql.Tx, msgID int64, msg *FMsgHeader) error { now := timeutil.TimestampNow().Float64() + // This host is recording a batch it RECEIVED: delivering the batch to + // other domains is the batch sender's job, not ours (SPEC §10.2 — a host + // delivers iff from or add to from belongs to its domain). Rows for other + // domains are recorded with response code 11 (accept add to) so the + // sender's pending queries never treat them as our delivery work. for _, addr := range msg.To { var delivered interface{} + var code interface{} if addr.Domain == Domain { delivered = now + } else { + code = int16(AcceptCodeAddTo) } - if _, err := tx.Exec(`insert into msg_to (msg_id, addr, time_delivered) -values ($1, $2, $3) -on conflict (msg_id, addr) do nothing`, msgID, addr.ToString(), delivered); err != nil { + if _, err := tx.Exec(`insert into msg_to (msg_id, addr, time_delivered, response_code) +values ($1, $2, $3, $4) +on conflict (msg_id, addr) do nothing`, msgID, addr.ToString(), delivered, code); err != nil { return err } } @@ -291,12 +330,15 @@ on conflict (msg_id, addr) do nothing`, msgID, addr.ToString(), delivered); err for _, addr := range msg.AddTo { var delivered interface{} + var code interface{} if addr.Domain == Domain { delivered = now + } else { + code = int16(AcceptCodeAddTo) } - if _, err := tx.Exec(`insert into msg_add_to (msg_id, batch_id, addr, time_delivered) -values ($1, $2, $3, $4) -on conflict (msg_id, addr) do nothing`, msgID, batchID, addr.ToString(), delivered); err != nil { + if _, err := tx.Exec(`insert into msg_add_to (msg_id, batch_id, addr, time_delivered, response_code) +values ($1, $2, $3, $4, $5) +on conflict (msg_id, addr) do nothing`, msgID, batchID, addr.ToString(), delivered, code); err != nil { return err } } diff --git a/dd.sql b/dd.sql index a2e8c2b..e32dcee 100644 --- a/dd.sql +++ b/dd.sql @@ -253,3 +253,79 @@ create trigger trg_msg_add_to_delivered for each row when (OLD.time_delivered is null and NEW.time_delivered is not null) execute function notify_delivered(); + +-- Sender-side state for add-to participant notification (SPEC §10.2): an +-- add-to message is sent to every participant domain of the message being +-- added to -- the domains of from and every to address as well as the new +-- recipients' -- so all participants learn recipients were added, not only +-- the domains hosting the new recipients. Domains hosting a recipient of the +-- batch itself learn through normal recipient delivery; every other +-- participant domain gets one row here per batch and receives the add-to as +-- a notification-only exchange completing at code 11. Rows are created by +-- the Web API when recipients are added through it (the local domain itself +-- needs no row -- this database is its record). +create table if not exists msg_add_to_notify ( + id bigserial primary key, + batch_id bigint not null references msg_add_to_batch (id), + domain varchar(255) not null, + time_notified double precision, -- time remote host acknowledged the batch; null means pending + time_last_attempt double precision, -- time of last failed attempt; drives exponential back-off + response_code smallint, -- response code of last attempt + attempt_count int not null default 0, + unique (batch_id, domain) +); + +-- Wake the sender's outgoing worker (channel new_msg_to) for a pending +-- participant notification, mirroring notify_msg_sent for recipient rows. +-- The payload is advisory only: the worker re-polls fully on any wake-up. +create or replace function notify_add_to_notify_pending() returns trigger as $$ +begin + perform pg_notify('new_msg_to', b.msg_id::text || ',' || NEW.domain) + from msg_add_to_batch b + inner join msg m on m.id = b.msg_id + where b.id = NEW.batch_id and m.time_sent is not null; + return NEW; +end; +$$ language plpgsql; + +drop trigger if exists trg_msg_add_to_notify_insert on msg_add_to_notify; +create trigger trg_msg_add_to_notify_insert + after insert on msg_add_to_notify + for each row execute function notify_add_to_notify_pending(); + +-- Notify listeners (channel recipients_added) that an add-to batch was +-- recorded against a sent message, so existing participants' clients learn of +-- the new recipients without polling. Fires wherever a batch is recorded -- +-- added locally through the Web API or received from a remote host -- because +-- both paths insert a msg_add_to_batch row. Payload is ",", one +-- notification per participant (from, every msg_to and every msg_add_to +-- address, including the new batch's own recipients, who have no other +-- realtime event for a message that was sent before they were added); the +-- listener checks addr against its currently-connected clients, exactly as +-- new_msg. Like trg_new_msg this is a deferred constraint trigger: the +-- batch's own msg_add_to rows are inserted after the batch row, so only at +-- commit is the full recipient set visible. +create or replace function notify_recipients_added() returns trigger as $$ +begin + if not exists (select 1 from msg where id = NEW.msg_id and time_sent is not null) then + return NEW; + end if; + + perform pg_notify('recipients_added', NEW.msg_id::text || ',' || from_addr) + from msg where id = NEW.msg_id; + + perform pg_notify('recipients_added', NEW.msg_id::text || ',' || addr) + from msg_to where msg_id = NEW.msg_id; + + perform pg_notify('recipients_added', NEW.msg_id::text || ',' || addr) + from msg_add_to where msg_id = NEW.msg_id; + + return NEW; +end; +$$ language plpgsql; + +drop trigger if exists trg_recipients_added on msg_add_to_batch; +create constraint trigger trg_recipients_added + after insert on msg_add_to_batch + deferrable initially deferred + for each row execute function notify_recipients_added();