feat(sync): quorum/sync subsystem hardening (proof spot-check, bad-tfile cache, chain re-selection)

src/bin/mochimo.c

@@ -402,7 +402,7 @@ int init(void)
       }
       if (iszero(Cblocknum, 8)) {
          /* we're starting from scratch... */
-         resync(quorum, &qlen, netweight, netbnum);
+         resync(quorum, &qlen, nethash, netweight, netbnum);
       }
       status = VEOK;  /* don't panic... EVERYTHING IS FINE! */
       result = cmp256(Weight, netweight);  /* compare network weight */
@@ -467,7 +467,7 @@ int init(void)
             } */
          }
          pdebug("...attempting resync()...");
-         resync(quorum, &qlen, netweight, netbnum);
+         resync(quorum, &qlen, nethash, netweight, netbnum);
          break;
       }  /* ... did we catch up? */
       if (cmp256(Weight, netweight) >= 0) {

src/network.c

@@ -20,6 +20,7 @@
 #include "ledger.h"
 #include "global.h"
 #include "error.h"
+#include "syncguard.h"
 
 /* external support */
 #include <string.h>
@@ -731,6 +732,66 @@ int get_file(word32 ip, word8 *bnum, char *fname)
    return ecode;
 }  /* end get_file() */
 
+/**
+ * Get a proof segment (partial tfile) from a peer via OP_TF.
+ * Fetches `count` trailers starting at `first_bnum` into `out_proof`.
+ * Returns VEOK on success with proof buffer filled, else VERROR.
+ *
+ * Used by scan_quorum() to spot-check candidate quorum members before
+ * admitting them. Does NOT validate the contents — caller should run
+ * sg_validate_proof_chain() to verify structural integrity.
+ *
+ * Uses a per-IP temp file on disk to avoid collisions when called from
+ * within the scan_quorum() OMP parallel loop. */
+int get_tf_proof(word32 ip, const word8 first_bnum[8], word32 count,
+   BTRAILER *out_proof)
+{
+   NODE node;
+   FILE *fp;
+   char fname[32];
+   char ipstr[16];
+   int ecode;
+   word32 i;
+
+   if (out_proof == NULL || count == 0 || count > NTFTX) return VERROR;
+
+   ntoa(&ip, ipstr);
+   snprintf(fname, sizeof(fname), "tfp_%s.tmp", ipstr);
+   /* replace dots so it is a safe filename */
+   for (i = 0; fname[i]; i++) if (fname[i] == '.') fname[i] = '_';
+
+   ecode = callserver(&node, ip);
+   if (ecode != VEOK) return ecode;
+
+   put64(node.tx.blocknum, first_bnum);
+   put32(&node.tx.blocknum[4], count);
+
+   ecode = send_op(&node, OP_TF);
+   if (ecode == VEOK) ecode = recv_file(&node, fname);
+
+   sock_close(node.sd);
+   node.sd = INVALID_SOCKET;
+
+   if (ecode != VEOK) {
+      remove(fname);
+      return VERROR;
+   }
+
+   fp = fopen(fname, "rb");
+   if (fp == NULL) {
+      remove(fname);
+      return VERROR;
+   }
+   if (fread(out_proof, sizeof(BTRAILER), count, fp) != count) {
+      fclose(fp);
+      remove(fname);
+      return VERROR;
+   }
+   fclose(fp);
+   remove(fname);
+   return VEOK;
+}  /* end get_tf_proof() */
+
 /**
  * Get an ip list from ip, and call addrecent() on the list.
  * Return VEOK if successful, else error code.
@@ -972,37 +1033,85 @@ int scan_quorum
       /* prepare parallel processing scope */
       OMP_PARALLEL_(for private(node, peer, len, ipstr) num_threads(qlen))
       for (word32 idx = scanidx; idx < netplistidx; idx++) {
+         /* per-iteration thread-private state */
+         word8 peer_weight[32];
+         word8 peer_hash[HASHLEN];
+         word8 peer_bnum[8];
+         word8 proof_first[8];
+         BTRAILER proof_buf[NTFTX];
+         int proof_ok;
+         int excluded;
+
          /* get IP list from peer */
          peer = netplist[idx];
          if (get_ipl(&node, peer) == VEOK) {
+            /* capture peer's advertised chain identity */
+            memcpy(peer_weight, node.tx.weight, 32);
+            memcpy(peer_hash, node.tx.cblockhash, HASHLEN);
+            put64(peer_bnum, node.tx.cblock);
+
+            /* Phase 4: check bad-chain exclusion list before expensive
+             * proof fetch — skip if this chain has already been marked bad
+             * in the current sync session */
+            OMP_CRITICAL_()
+            excluded = sg_bad_chain_check(peer_weight, peer_hash);
+
+            /* Phase 2: spot-check the peer's proof segment BEFORE we admit
+             * them to the quorum. Request their last NTFTX trailers; verify
+             * structural integrity (bnum chain-climb, phash linkage, tip
+             * matches advertised). Full PoW validation is skipped here
+             * because it is too expensive per-peer and is validated
+             * downstream in validate_tfile_pow(). */
+            proof_ok = 0;
+            if (!excluded && cmp64(peer_bnum, CL64_32(NTFTX)) >= 0) {
+               /* compute first_bnum = peer_bnum - (NTFTX - 1) */
+               memcpy(proof_first, peer_bnum, 8);
+               if (sub64(proof_first, CL64_32(NTFTX - 1), proof_first) == 0) {
+                  if (get_tf_proof(peer, proof_first, NTFTX, proof_buf)
+                      == VEOK &&
+                      sg_validate_proof_chain(proof_buf, NTFTX,
+                         peer_hash, peer_bnum) == VEOK) {
+                     proof_ok = 1;
+                  } else {
+                     pdebug("%s proof spot-check FAILED",
+                        ntoa(&peer, (char[16]){0}));
+                  }
+               }
+            }
+
             OMP_CRITICAL_()
             {
                /* check peer's chain weight against highweight */
-               result = cmp256(node.tx.weight, highweight);
-               if (result >= 0) {
+               result = cmp256(peer_weight, highweight);
+               if (!excluded && result >= 0) {
                   /* new best chain: strictly higher weight, OR equal weight
                    * with numerically higher block hash (deterministic
                    * tiebreaker to ensure quorum members share a single chain,
                    * regardless of peer scan order) */
                   if (result > 0 ||
-                      memcmp(node.tx.cblockhash, highhash, HASHLEN) > 0) {
+                      memcmp(peer_hash, highhash, HASHLEN) > 0) {
                      pdebug("new high chain");
-                     memcpy(highhash, node.tx.cblockhash, HASHLEN);
-                     memcpy(highweight, node.tx.weight, 32);
-                     put64(highbnum, node.tx.cblock);
+                     memcpy(highhash, peer_hash, HASHLEN);
+                     memcpy(highweight, peer_weight, 32);
+                     put64(highbnum, peer_bnum);
                      qcount = 0;
                      if (quorum) {
                         memset(quorum, 0, qlen * sizeof(word32));
                         pdebug("new high chain found, quorum reset...");
                      }
                   }
                   /* add ip to quorum only if it shares the exact high chain
-                   * hash -- peers on different chains of equal weight must
-                   * not be combined into a single quorum */
-                  if (memcmp(node.tx.cblockhash, highhash, HASHLEN) == 0) {
+                   * hash AND passed the proof spot-check -- peers on
+                   * different chains of equal weight must not be combined
+                   * into a single quorum, and peers with fabricated or
+                   * inconsistent chain state must not be trusted */
+                  if (proof_ok &&
+                      memcmp(peer_hash, highhash, HASHLEN) == 0) {
                      if (quorum && qcount < qlen) {
                         quorum[qcount++] = peer;
-                        pdebug("%s qualified", ntoa(&peer, NULL));
+                        sg_proof_store(peer, proof_buf, NTFTX);
+                        pdebug("%s qualified (proof verified)",
+                           ntoa(&peer, NULL));
                      } else if (quorum == NULL) qcount++;
                   }
                }  /* end if higher or same chain */

src/network.h

@@ -66,6 +66,8 @@ int send_identify(NODE *np);
 int send_found(void);
 int callserver(NODE *np, word32 ip);
 int get_file(word32 ip, word8 *bnum, char *fname);
+int get_tf_proof(word32 ip, const word8 first_bnum[8], word32 count,
+   BTRAILER *out_proof);
 int get_ipl(NODE *np, word32 ip);
 int get_hash(NODE *np, word32 ip, void *bnum, void *blockhash);
 int gettx(NODE *np, SOCKET sd);

src/sync.c

@@ -23,6 +23,7 @@
 #include "bval.h"
 #include "bup.h"
 #include "util.h"
+#include "syncguard.h"
 
 /* external support */
 #include "extthrd.h"
@@ -202,7 +203,8 @@ int catchup(word32 plist[], word32 count)
 /**
  * Resynchronize blockchain up to network weight/bnum using quorum[qidx].
  * Returns VEOK on success, else restarts. */
-int resync(word32 quorum[], word32 *qidx, void *highweight, void *highbnum)
+int resync(word32 quorum[], word32 *qidx, void *highhash, void *highweight,
+           void *highbnum)
 {
    char ipaddr[16], fname[FILENAME_MAX], bcfname[21];
    word8 bnum[8], weight[HASHLEN] = { 0 };
@@ -214,44 +216,135 @@ int resync(word32 quorum[], word32 *qidx, void *highweight, void *highbnum)
       return VERROR;
    }
 
+   /* NOTE: session-local caches (bad-chain, bad-tfile, proofs) are not
+    * reset here. The proof cache was populated by scan_quorum() and must
+    * persist through the gettfile loop for the spot-check below to work.
+    * bad-chain and bad-tfile caches must persist across resync retries
+    * within the same process so we don't fall into repeated failures
+    * on the same malicious peers. All caches are zero-initialized at
+    * process start via static storage duration. */
+
    show("gettfile");  /* get tfile */
    pdebug("fetching tfile.dat from %s", ntoa(&quorum[0], ipaddr));
    pdebug("... this is a large file, please be patient !!!");
+
+   /* Download and validate tfile from quorum members in turn. Apply
+    * defense-in-depth checks before accepting a tfile:
+    *   1. Bad-tfile hash cache: skip downloads whose hash matches a
+    *      previously-failed tfile from an earlier quorum member.
+    *   2. Tail spot-check: the last NTFTX trailers of the downloaded
+    *      tfile must byte-exactly match the proof segment that this
+    *      peer provided during scan_quorum(). A mismatch means the
+    *      peer served a different chain than it advertised.
+    *   3. Structural validation (existing validate_tfile).
+    *   4. PoW validation (existing validate_tfile_pow).
+    * Any failure adds the tfile hash to the bad-tfile cache and drops
+    * the peer from the quorum before trying the next member. */
    while(Running && *quorum) {
+      word8 tfile_hash[HASHLEN];
+      int tfile_fetched = 0;
+
       remove("tfile.dat");
-      if (get_file(*quorum, NULL, "tfile.tmp") == VEOK) {
-         if (rename("tfile.tmp", "tfile.dat") == 0) break;
+      remove("tfile.tmp");
+      if (get_file(*quorum, NULL, "tfile.tmp") != VEOK) {
+         pdebug("gettfile: %s get_file() failed", ntoa(quorum, ipaddr));
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
+      }
+      tfile_fetched = 1;
+
+      /* hash the downloaded tfile and consult the bad-tfile cache */
+      if (sg_hash_file("tfile.tmp", tfile_hash) != VEOK) {
+         pdebug("gettfile: failed to hash tfile.tmp");
+         remove("tfile.tmp");
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
+      }
+      if (sg_bad_tfile_check(tfile_hash)) {
+         pdebug("gettfile: %s served a known-bad tfile (cached)",
+            ntoa(quorum, ipaddr));
+         remove("tfile.tmp");
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
+      }
+
+      /* rename into place for subsequent validation */
+      if (rename("tfile.tmp", "tfile.dat") != 0) {
          perrno("failed to rename tfile.dat");
+         remove("tfile.tmp");
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
       }
-      /* remove quorum member, and try again */
-      remove32(*quorum, quorum, *qidx, qidx);
-   }
-   if (!(*quorum)) restart("gettfile no quorum");
-   if (!Running) resign("gettfile exiting");
 
-   show("tfval");  /* validate tfile */
-   /* do some quick maths to estimate time for tfile validation */
-   memcpy(&hb, highbnum, sizeof(hb));
-   pdebug("validating tfile (est. %u seconds)...",
-      (word32) (hb / 300 / OMP_MAX_THREADS));
-   if (validate_tfile("tfile.dat", bnum, weight, 0) != VEOK) {
-      remove("tfile.dat.fail");
-      rename("tfile.dat", "tfile.dat.fail");
-      perrno("validate_tfile(tfile.dat, 0x%s, 0x%s, 0) FAILURE",
-         bnum2hex(bnum, NULL), weight2hex(weight, NULL));
-      return VERROR;
-   } else if (validate_tfile_pow("tfile.dat", Trustblock) != VEOK) {
-      remove("tfile.pow.fail");
-      rename("tfile.dat", "tfile.pow.fail");
-      perrno("validate_tfile_pow(tfile.dat, 0) FAILURE");
+      /* Phase 3 spot-check: the peer's validated proof segment from
+       * scan_quorum() must appear byte-exactly at the corresponding
+       * bnum offset in the tfile they just served. The tfile may have
+       * advanced past the proof's tip if the peer received new blocks
+       * between scan_quorum() and resync(); we only require that the
+       * historical trailers at the proof's bnum range match exactly.
+       * Mismatch = they served a different chain than they advertised. */
+      if (sg_proof_match_tfile(*quorum, "tfile.dat") != VEOK) {
+         pdebug("gettfile: %s tfile does NOT match its advertised "
+            "proof -- marking tfile bad", ntoa(quorum, ipaddr));
+         sg_bad_tfile_add(tfile_hash);
+         remove("tfile.dat");
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
+      }
+
+      show("tfval");
+      memcpy(&hb, highbnum, sizeof(hb));
+      pdebug("validating tfile (est. %u seconds)...",
+         (word32) (hb / 300 / OMP_MAX_THREADS));
+      if (validate_tfile("tfile.dat", bnum, weight, 0) != VEOK) {
+         sg_bad_tfile_add(tfile_hash);
+         remove("tfile.dat.fail");
+         rename("tfile.dat", "tfile.dat.fail");
+         perrno("validate_tfile(tfile.dat, 0x%s, 0x%s, 0) FAILURE",
+            bnum2hex(bnum, NULL), weight2hex(weight, NULL));
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
+      }
+      if (validate_tfile_pow("tfile.dat", Trustblock) != VEOK) {
+         sg_bad_tfile_add(tfile_hash);
+         remove("tfile.pow.fail");
+         rename("tfile.dat", "tfile.pow.fail");
+         perrno("validate_tfile_pow(tfile.dat, 0) FAILURE");
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
+      }
+      pdebug("tfile.dat is valid");
+      if (cmp256(weight, highweight) < 0 || cmp64(bnum, highbnum) < 0) {
+         sg_bad_tfile_add(tfile_hash);
+         pdebug("tfile.dat does NOT match advertised bnum and weight");
+         remove("tfile.dat");
+         remove32(*quorum, quorum, *qidx, qidx);
+         continue;
+      }
+      pdebug("tfile.dat matches advertised bnum and weight.");
+
+      /* tfile accepted */
+      (void)tfile_fetched;  /* suppress unused warning on some builds */
+      break;
+   }  /* end while (Running && *quorum) */
+
+   /* Phase 4: if all quorum members failed, mark this chain as bad for
+    * the remainder of this session and return VERROR so the bootstrap
+    * loop can re-scan and select a different chain. This replaces the
+    * previous restart() call which exit()ed the process entirely and
+    * lost the bad-chain cache along with it. */
+   if (!(*quorum)) {
+      if (highhash && highweight) {
+         sg_bad_chain_add((const word8 *)highweight, (const word8 *)highhash);
+         perr("gettfile: all quorum members exhausted for chain 0x%s - "
+            "marking bad and returning for re-scan",
+            weight2hex((void *)highweight, NULL));
+      } else {
+         perr("gettfile: all quorum members exhausted");
+      }
       return VERROR;
    }
-   pdebug("tfile.dat is valid");
-   if (cmp256(weight, highweight) >= 0 && cmp64(bnum, highbnum) >= 0) {
-      pdebug("tfile.dat matches advertised bnum and weight.");
-   } else return VERROR;
-   if (!(*quorum)) restart("tfval no quorum");
-   if (!Running) resign("tfval exiting");
+   if (!Running) resign("gettfile exiting");
 
    /* determine starting neo-genesis block -- bump to V30TRIGGER */
    put64(bnum, highbnum); bnum[0] = 0;
@@ -281,7 +374,15 @@ int resync(word32 quorum[], word32 *qidx, void *highweight, void *highbnum)
          /* remove quorum member, and try again */
          remove32(*quorum, quorum, *qidx, qidx);
       }
-      if (!(*quorum)) restart("getneo no quorum");
+      if (!(*quorum)) {
+         if (highhash && highweight) {
+            sg_bad_chain_add((const word8 *)highweight,
+               (const word8 *)highhash);
+            perr("getneo: all quorum members exhausted - marking chain bad "
+               "and returning for re-scan");
+         }
+         return VERROR;
+      }
       if (!Running) resign("getneo exiting");
       /* transfer neo-genesis block to bcdir */
       bnum2fname(bnum, bcfname);