diff --git a/.gitignore b/.gitignore index 5280472a..1dfdfc85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ cache out .certora_internal +sql/test/__pycache__/ +sql/test/events/ +sql/test/expected_state.json + diff --git a/foundry.toml b/foundry.toml index 1c268f0f..1c26c070 100644 --- a/foundry.toml +++ b/foundry.toml @@ -4,7 +4,7 @@ optimizer = true optimizer_runs = 800 bytecode_hash = "none" evm_version = "osaka" -fs_permissions = [{ access = "read", path = "test/ticks_exact.json" }] +fs_permissions = [{ access = "read", path = "test/ticks_exact.json" }, { access = "read-write", path = "sql/test/" }] [profile.default.fmt] wrap_comments = true diff --git a/sql/README.md b/sql/README.md new file mode 100644 index 00000000..85c7cfd3 --- /dev/null +++ b/sql/README.md @@ -0,0 +1,60 @@ +# Midnight SQL Queries + +Each `.sql` file reconstructs one piece of Midnight's on-chain state from events alone, targeting **Dune Analytics** (Trino SQL with native `uint256`). The queries are self-contained — no off-chain storage is assumed. + +## Queries + +| File | State variable reconstructed | +|---|---| +| `position.sql` | `position[id][user]` — credit, debt, pendingFee, lastLossFactor, lastAccrual | +| `position_collateral.sql` | `position[id][user].collateral[index]` + collateralBitmap | +| `market_state.sql` | `marketState[id]` — all fields including continuousFeeCredit (recursive CTE) | +| `consumed.sql` | `consumed[user][group]` | +| `is_authorized.sql` | `isAuthorized[authorizer][authorized]` | +| `claimable_settlement_fee.sql` | `claimableSettlementFee[token]` | +| `default_settlement_fee.sql` | `defaultSettlementFeeCbp[loanToken][index]` | +| `default_continuous_fee.sql` | `defaultContinuousFee[loanToken]` | +| `roles.sql` | `roleSetter`, `feeSetter`, `feeClaimer`, `tickSpacingSetter` | + +Dune table names follow the pattern `midnight.midnight_evt_` (all lowercase). + +## Testing + +The test harness verifies every query against a live Foundry scenario: + +1. **`test/SqlScenarioTest.sol`** deploys Midnight, runs a randomised sequence of operations (takes, repays, liquidations, fee claims, …), captures all emitted events via `vm.recordLogs()`, and writes two artefacts: + - `sql/test/events/.json` — one JSON array per event type, used as the SQL input tables + - `sql/test/expected_state.json` — the actual contract state read at the end, used as ground truth + +2. **`test/verify.py`** loads the event JSON files into an in-memory DuckDB instance, executes each `.sql` file against them (adapting Trino-specific syntax on the fly), and asserts that the results match `expected_state.json`. + +### Run the tests + +```bash +bash sql/test/run.sh +``` + +This runs the Forge scenario test then the Python verifier in one step. [uv](https://github.com/astral-sh/uv) is required; it installs the pinned Python dependencies automatically. + +### Dependencies + +Python dependencies are pinned in `sql/test/requirements.txt`: + +``` +duckdb==1.5.3 +pandas==3.0.3 +``` + +The DuckDB version matters: `UHUGEINT` values (used for `uint128` fields such as `lossFactor`) must be returned as exact Python integers, which requires DuckDB ≥ 1.5. + +### How the SQL adaptation works + +`verify.py` translates Trino SQL to DuckDB SQL on the fly (`adapt_sql`): + +- Table references `midnight.midnight_evt_*` → bare table names +- `UINT256 '…'` / `BIGINT '…'` literals → plain integer literals +- `CAST(… AS uint256)` → `CAST(… AS BIGINT)` +- `MAX_BY(x, ROW(block, idx))` → `arg_max(x, block * 1e9 + idx)` +- `WITH` → `WITH RECURSIVE` (DuckDB requires the keyword; Trino infers it) +- The `MAX_U128` sentinel (`2¹²⁸ − 1`) → `…::UHUGEINT` so that CFC arithmetic does not truncate +- The `cfc_state` recursive CTE seed row is rewritten to initialize `cfc` and `prev_lf` as `UHUGEINT` so DuckDB preserves the type throughout the recursion diff --git a/sql/claimable_settlement_fee.sql b/sql/claimable_settlement_fee.sql new file mode 100644 index 00000000..b04bce4e --- /dev/null +++ b/sql/claimable_settlement_fee.sql @@ -0,0 +1,28 @@ +-- Reconstructs claimableSettlementFee[token] from events. +-- Take: claimableSettlementFee[loanToken] += (buyerAssets - sellerAssets) +-- ClaimSettlementFee: claimableSettlementFee[token] -= amount +-- Platform: Dune Analytics (Trino SQL, native uint256) + +WITH + +market_loan_tokens AS ( + SELECT id_, market_loantoken AS loan_token + FROM midnight.midnight_evt_marketcreated +), + +deltas AS ( + -- Settlement fee accrues on every Take: buyer pays more than seller receives + SELECT mlt.loan_token AS token, t.buyerassets - t.sellerassets AS add_, UINT256 '0' AS sub_ + FROM midnight.midnight_evt_take t + JOIN market_loan_tokens mlt ON mlt.id_ = t.id_ + + UNION ALL + + -- Fee claimer withdraws accumulated fees + SELECT token, UINT256 '0' AS add_, amount AS sub_ + FROM midnight.midnight_evt_claimsettlementfee +) + +SELECT token, SUM(add_) - SUM(sub_) AS claimable_settlement_fee +FROM deltas +GROUP BY token diff --git a/sql/consumed.sql b/sql/consumed.sql new file mode 100644 index 00000000..112680ac --- /dev/null +++ b/sql/consumed.sql @@ -0,0 +1,45 @@ +-- Reconstructs consumed[user][group] from events. +-- Both Take and SetConsumed emit the ABSOLUTE current value (not a delta). +-- Returns the last value per (user, group). +-- Platform: Dune Analytics (Trino SQL, native uint256) +-- +-- In Take: consumed[offer.maker][offer.group] = consumed field (newConsumed after increment) +-- In SetConsumed: consumed[onBehalf][group] = amount (monotone set, may skip to max to cancel) + +WITH + +all_consumed_updates AS ( + -- From Take: the offer's maker consumed amount is updated + SELECT + maker AS user, + "group" AS "group", + consumed AS amount, + evt_block_number, + evt_index + FROM midnight.midnight_evt_take + + UNION ALL + + -- From SetConsumed: explicit set (e.g. to cancel all offers in a group) + SELECT + onbehalf AS user, + "group" AS "group", + amount AS amount, + evt_block_number, + evt_index + FROM midnight.midnight_evt_setconsumed +) + +SELECT user, "group", amount +FROM ( + SELECT + user, + "group", + amount, + ROW_NUMBER() OVER ( + PARTITION BY user, "group" + ORDER BY evt_block_number DESC, evt_index DESC + ) AS rn + FROM all_consumed_updates +) +WHERE rn = 1 diff --git a/sql/default_continuous_fee.sql b/sql/default_continuous_fee.sql new file mode 100644 index 00000000..9fd74c49 --- /dev/null +++ b/sql/default_continuous_fee.sql @@ -0,0 +1,16 @@ +-- Reconstructs defaultContinuousFee[loanToken] from events. +-- SetDefaultContinuousFee emits the new value; last value per loanToken wins. +-- Platform: Dune Analytics (Trino SQL, native uint256) + +SELECT loantoken AS loan_token, fee +FROM ( + SELECT + loantoken, + newcontinuousfee AS fee, + ROW_NUMBER() OVER ( + PARTITION BY loantoken + ORDER BY evt_block_number DESC, evt_index DESC + ) AS rn + FROM midnight.midnight_evt_setdefaultcontinuousfee +) +WHERE rn = 1 diff --git a/sql/default_settlement_fee.sql b/sql/default_settlement_fee.sql new file mode 100644 index 00000000..4b7d0b20 --- /dev/null +++ b/sql/default_settlement_fee.sql @@ -0,0 +1,17 @@ +-- Reconstructs defaultSettlementFeeCbp[loanToken][index] from events. +-- SetDefaultSettlementFee emits the new value; last value per (loanToken, index) wins. +-- Platform: Dune Analytics (Trino SQL, native uint256) + +SELECT loantoken AS loan_token, index, fee_cbp +FROM ( + SELECT + loantoken, + index, + newsettlementfee / 1000000000000 AS fee_cbp, + ROW_NUMBER() OVER ( + PARTITION BY loantoken, index + ORDER BY evt_block_number DESC, evt_index DESC + ) AS rn + FROM midnight.midnight_evt_setdefaultsettlementfee +) +WHERE rn = 1 diff --git a/sql/is_authorized.sql b/sql/is_authorized.sql new file mode 100644 index 00000000..be5e2c57 --- /dev/null +++ b/sql/is_authorized.sql @@ -0,0 +1,17 @@ +-- Reconstructs isAuthorized[authorizer][authorized] from events. +-- Returns the last newIsAuthorized value per (authorizer, authorized) pair. +-- Platform: Dune Analytics (Trino SQL) + +SELECT authorizer, authorized, is_authorized +FROM ( + SELECT + onbehalf AS authorizer, + authorized AS authorized, + newisauthorized AS is_authorized, + ROW_NUMBER() OVER ( + PARTITION BY onbehalf, authorized + ORDER BY evt_block_number DESC, evt_index DESC + ) AS rn + FROM midnight.midnight_evt_setisauthorized +) +WHERE rn = 1 diff --git a/sql/market_state.sql b/sql/market_state.sql new file mode 100644 index 00000000..fd5e1bbc --- /dev/null +++ b/sql/market_state.sql @@ -0,0 +1,203 @@ +-- Reconstructs marketState[id] from events. +-- Fields: total_units, loss_factor, withdrawable, continuous_fee_credit, +-- settlement_fee_cbp_0..6, continuous_fee, tick_spacing +-- Platform: Dune Analytics (Trino SQL, native uint256) + +WITH + +-- ── totalUnits ──────────────────────────────────────────────────────────────── +-- Take: +(buyerCreditIncrease - sellerCreditDecrease) +-- Withdraw: -units +-- Liquidate: -badDebt +-- ClaimContinuousFee: -amount + +total_units_deltas AS ( + SELECT id_, buyercreditincrease AS add_, sellercreditdecrease AS sub_ + FROM midnight.midnight_evt_take + UNION ALL + SELECT id_, UINT256 '0', units FROM midnight.midnight_evt_withdraw + UNION ALL + SELECT id_, UINT256 '0', baddebt FROM midnight.midnight_evt_liquidate + UNION ALL + SELECT id_, UINT256 '0', amount FROM midnight.midnight_evt_claimcontinuousfee +), + +total_units AS ( + SELECT id_, SUM(add_) - SUM(sub_) AS total_units + FROM total_units_deltas GROUP BY id_ +), + +-- ── lossFactor: last latestLossFactor per market ────────────────────────────── +-- Initial = 0 (before any liquidation) + +loss_factor AS ( + SELECT id_, + COALESCE(MAX_BY(latestlossfactor, ROW(evt_block_number, evt_index)), UINT256 '0') AS loss_factor + FROM midnight.midnight_evt_liquidate + GROUP BY id_ +), + +-- ── withdrawable ───────────────────────────────────────────────────────────── +-- Repay: +units +-- Liquidate: +repaidUnits +-- Withdraw: -units +-- ClaimContinuousFee: -amount + +withdrawable_deltas AS ( + SELECT id_, units AS add_, UINT256 '0' AS sub_ FROM midnight.midnight_evt_repay + UNION ALL + SELECT id_, repaidunits, UINT256 '0' FROM midnight.midnight_evt_liquidate + UNION ALL + SELECT id_, UINT256 '0', units FROM midnight.midnight_evt_withdraw + UNION ALL + SELECT id_, UINT256 '0', amount FROM midnight.midnight_evt_claimcontinuousfee +), + +withdrawable AS ( + SELECT id_, SUM(add_) - SUM(sub_) AS withdrawable + FROM withdrawable_deltas GROUP BY id_ +), + +-- ── continuousFeeCredit ─────────────────────────────────────────────────────── +-- Liquidate emits latestContinuousFeeCredit = the CFC value immediately after +-- the liquidation. Start from the last liquidation's emitted CFC, then add +-- UpdatePosition accruedFee and subtract ClaimContinuousFee amounts that +-- follow it (or process all events if no liquidation has occurred yet). + +last_liq AS ( + SELECT id_, + MAX_BY(latestcontinuousfeecredit, ROW(evt_block_number, evt_index)) AS cfc_after_last_liq, + MAX_BY(evt_block_number, ROW(evt_block_number, evt_index)) AS last_liq_block, + MAX_BY(evt_index, ROW(evt_block_number, evt_index)) AS last_liq_index + FROM midnight.midnight_evt_liquidate + GROUP BY id_ +), + +up_after_liq AS ( + SELECT u.id_, SUM(u.accruedfee) AS total_up + FROM midnight.midnight_evt_updateposition u + LEFT JOIN last_liq ll ON ll.id_ = u.id_ + WHERE ll.id_ IS NULL + OR u.evt_block_number > ll.last_liq_block + OR (u.evt_block_number = ll.last_liq_block AND u.evt_index > ll.last_liq_index) + GROUP BY u.id_ +), + +claim_after_liq AS ( + SELECT c.id_, SUM(c.amount) AS total_claim + FROM midnight.midnight_evt_claimcontinuousfee c + LEFT JOIN last_liq ll ON ll.id_ = c.id_ + WHERE ll.id_ IS NULL + OR c.evt_block_number > ll.last_liq_block + OR (c.evt_block_number = ll.last_liq_block AND c.evt_index > ll.last_liq_index) + GROUP BY c.id_ +), + +continuous_fee_credit AS ( + SELECT + ids.id_, + COALESCE(ll.cfc_after_last_liq, UINT256 '0') + + COALESCE(up.total_up, UINT256 '0') + - COALESCE(tc.total_claim, UINT256 '0') AS continuous_fee_credit + FROM ( + SELECT id_ FROM last_liq + UNION SELECT id_ FROM up_after_liq + UNION SELECT id_ FROM claim_after_liq + ) ids + LEFT JOIN last_liq ll ON ll.id_ = ids.id_ + LEFT JOIN up_after_liq up ON up.id_ = ids.id_ + LEFT JOIN claim_after_liq tc ON tc.id_ = ids.id_ +), + +-- ── settlementFeeCbp[0..6] and continuousFee ───────────────────────────────────── +-- Initial values come from MarketCreated (= defaults at creation time). +-- Override: SetMarketSettlementFee / SetMarketContinuousFee. + +-- Build a combined stream of (id, index, value) for settlement fee per breakpoint. +settlement_fee_stream AS ( + -- MarketCreated seeds all 7 breakpoints (indices 0-6) + SELECT id_, UINT256 '0' AS index, market_settlementfeecbp0 AS fee, evt_block_number, evt_index FROM midnight.midnight_evt_marketcreated + UNION ALL SELECT id_, UINT256 '1', market_settlementfeecbp1, evt_block_number, evt_index FROM midnight.midnight_evt_marketcreated + UNION ALL SELECT id_, UINT256 '2', market_settlementfeecbp2, evt_block_number, evt_index FROM midnight.midnight_evt_marketcreated + UNION ALL SELECT id_, UINT256 '3', market_settlementfeecbp3, evt_block_number, evt_index FROM midnight.midnight_evt_marketcreated + UNION ALL SELECT id_, UINT256 '4', market_settlementfeecbp4, evt_block_number, evt_index FROM midnight.midnight_evt_marketcreated + UNION ALL SELECT id_, UINT256 '5', market_settlementfeecbp5, evt_block_number, evt_index FROM midnight.midnight_evt_marketcreated + UNION ALL SELECT id_, UINT256 '6', market_settlementfeecbp6, evt_block_number, evt_index FROM midnight.midnight_evt_marketcreated + -- Overrides (newsettlementfee is raw; divide by CBP to get stored uint16 value) + UNION ALL + SELECT id_, index, newsettlementfee / 1000000000000, evt_block_number, evt_index + FROM midnight.midnight_evt_setmarketsettlementfee +), + +latest_settlement_fees AS ( + SELECT id_, index, + MAX_BY(fee, ROW(evt_block_number, evt_index)) AS fee + FROM settlement_fee_stream GROUP BY id_, index +), + +settlement_fees_pivoted AS ( + SELECT + id_, + MAX(CASE WHEN index = UINT256 '0' THEN fee END) AS settlement_fee_cbp_0, + MAX(CASE WHEN index = UINT256 '1' THEN fee END) AS settlement_fee_cbp_1, + MAX(CASE WHEN index = UINT256 '2' THEN fee END) AS settlement_fee_cbp_2, + MAX(CASE WHEN index = UINT256 '3' THEN fee END) AS settlement_fee_cbp_3, + MAX(CASE WHEN index = UINT256 '4' THEN fee END) AS settlement_fee_cbp_4, + MAX(CASE WHEN index = UINT256 '5' THEN fee END) AS settlement_fee_cbp_5, + MAX(CASE WHEN index = UINT256 '6' THEN fee END) AS settlement_fee_cbp_6 + FROM latest_settlement_fees GROUP BY id_ +), + +-- continuousFee: last value per market +continuous_fee_stream AS ( + SELECT id_, market_continuousfee AS fee, evt_block_number, evt_index + FROM midnight.midnight_evt_marketcreated + UNION ALL + SELECT id_, newcontinuousfee, evt_block_number, evt_index + FROM midnight.midnight_evt_setmarketcontinuousfee +), + +continuous_fee AS ( + SELECT id_, MAX_BY(fee, ROW(evt_block_number, evt_index)) AS continuous_fee + FROM continuous_fee_stream GROUP BY id_ +), + +-- ── tickSpacing: last value per market ──────────────────────────────────────── +tick_spacing_stream AS ( + SELECT id_, UINT256 '4' AS tick_spacing, evt_block_number, evt_index -- DEFAULT_TICK_SPACING = 4 + FROM midnight.midnight_evt_marketcreated + UNION ALL + SELECT id_, newtickspacing, evt_block_number, evt_index + FROM midnight.midnight_evt_setmarkettickspacing +), + +tick_spacing AS ( + SELECT id_, MAX_BY(tick_spacing, ROW(evt_block_number, evt_index)) AS tick_spacing + FROM tick_spacing_stream GROUP BY id_ +) + +-- ── Final join ──────────────────────────────────────────────────────────────── + +SELECT + m.id_, + COALESCE(tu.total_units, UINT256 '0') AS total_units, + COALESCE(lf.loss_factor, UINT256 '0') AS loss_factor, + COALESCE(w.withdrawable, UINT256 '0') AS withdrawable, + COALESCE(cfc.continuous_fee_credit, UINT256 '0') AS continuous_fee_credit, + tf.settlement_fee_cbp_0, + tf.settlement_fee_cbp_1, + tf.settlement_fee_cbp_2, + tf.settlement_fee_cbp_3, + tf.settlement_fee_cbp_4, + tf.settlement_fee_cbp_5, + tf.settlement_fee_cbp_6, + cf.continuous_fee, + ts.tick_spacing +FROM midnight.midnight_evt_marketcreated m +LEFT JOIN total_units tu ON tu.id_ = m.id_ +LEFT JOIN loss_factor lf ON lf.id_ = m.id_ +LEFT JOIN withdrawable w ON w.id_ = m.id_ +LEFT JOIN continuous_fee_credit cfc ON cfc.id_ = m.id_ +LEFT JOIN settlement_fees_pivoted tf ON tf.id_ = m.id_ +LEFT JOIN continuous_fee cf ON cf.id_ = m.id_ +LEFT JOIN tick_spacing ts ON ts.id_ = m.id_ diff --git a/sql/position.sql b/sql/position.sql new file mode 100644 index 00000000..bab56db6 --- /dev/null +++ b/sql/position.sql @@ -0,0 +1,145 @@ +-- Reconstructs position[id][user] from events. +-- Fields: credit, debt, pending_fee, last_loss_factor, last_accrual +-- Collateral amounts → see position_collateral.sql +-- Platform: Dune Analytics (Trino SQL, native uint256) + +WITH + +-- ── Credit ──────────────────────────────────────────────────────────────────── +-- buyer in Take: +buyerCreditIncrease +-- seller in Take: -sellerCreditDecrease +-- Withdraw: -units +-- UpdatePosition: -creditDecrease (slashing + continuous-fee accrual) + +credit_deltas AS ( + SELECT id_, IF(offerisbuy, maker, taker) AS user, buyercreditincrease AS add_, UINT256 '0' AS sub_ + FROM midnight.midnight_evt_take + UNION ALL + SELECT id_, IF(offerisbuy, taker, maker) AS user, UINT256 '0' AS add_, sellercreditdecrease AS sub_ + FROM midnight.midnight_evt_take + UNION ALL + SELECT id_, onbehalf AS user, UINT256 '0', units + FROM midnight.midnight_evt_withdraw + UNION ALL + SELECT id_, user, UINT256 '0', creditdecrease + FROM midnight.midnight_evt_updateposition +), + +-- ── Debt ────────────────────────────────────────────────────────────────────── +-- seller in Take: +(units - sellerCreditDecrease) [new debt, after exhausting credit] +-- buyer in Take: -(units - buyerCreditIncrease) [repays existing debt] +-- Repay: -units +-- Liquidate: -(badDebt + repaidUnits) + +debt_deltas AS ( + SELECT id_, IF(offerisbuy, taker, maker) AS user, units - sellercreditdecrease AS add_, UINT256 '0' AS sub_ + FROM midnight.midnight_evt_take + UNION ALL + SELECT id_, IF(offerisbuy, maker, taker) AS user, UINT256 '0' AS add_, units - buyercreditincrease AS sub_ + FROM midnight.midnight_evt_take + UNION ALL + SELECT id_, onbehalf AS user, UINT256 '0', units + FROM midnight.midnight_evt_repay + UNION ALL + SELECT id_, borrower AS user, UINT256 '0', baddebt + repaidunits + FROM midnight.midnight_evt_liquidate +), + +-- ── Pending fee ─────────────────────────────────────────────────────────────── +-- buyer in Take: +buyerPendingFeeIncrease (proportional to new credit × TTM × continuousFee) +-- seller in Take: -sellerPendingFeeDecrease (proportional credit sold) +-- Withdraw: -pendingFeeDecrease (proportional to units withdrawn) +-- UpdatePosition: -pendingFeeDecrease (continuousFee accrued to market + slashing) + +pending_fee_deltas AS ( + SELECT id_, IF(offerisbuy, maker, taker) AS user, buyerpendingfeeincrease AS add_, UINT256 '0' AS sub_ + FROM midnight.midnight_evt_take + UNION ALL + SELECT id_, IF(offerisbuy, taker, maker) AS user, UINT256 '0' AS add_, sellerpendingfeedecrease AS sub_ + FROM midnight.midnight_evt_take + UNION ALL + SELECT id_, onbehalf AS user, UINT256 '0', pendingfeedecrease + FROM midnight.midnight_evt_withdraw + UNION ALL + SELECT id_, user, UINT256 '0', pendingfeedecrease + FROM midnight.midnight_evt_updateposition +), + +-- ── Aggregate the three running fields ─────────────────────────────────────── + +credit_per_user AS ( + SELECT id_, user, SUM(add_) - SUM(sub_) AS credit + FROM credit_deltas GROUP BY id_, user +), + +debt_per_user AS ( + SELECT id_, user, SUM(add_) - SUM(sub_) AS debt + FROM debt_deltas GROUP BY id_, user +), + +pending_fee_per_user AS ( + SELECT id_, user, SUM(add_) - SUM(sub_) AS pending_fee + FROM pending_fee_deltas GROUP BY id_, user +), + +-- ── lastAccrual: block_time of the most recent UpdatePosition per (id, user) ── + +last_update_pos AS ( + SELECT id_, user, evt_block_number, evt_index, + CAST(to_unixtime(evt_block_time) AS uint256) AS last_accrual + FROM ( + SELECT id_, user, evt_block_number, evt_index, evt_block_time, + ROW_NUMBER() OVER ( + PARTITION BY id_, user + ORDER BY evt_block_number DESC, evt_index DESC + ) AS rn + FROM midnight.midnight_evt_updateposition + ) + WHERE rn = 1 +), + +-- ── lastLossFactor: market lossFactor at the time of the last UpdatePosition ── +-- lossFactor only changes on Liquidate (via latestLossFactor field). +-- For each (id, user), find the most recent Liquidate for that market +-- that was emitted before the last UpdatePosition of that user. + +last_loss_factor AS ( + SELECT + lu.id_, + lu.user, + COALESCE( + MAX_BY(liq.latestlossfactor, ROW(liq.evt_block_number, liq.evt_index)), + UINT256 '0' + ) AS last_loss_factor + FROM last_update_pos lu + LEFT JOIN midnight.midnight_evt_liquidate liq + ON liq.id_ = lu.id_ + AND ( liq.evt_block_number < lu.evt_block_number + OR (liq.evt_block_number = lu.evt_block_number AND liq.evt_index < lu.evt_index)) + GROUP BY lu.id_, lu.user +), + +-- ── All (id, user) pairs that ever had any activity ─────────────────────────── + +all_users AS ( + SELECT DISTINCT id_, user FROM credit_deltas + UNION + SELECT DISTINCT id_, user FROM debt_deltas + UNION + SELECT DISTINCT id_, user FROM pending_fee_deltas +) + +SELECT + u.id_, + u.user, + COALESCE(c.credit, UINT256 '0') AS credit, + COALESCE(d.debt, UINT256 '0') AS debt, + COALESCE(p.pending_fee, UINT256 '0') AS pending_fee, + COALESCE(llf.last_loss_factor,UINT256 '0') AS last_loss_factor, + COALESCE(lu.last_accrual, UINT256 '0') AS last_accrual +FROM all_users u +LEFT JOIN credit_per_user c ON c.id_ = u.id_ AND c.user = u.user +LEFT JOIN debt_per_user d ON d.id_ = u.id_ AND d.user = u.user +LEFT JOIN pending_fee_per_user p ON p.id_ = u.id_ AND p.user = u.user +LEFT JOIN last_loss_factor llf ON llf.id_ = u.id_ AND llf.user = u.user +LEFT JOIN last_update_pos lu ON lu.id_ = u.id_ AND lu.user = u.user diff --git a/sql/position_collateral.sql b/sql/position_collateral.sql new file mode 100644 index 00000000..bd24dc6f --- /dev/null +++ b/sql/position_collateral.sql @@ -0,0 +1,84 @@ +-- Reconstructs position[id][user].collateral[index] from events. +-- Returns one row per (id, user, collateral_index) with the current collateral amount. +-- collateralBitmap is derived: bit i is set iff amount > 0. +-- Platform: Dune Analytics (Trino SQL, native uint256) +-- +-- Assumption: midnight.midnight_evt_marketcreated has a column +-- market_collateralparams of type ARRAY(ROW(token VARBINARY, lltv UINT256, maxlif UINT256, oracle VARBINARY)) +-- allowing UNNEST with ORDINALITY to recover collateral_index from token address. + +WITH + +-- Build a (market_id, token, collateral_index) lookup from MarketCreated. +-- ORDINALITY starts at 1; subtract 1 to match the 0-based Solidity array index. +collateral_index_lookup AS ( + SELECT + id_, + cp.token AS collateral_token, + CAST(idx - 1 AS uint256) AS collateral_index + FROM midnight.midnight_evt_marketcreated + CROSS JOIN UNNEST(market_collateralparams) WITH ORDINALITY AS t(cp, idx) +), + +-- ── Collateral deltas ───────────────────────────────────────────────────────── +-- SupplyCollateral: +assets +-- WithdrawCollateral: -assets +-- Liquidate: -seizedAssets (for the liquidated collateral only) + +collateral_deltas AS ( + SELECT + s.id_, + s.onbehalf AS user, + cil.collateral_index, + s.assets AS add_, + UINT256 '0' AS sub_ + FROM midnight.midnight_evt_supplycollateral s + JOIN collateral_index_lookup cil + ON cil.id_ = s.id_ AND cil.collateral_token = s.collateral + + UNION ALL + + SELECT + w.id_, + w.onbehalf AS user, + cil.collateral_index, + UINT256 '0' AS add_, + w.assets AS sub_ + FROM midnight.midnight_evt_withdrawcollateral w + JOIN collateral_index_lookup cil + ON cil.id_ = w.id_ AND cil.collateral_token = w.collateral + + UNION ALL + + SELECT + l.id_, + l.borrower AS user, + cil.collateral_index, + UINT256 '0' AS add_, + l.seizedassets AS sub_ + FROM midnight.midnight_evt_liquidate l + JOIN collateral_index_lookup cil + ON cil.id_ = l.id_ AND cil.collateral_token = l.collateral +), + +aggregated AS ( + SELECT + id_, + user, + collateral_index, + SUM(add_) - SUM(sub_) AS amount + FROM collateral_deltas + GROUP BY id_, user, collateral_index +) + +SELECT + id_, + user, + collateral_index, + amount, + -- collateralBitmap: bit i is set iff amount > 0 + -- Reconstruct as a bitmask: SUM over all indices with set bits + -- (computed separately per user in the outer query if needed) + amount > UINT256 '0' AS bit_is_set +FROM aggregated +WHERE amount > UINT256 '0' diff --git a/sql/roles.sql b/sql/roles.sql new file mode 100644 index 00000000..a68ac742 --- /dev/null +++ b/sql/roles.sql @@ -0,0 +1,32 @@ +-- Reconstructs the four admin role addresses from events. +-- roleSetter: set in Constructor, then overridden by SetRoleSetter +-- feeSetter: set by SetFeeSetter (initial = address(0)) +-- feeClaimer: set by SetFeeClaimer (initial = address(0)) +-- tickSpacingSetter: set by SetTickSpacingSetter (initial = address(0)) +-- Platform: Dune Analytics (Trino SQL) + +WITH + +role_setter_stream AS ( + SELECT rolesetter AS addr, evt_block_number, evt_index FROM midnight.midnight_evt_constructor + UNION ALL + SELECT rolesetter, evt_block_number, evt_index FROM midnight.midnight_evt_setrolesetter +), + +fee_setter_stream AS ( + SELECT feesetter AS addr, evt_block_number, evt_index FROM midnight.midnight_evt_setfeesetter +), + +fee_claimer_stream AS ( + SELECT feeclaimer AS addr, evt_block_number, evt_index FROM midnight.midnight_evt_setfeeclaimer +), + +tick_spacing_setter_stream AS ( + SELECT tickspacingsetter AS addr, evt_block_number, evt_index FROM midnight.midnight_evt_settickspacingsetter +) + +SELECT + (SELECT MAX_BY(addr, ROW(evt_block_number, evt_index)) FROM role_setter_stream) AS role_setter, + (SELECT MAX_BY(addr, ROW(evt_block_number, evt_index)) FROM fee_setter_stream) AS fee_setter, + (SELECT MAX_BY(addr, ROW(evt_block_number, evt_index)) FROM fee_claimer_stream) AS fee_claimer, + (SELECT MAX_BY(addr, ROW(evt_block_number, evt_index)) FROM tick_spacing_setter_stream) AS tick_spacing_setter diff --git a/sql/test/requirements.txt b/sql/test/requirements.txt new file mode 100644 index 00000000..009ec9ca --- /dev/null +++ b/sql/test/requirements.txt @@ -0,0 +1,2 @@ +duckdb==1.5.3 +pandas==3.0.3 diff --git a/sql/test/run.sh b/sql/test/run.sh new file mode 100755 index 00000000..f0799d47 --- /dev/null +++ b/sql/test/run.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Runs the Forge scenario test then verifies all SQL queries with DuckDB. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$REPO_ROOT" + +echo "=== Generating test scenario ===" +forge test --match-test testGenerateScenario -vv + +echo "" +echo "=== Verifying SQL queries ===" +uv run --with-requirements sql/test/requirements.txt python3 sql/test/verify.py diff --git a/sql/test/verify.py b/sql/test/verify.py new file mode 100644 index 00000000..a492ae52 --- /dev/null +++ b/sql/test/verify.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +""" +Verifies Midnight SQL queries against exported events and expected state. + +Usage (from repo root): + python3 sql/test/verify.py + +Prerequisites: + pip install duckdb pandas + forge test --match-test testGenerateScenario -vv (run first) +""" + +import json +import os +import re +import sys + +import duckdb +import pandas as pd + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +EVENTS_DIR = os.path.join(REPO_ROOT, "sql", "test", "events") +EXPECTED_FILE = os.path.join(REPO_ROOT, "sql", "test", "expected_state.json") +SQL_DIR = os.path.join(REPO_ROOT, "sql") + + +# ── SQL adaptation (Dune/Trino → DuckDB) ───────────────────────────────────── + +def adapt_sql(sql: str) -> str: + sql = sql.replace("midnight.midnight_evt_", "midnight_evt_") + # DuckDB requires WITH RECURSIVE for recursive CTEs (Trino infers it automatically) + sql = re.sub(r"\bWITH\b", "WITH RECURSIVE", sql, count=1) + # to_unixtime must come before the CAST regex (the CAST may wrap it) + sql = sql.replace("to_unixtime(evt_block_time)", "evt_block_time") + sql = re.sub(r"UINT256 '(\d+)'", r"\1", sql) + sql = re.sub(r"BIGINT '(\d+)'", r"\1", sql) + sql = re.sub( + r"(?i)\bCAST\(([^)]+)\s+AS\s+uint256\)", + lambda m: f"CAST({m.group(1)} AS BIGINT)", + sql, + ) + sql = re.sub( + r"MAX_BY\(([\s\S]+?),\s*ROW\(([\s\S]+?),\s*([\s\S]+?)\)\)", + lambda m: ( + f"arg_max({m.group(1).strip()}, " + f"CAST({m.group(2).strip()} AS BIGINT) * 1000000000 + " + f"CAST({m.group(3).strip()} AS BIGINT))" + ), + sql, + ) + return sql + + +# ── Event table schemas ─────────────────────────────────────────────────────── +# (table_name -> (filename, int_cols, bool_cols)) + +# (table_name -> (filename, int_cols, bool_cols, str_cols)) +# str_cols is required to correctly build empty tables when the JSON array is []. +EVENT_SCHEMAS: dict[str, tuple[str, list[str], list[str], list[str]]] = { + "midnight_evt_take": ( + "take.json", + ["buyerassets", "sellerassets", "units", "consumed", + "buyerpendingfeeincrease", "sellerpendingfeedecrease", + "buyercreditincrease", "sellercreditdecrease", + "evt_block_number", "evt_index"], + ["offerisbuy"], + ["id_", "maker", "taker", "group"], + ), + "midnight_evt_updateposition": ( + "updateposition.json", + ["creditdecrease", "pendingfeedecrease", "accruedfee", + "evt_block_number", "evt_index", "evt_block_time"], + [], + ["id_", "user"], + ), + "midnight_evt_withdraw": ( + "withdraw.json", + ["units", "pendingfeedecrease", "evt_block_number", "evt_index"], + [], + ["id_", "onbehalf", "receiver"], + ), + "midnight_evt_repay": ( + "repay.json", + ["units", "evt_block_number", "evt_index"], + [], + ["id_", "onbehalf"], + ), + "midnight_evt_supplycollateral": ( + "supplycollateral.json", + ["assets", "evt_block_number", "evt_index"], + [], + ["id_", "collateral", "onbehalf"], + ), + "midnight_evt_withdrawcollateral": ( + "withdrawcollateral.json", + ["assets", "evt_block_number", "evt_index"], + [], + ["id_", "collateral", "onbehalf"], + ), + "midnight_evt_marketcreated": ( + "marketcreated.json", + ["market_maturity", + "market_settlementfeecbp0", "market_settlementfeecbp1", "market_settlementfeecbp2", + "market_settlementfeecbp3", "market_settlementfeecbp4", "market_settlementfeecbp5", + "market_settlementfeecbp6", "market_continuousfee", + "evt_block_number", "evt_index"], + [], + ["id_", "market_loantoken"], + ), + "midnight_evt_setconsumed": ( + "setconsumed.json", + ["amount", "evt_block_number", "evt_index"], + [], + ["onbehalf", "group"], + ), + "midnight_evt_setisauthorized": ( + "setisauthorized.json", + ["evt_block_number", "evt_index"], + ["newisauthorized"], + ["onbehalf", "authorized"], + ), + "midnight_evt_setdefaultsettlementfee": ( + "setdefaultsettlementfee.json", + ["index", "newsettlementfee", "evt_block_number", "evt_index"], + [], + ["loantoken"], + ), + "midnight_evt_setmarketsettlementfee": ( + "setmarketsettlementfee.json", + ["index", "newsettlementfee", "evt_block_number", "evt_index"], + [], + ["id_"], + ), + "midnight_evt_setmarketcontinuousfee": ( + "setmarketcontinuousfee.json", + ["newcontinuousfee", "evt_block_number", "evt_index"], + [], + ["id_"], + ), + "midnight_evt_claimsettlementfee": ( + "claimsettlementfee.json", + ["amount", "evt_block_number", "evt_index"], + [], + ["token"], + ), + "midnight_evt_claimcontinuousfee": ( + "claimcontinuousfee.json", + ["amount", "evt_block_number", "evt_index"], + [], + ["id_"], + ), + "midnight_evt_constructor": ( + "constructor.json", + ["evt_block_number", "evt_index"], + [], + ["rolesetter"], + ), + "midnight_evt_setfeesetter": ( + "setfeesetter.json", + ["evt_block_number", "evt_index"], + [], + ["feesetter"], + ), + "midnight_evt_setfeeclaimer": ( + "setfeeclaimer.json", + ["evt_block_number", "evt_index"], + [], + ["feeclaimer"], + ), + "midnight_evt_settickspacingsetter": ( + "settickspacingsetter.json", + ["evt_block_number", "evt_index"], + [], + ["tickspacingsetter"], + ), + "midnight_evt_setdefaultcontinuousfee": ( + "setdefaultcontinuousfee.json", + ["newcontinuousfee", "evt_block_number", "evt_index"], + [], + ["loantoken"], + ), + "midnight_evt_liquidate": ( + "liquidate.json", + ["seizedassets", "repaidunits", "baddebt", + "evt_block_number", "evt_index"], + ["postmaturitymode"], + ["id_", "collateral", "borrower"], + ["latestlossfactor", "latestcontinuousfeecredit"], # uint128 — stored as UHUGEINT to hold values up to MAX_U128 + ), +} + +# Tables referenced by some SQL files but not emitted in our test scenario +EMPTY_TABLE_SCHEMAS: dict[str, str] = { + "midnight_evt_setrolesetter": ( + "rolesetter VARCHAR, evt_block_number BIGINT, evt_index BIGINT" + ), + "midnight_evt_setmarkettickspacing": ( + "id_ VARCHAR, newtickspacing BIGINT, evt_block_number BIGINT, evt_index BIGINT" + ), +} + + +def load_events(conn: duckdb.DuckDBPyConnection) -> None: + for table_name, schema_tuple in EVENT_SCHEMAS.items(): + if len(schema_tuple) == 5: + filename, int_cols, bool_cols, str_cols, uhugeint_cols = schema_tuple + else: + filename, int_cols, bool_cols, str_cols = schema_tuple + uhugeint_cols = [] + + filepath = os.path.join(EVENTS_DIR, filename) + with open(filepath) as f: + rows = json.load(f) + + all_cols = str_cols + int_cols + uhugeint_cols + bool_cols + + if not rows or uhugeint_cols: + # Use explicit DDL for empty tables or tables with UHUGEINT columns. + # Pandas type inference cannot reliably represent uint128 values. + col_defs = ", ".join( + f"{c} UHUGEINT" if c in uhugeint_cols else + f"{c} BIGINT" if c in int_cols else + f"{c} BOOLEAN" if c in bool_cols else + f"{c} VARCHAR" + for c in (all_cols or ["dummy"]) + ) + conn.execute(f"CREATE TABLE {table_name} ({col_defs})") + if rows: + placeholders = ", ".join("?" * len(all_cols)) + data = [] + for row in rows: + values = [] + for c in all_cols: + v = row.get(c) + if c in uhugeint_cols or c in int_cols: + values.append(int(v) if v is not None else 0) + elif c in bool_cols: + values.append(bool(v) if v is not None else False) + else: + values.append(str(v).lower() if v is not None else "") + data.append(values) + conn.executemany(f"INSERT INTO {table_name} VALUES ({placeholders})", data) + continue + + # Pandas path — no UHUGEINT columns, table has data + converted = [] + for row in rows: + new_row: dict = {} + for k, v in row.items(): + if k in int_cols: + new_row[k] = int(v) if v is not None else 0 + elif k in bool_cols: + new_row[k] = bool(v) + else: + new_row[k] = str(v).lower() if v is not None else "" + converted.append(new_row) + + df = pd.DataFrame(converted) + conn.register(f"_df_{table_name}", df) + conn.execute(f"CREATE TABLE {table_name} AS SELECT * FROM _df_{table_name}") + + for table_name, col_defs in EMPTY_TABLE_SCHEMAS.items(): + conn.execute(f"CREATE TABLE {table_name} ({col_defs})") + + +def run_sql(conn: duckdb.DuckDBPyConnection, sql_path: str) -> pd.DataFrame: + with open(sql_path) as f: + raw = f.read() + adapted = adapt_sql(raw) + try: + result = conn.execute(adapted) + columns = [desc[0] for desc in result.description] + rows = result.fetchall() + return pd.DataFrame(rows, columns=columns) + except Exception as exc: + print(f"ERROR in {os.path.basename(sql_path)}:") + print(exc) + raise + + +def verify_all() -> None: + with open(EXPECTED_FILE) as f: + exp = json.load(f) + + conn = duckdb.connect() + load_events(conn) + + failures: list[str] = [] + + def fail(msg: str) -> None: + failures.append(f" {msg}") + + def check_int(label: str, actual, expected_val) -> None: + try: + a, e = int(actual), int(expected_val) + if a != e: + fail(f"{label}: got {a}, expected {e}") + except Exception as exc: + fail(f"{label}: comparison error – {exc}") + + def check_str(label: str, actual, expected_val) -> None: + a = str(actual).lower().strip() + e = str(expected_val).lower().strip() + if a != e: + fail(f"{label}: got {a!r}, expected {e!r}") + + # ── position.sql ───────────────────────────────────────────────────────── + print("Verifying position.sql …") + df_pos = run_sql(conn, os.path.join(SQL_DIR, "position.sql")) + df_pos["user"] = df_pos["user"].str.lower() + for user_key, addr_key in [ + ("position_lender", "lender_addr"), + ("position_borrower", "borrower_addr"), + ("position_other_borrower", "other_borrower_addr"), + ]: + addr = exp[addr_key].lower() + row = df_pos[df_pos["user"] == addr] + if row.empty: + fail(f"position.sql: no row for {user_key}") + continue + r = row.iloc[0] + pos = exp[user_key] + for field in ["credit", "debt", "pending_fee", "last_loss_factor", "last_accrual"]: + check_int(f"{user_key}.{field}", r[field], pos[field]) + + # ── market_state.sql ────────────────────────────────────────────────────── + print("Verifying market_state.sql …") + df_ms = run_sql(conn, os.path.join(SQL_DIR, "market_state.sql")) + if df_ms.empty: + fail("market_state.sql: no rows") + else: + r = df_ms.iloc[0] + ms = exp["market_state"] + for field in [ + "total_units", "loss_factor", "withdrawable", "continuous_fee_credit", + "settlement_fee_cbp_0", "settlement_fee_cbp_1", "settlement_fee_cbp_2", + "settlement_fee_cbp_3", "settlement_fee_cbp_4", "settlement_fee_cbp_5", + "settlement_fee_cbp_6", "continuous_fee", "tick_spacing", + ]: + check_int(f"market_state.{field}", r[field], ms[field]) + + # ── consumed.sql ────────────────────────────────────────────────────────── + print("Verifying consumed.sql …") + df_con = run_sql(conn, os.path.join(SQL_DIR, "consumed.sql")) + df_con["user"] = df_con["user"].str.lower() + df_con["group"] = df_con["group"].str.lower() + borrower_addr = exp["borrower_addr"].lower() + group_a = exp["group_a"].lower() + row = df_con[ + (df_con["user"] == borrower_addr) & (df_con["group"] == group_a) + ] + if row.empty: + fail("consumed.sql: no row for borrower+GROUP_A") + else: + check_int( + "consumed.borrower_group_a", + row.iloc[0]["amount"], + exp["consumed_borrower_group_a"], + ) + + # ── is_authorized.sql ───────────────────────────────────────────────────── + print("Verifying is_authorized.sql …") + df_auth = run_sql(conn, os.path.join(SQL_DIR, "is_authorized.sql")) + df_auth["authorizer"] = df_auth["authorizer"].str.lower() + df_auth["authorized"] = df_auth["authorized"].str.lower() + lender_addr = exp["lender_addr"].lower() + other_lender_addr = exp["other_lender_addr"].lower() + row = df_auth[ + (df_auth["authorizer"] == lender_addr) + & (df_auth["authorized"] == other_lender_addr) + ] + exp_val = exp["is_authorized_lender_other_lender"] + if row.empty: + if exp_val: + fail("is_authorized.sql: no row for lender→otherLender (expected true)") + else: + check_str( + "is_authorized.lender_other_lender", + row.iloc[0]["is_authorized"], + str(exp_val).lower(), + ) + + # ── claimable_settlement_fee.sql ───────────────────────────────────────────── + print("Verifying claimable_settlement_fee.sql …") + df_ctf = run_sql(conn, os.path.join(SQL_DIR, "claimable_settlement_fee.sql")) + loan_token = exp["loan_token_addr"].lower() + if not df_ctf.empty: + df_ctf["token"] = df_ctf["token"].str.lower() + row = df_ctf[df_ctf["token"] == loan_token] if not df_ctf.empty else df_ctf + if row.empty: + check_int("claimable_settlement_fee", 0, exp["claimable_settlement_fee"]) + else: + check_int( + "claimable_settlement_fee", + row.iloc[0]["claimable_settlement_fee"], + exp["claimable_settlement_fee"], + ) + + # ── default_settlement_fee.sql ─────────────────────────────────────────────── + print("Verifying default_settlement_fee.sql …") + df_dtf = run_sql(conn, os.path.join(SQL_DIR, "default_settlement_fee.sql")) + df_dtf["loan_token"] = df_dtf["loan_token"].str.lower() + row = df_dtf[(df_dtf["loan_token"] == loan_token) & (df_dtf["index"] == 3)] + if row.empty: + check_int("default_settlement_fee_cbp_3", 0, exp["default_settlement_fee_cbp_3"]) + else: + check_int( + "default_settlement_fee_cbp_3", + row.iloc[0]["fee_cbp"], + exp["default_settlement_fee_cbp_3"], + ) + + # ── default_continuous_fee.sql ──────────────────────────────────────────── + print("Verifying default_continuous_fee.sql …") + df_dcf = run_sql(conn, os.path.join(SQL_DIR, "default_continuous_fee.sql")) + df_dcf["loan_token"] = df_dcf["loan_token"].str.lower() + row = df_dcf[df_dcf["loan_token"] == loan_token] + if row.empty: + check_int("default_continuous_fee", 0, exp["default_continuous_fee"]) + else: + check_int("default_continuous_fee", row.iloc[0]["fee"], exp["default_continuous_fee"]) + + # ── roles.sql ───────────────────────────────────────────────────────────── + print("Verifying roles.sql …") + df_roles = run_sql(conn, os.path.join(SQL_DIR, "roles.sql")) + if df_roles.empty: + fail("roles.sql: no rows") + else: + r = df_roles.iloc[0] + check_str("role_setter", r["role_setter"], exp["role_setter"]) + check_str("fee_setter", r["fee_setter"], exp["fee_setter"]) + check_str("fee_claimer", r["fee_claimer"], exp["fee_claimer"]) + check_str("tick_spacing_setter", r["tick_spacing_setter"], exp["tick_spacing_setter"]) + + # ── Summary ─────────────────────────────────────────────────────────────── + if failures: + print(f"\nFAILURES ({len(failures)}):") + for msg in failures: + print(msg) + sys.exit(1) + else: + print("\nAll SQL queries match expected state.") + + +if __name__ == "__main__": + verify_all() diff --git a/test/SqlScenarioTest.sol b/test/SqlScenarioTest.sol new file mode 100644 index 00000000..3df05d48 --- /dev/null +++ b/test/SqlScenarioTest.sol @@ -0,0 +1,1059 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (c) 2025 Morpho Association +pragma solidity ^0.8.0; + +import {Test, Vm} from "../lib/forge-std/src/Test.sol"; +import {BaseTest, LLTV_5} from "./BaseTest.sol"; +import {MAX_TICK} from "../src/libraries/TickLib.sol"; +import {WAD, ORACLE_PRICE_SCALE, LIQUIDATION_CURSOR_LOW, maxLif as _maxLif} from "../src/libraries/ConstantsLib.sol"; +import {Market, Offer, CollateralParams} from "../src/interfaces/IMidnight.sol"; +import {Signature} from "../src/ratifiers/interfaces/IEcrecoverRatifier.sol"; +import {HashLib} from "../src/ratifiers/libraries/HashLib.sol"; +import {UtilsLib} from "../src/libraries/UtilsLib.sol"; + +/// @dev Generates a randomised scenario covering all Midnight event types, including liquidation. +/// Writes event tables (JSON arrays) and expected on-chain state to sql/test/events/. +/// +/// Block layout +/// ───────────────────────────────────────────────────────────────────────────── +/// Block 1: setUp events (Constructor, SetFeeSetter, SetTickSpacingSetter, +/// SetIsAuthorized ×4, SetFeeClaimer) +/// Block 2: SetDefaultSettlementFee (index 3, 100 CBP) +/// Block 3: SetDefaultContinuousFee +/// Block 4: SupplyCollateral → MarketCreated + SupplyCollateral (borrower) +/// Block 5: Take #1 (lender buy offer, borrower takes) +/// Block 6: SupplyCollateral (otherBorrower, liquidation victim) +/// Block 7: Take #2 (otherLender buy offer, otherBorrower takes) +/// Block 8: Repay partial (borrower) +/// Block 9: WithdrawCollateral (borrower) +/// Block 10: UpdatePosition (lender, explicit) +/// Block 11: SetMarketSettlementFee (index 3 → 50 CBP) +/// Block 12: SetMarketContinuousFee +/// Block 13: UpdatePosition (large time-warp → continuousFee accrues) +/// Block 14: SetIsAuthorized (lender → otherLender) +/// Block 15: SetConsumed (borrower, cancel GROUP_A) +/// Block 16: Liquidate (oracle drop → liquidate otherBorrower → restore price) +/// Block 17: Withdraw (lender) +/// Block 18: ClaimSettlementFee +/// Block 19: ClaimContinuousFee +/// +/// Run with: forge test --match-test testGenerateScenario -vv +/// Verify: python3 sql/test/verify.py +contract SqlScenarioTest is BaseTest { + using UtilsLib for uint256; + + // ── Scenario parameters + // ────────────────────────────────────────────────── + // Settlement fees must be multiples of CBP (1e12). + uint256 internal constant DEFAULT_SETTLEMENT_FEE_RAW = 100 * 1_000_000_000_000; + uint256 internal constant MARKET_SETTLEMENT_FEE_RAW = 50 * 1_000_000_000_000; + bytes32 internal constant GROUP_A = keccak256("group-a"); + bytes32 internal constant GROUP_B = keccak256("group-b"); + + // ── Market + // ─────────────────────────────────────────────────────────────── + Market internal market; + bytes32 internal id; + + // ── Event type selectors + // ───────────────────────────────────────────────── + bytes32 internal constant SEL_TAKE = keccak256( + "Take(address,bytes32,uint256,address,address,bool,bytes32,uint256,uint256,uint256,uint256,uint256,uint256,uint256,address,address)" + ); + bytes32 internal constant SEL_UPDATE_POS = keccak256("UpdatePosition(bytes32,address,uint256,uint256,uint256)"); + bytes32 internal constant SEL_WITHDRAW = keccak256("Withdraw(address,bytes32,uint256,address,address,uint256)"); + bytes32 internal constant SEL_REPAY = keccak256("Repay(address,bytes32,uint256,address,address)"); + bytes32 internal constant SEL_SUPPLY_COL = keccak256("SupplyCollateral(address,bytes32,address,uint256,address)"); + bytes32 internal constant SEL_WITHDRAW_COL = + keccak256("WithdrawCollateral(address,bytes32,address,uint256,address,address)"); + bytes32 internal constant SEL_MARKET_CREATED = keccak256( + "MarketCreated((uint256,address,address,(address,uint256,uint256,address)[],uint256,uint256,address,address),bytes32)" + ); + bytes32 internal constant SEL_SET_CONSUMED = keccak256("SetConsumed(address,bytes32,uint256,address)"); + bytes32 internal constant SEL_SET_IS_AUTH = keccak256("SetIsAuthorized(address,address,bool,address)"); + bytes32 internal constant SEL_SET_DEF_TF = keccak256("SetDefaultSettlementFee(address,uint256,uint256)"); + bytes32 internal constant SEL_SET_MKT_TF = keccak256("SetMarketSettlementFee(bytes32,uint256,uint256)"); + bytes32 internal constant SEL_SET_MKT_CF = keccak256("SetMarketContinuousFee(bytes32,uint256)"); + bytes32 internal constant SEL_CLAIM_TF = keccak256("ClaimSettlementFee(address,address,uint256,address)"); + bytes32 internal constant SEL_CLAIM_CF = keccak256("ClaimContinuousFee(address,bytes32,uint256,address)"); + bytes32 internal constant SEL_CONSTRUCTOR = keccak256("Constructor(address)"); + bytes32 internal constant SEL_SET_FEE_SETTER = keccak256("SetFeeSetter(address)"); + bytes32 internal constant SEL_SET_FEE_CLAIMER = keccak256("SetFeeClaimer(address)"); + bytes32 internal constant SEL_SET_TS_SETTER = keccak256("SetTickSpacingSetter(address)"); + bytes32 internal constant SEL_SET_DEF_CF = keccak256("SetDefaultContinuousFee(address,uint256)"); + bytes32 internal constant SEL_LIQUIDATE = keccak256( + "Liquidate(address,bytes32,address,uint256,uint256,address,bool,address,address,uint256,uint256,uint256)" + ); + + // ── Per-event JSON accumulators + // ─────────────────────────────────────────── + string internal jTake; + string internal jUpdatePos; + string internal jWithdraw; + string internal jRepay; + string internal jSupplyCol; + string internal jWithdrawCol; + string internal jMarketCreated; + string internal jSetConsumed; + string internal jSetIsAuth; + string internal jSetDefTf; + string internal jSetMktTf; + string internal jSetMktCf; + string internal jClaimTf; + string internal jClaimCf; + string internal jConstructor; + string internal jSetFeeSetter; + string internal jSetFeeClaimer; + string internal jSetTsSetter; + string internal jSetDefCf; + string internal jLiquidate; + + uint256 internal logIdx; + uint256 internal currentBlockTimestamp; + + // ── Setup + // ───────────────────────────────────────────────────────────────── + + function setUp() public override { + vm.warp(1000); + vm.roll(1); + super.setUp(); + midnight.setFeeClaimer(address(this)); + + CollateralParams[] memory params = new CollateralParams[](1); + params[0] = CollateralParams({ + token: address(collateralToken1), + lltv: LLTV_5, + maxLif: _maxLif(LLTV_5, LIQUIDATION_CURSOR_LOW), + oracle: address(oracle1) + }); + market = Market({ + chainId: block.chainid, + midnight: address(midnight), + loanToken: address(loanToken), + collateralParams: params, + maturity: 1000 + 30 days, + rcfThreshold: 1, + enterGate: address(0), + liquidatorGate: address(0) + }); + oracle1.setPrice(ORACLE_PRICE_SCALE); + id = toId(market); + + jTake = "["; + jUpdatePos = "["; + jWithdraw = "["; + jRepay = "["; + jSupplyCol = "["; + jWithdrawCol = "["; + jMarketCreated = "["; + jSetConsumed = "["; + jSetIsAuth = "["; + jSetDefTf = "["; + jSetMktTf = "["; + jSetMktCf = "["; + jClaimTf = "["; + jClaimCf = "["; + jConstructor = "["; + jSetFeeSetter = "["; + jSetFeeClaimer = "["; + jSetTsSetter = "["; + jSetDefCf = "["; + jLiquidate = "["; + logIdx = 0; + } + + // ── Main test + // ───────────────────────────────────────────────────────────── + + function testGenerateScenario() public { + // Randomise key amounts; bounds chosen so all operations succeed. + uint256 units1 = vm.randomUint(300_000, 800_000); // borrower's debt + uint256 repayFraction = vm.randomUint(20, 70); // % of units1 repaid + uint256 withdrawCol = vm.randomUint(5, 50); // tokens withdrawn + uint256 otherUnits = vm.randomUint(500, 2_000); // otherBorrower's debt (small for liquidation) + + // Block 2: SetDefaultSettlementFee (100 CBP at 30-day breakpoint) + vm.roll(2); + vm.warp(2000); + _exec(abi.encodeCall(midnight.setDefaultSettlementFee, (address(loanToken), 3, DEFAULT_SETTLEMENT_FEE_RAW))); + + // Block 3: SetDefaultContinuousFee + vm.roll(3); + vm.warp(3000); + _exec(abi.encodeCall(midnight.setDefaultContinuousFee, (address(loanToken), 50))); + + // Block 4: borrower supplies collateral → creates market + vm.roll(4); + vm.warp(4000); + uint256 col1 = units1 * 12 / 10; // 120 % safety buffer over LLTV + deal(address(collateralToken1), borrower, col1); + vm.prank(borrower); + collateralToken1.approve(address(midnight), col1); + _execAs(borrower, abi.encodeWithSelector(midnight.supplyCollateral.selector, market, 0, col1, borrower)); + + // Block 5: Take #1 — lender posts buy offer, borrower takes + vm.roll(5); + vm.warp(5000); + deal(address(loanToken), lender, units1 * 2); + Offer memory offer1; + offer1.market = market; + offer1.buy = true; + offer1.maker = lender; + offer1.maxUnits = units1; + offer1.continuousFeeCap = type(uint256).max; + offer1.group = GROUP_A; + offer1.ratifier = address(ecrecoverRatifier); + offer1.start = vm.getBlockTimestamp(); + offer1.expiry = vm.getBlockTimestamp() + 200; + offer1.tick = MAX_TICK; + bytes memory rd1 = merkleRatifierData([offer1]); + _execAs(borrower, abi.encodeCall(midnight.take, (offer1, rd1, units1, borrower, borrower, address(0), hex""))); + + // Block 6: otherBorrower supplies collateral (liquidation victim setup) + vm.roll(6); + vm.warp(6000); + // Exact minimum collateral at current oracle price + uint256 col2 = otherUnits.mulDivUp(WAD, LLTV_5).mulDivUp(ORACLE_PRICE_SCALE, oracle1.price()); + deal(address(collateralToken1), otherBorrower, col2); + vm.prank(otherBorrower); + collateralToken1.approve(address(midnight), col2); + _execAs( + otherBorrower, abi.encodeWithSelector(midnight.supplyCollateral.selector, market, 0, col2, otherBorrower) + ); + + // Block 7: Take #2 — otherLender posts buy offer, otherBorrower takes + vm.roll(7); + vm.warp(7000); + deal(address(loanToken), otherLender, otherUnits * 2); + Offer memory offer2; + offer2.market = market; + offer2.buy = true; + offer2.maker = otherLender; + offer2.maxUnits = otherUnits; + offer2.continuousFeeCap = type(uint256).max; + offer2.group = GROUP_B; + offer2.ratifier = address(ecrecoverRatifier); + offer2.start = vm.getBlockTimestamp(); + offer2.expiry = vm.getBlockTimestamp() + 200; + offer2.tick = MAX_TICK; + bytes memory rd2 = merkleRatifierData([offer2]); + _execAs( + otherBorrower, + abi.encodeCall(midnight.take, (offer2, rd2, otherUnits, otherBorrower, otherBorrower, address(0), hex"")) + ); + + // Block 8: borrower repays partial debt + vm.roll(8); + vm.warp(8000); + uint256 repayAmount = units1 * repayFraction / 100; + deal(address(loanToken), borrower, repayAmount); + _execAs(borrower, abi.encodeCall(midnight.repay, (market, repayAmount, borrower, address(0), hex""))); + + // Block 9: borrower withdraws some collateral + vm.roll(9); + vm.warp(9000); + _execAs( + borrower, + abi.encodeWithSelector(midnight.withdrawCollateral.selector, market, 0, withdrawCol, borrower, borrower) + ); + + // Block 10: explicit updatePosition(lender) + vm.roll(10); + vm.warp(10000); + _exec(abi.encodeCall(midnight.updatePosition, (market, lender))); + + // Block 11: setMarketSettlementFee (index 3 → 50 CBP) + vm.roll(11); + vm.warp(11000); + _exec(abi.encodeCall(midnight.setMarketSettlementFee, (id, 3, MARKET_SETTLEMENT_FEE_RAW))); + + // Block 12: setMarketContinuousFee + vm.roll(12); + vm.warp(12000); + _exec(abi.encodeCall(midnight.setMarketContinuousFee, (id, 80))); + + // Block 13: updatePosition with large time-warp → continuousFee accrues + vm.roll(13); + vm.warp(20000); + _exec(abi.encodeCall(midnight.updatePosition, (market, lender))); + + // Block 14: setIsAuthorized (lender authorises otherLender) + vm.roll(14); + vm.warp(20100); + _execAs(lender, abi.encodeWithSelector(midnight.setIsAuthorized.selector, otherLender, true, lender)); + + // Block 15: setConsumed (borrower cancels GROUP_A) + vm.roll(15); + vm.warp(20200); + _execAs(borrower, abi.encodeCall(midnight.setConsumed, (GROUP_A, 1_000_000_000_000_000_000, borrower))); + + // Block 16: liquidate otherBorrower (oracle drop → liquidation → restore) + vm.roll(16); + vm.warp(20300); + oracle1.setPrice(ORACLE_PRICE_SCALE / 4); // 75 % price drop → position deeply underwater + _exec( + abi.encodeCall( + midnight.liquidate, (market, 0, 0, 0, otherBorrower, false, address(this), address(0), hex"") + ) + ); + oracle1.setPrice(ORACLE_PRICE_SCALE); // restore for subsequent operations + + // Block 17: lender withdraws — use at most half of available withdrawable + vm.roll(17); + vm.warp(20400); + { + (,, uint128 msW,,,,,,,,,,) = midnight.marketState(id); + (uint128 lCr,,,,,) = midnight.position(id, lender); + uint256 avail = msW < lCr ? uint256(msW) : uint256(lCr); + uint256 withdrawUnits = avail / 2; + if (withdrawUnits > 0) { + _execAs( + lender, abi.encodeWithSelector(midnight.withdraw.selector, market, withdrawUnits, lender, lender) + ); + } + } + + // Block 18: claimSettlementFee + vm.roll(18); + vm.warp(20500); + uint256 ctf = midnight.claimableSettlementFee(address(loanToken)); + if (ctf > 0) { + _exec(abi.encodeCall(midnight.claimSettlementFee, (address(loanToken), ctf, address(this)))); + } + + // Block 19: claimContinuousFee + vm.roll(19); + vm.warp(20600); + (,,, uint128 ms_cfc,,,,,,,,,) = midnight.marketState(id); + if (ms_cfc > 0) { + _exec(abi.encodeCall(midnight.claimContinuousFee, (market, ms_cfc, address(this)))); + } + + _appendSetUpEvents(); + _writeOutput(); + } + + // ── Execution helpers + // ───────────────────────────────────────────────────── + + function _exec(bytes memory callData) internal { + vm.recordLogs(); + uint256 bn = block.number; + currentBlockTimestamp = block.timestamp; + + (bool ok, bytes memory ret) = address(midnight).call(callData); + if (!ok) { + assembly ("memory-safe") { revert(add(ret, 32), mload(ret)) } + } + + Vm.Log[] memory logs = vm.getRecordedLogs(); + for (uint256 i = 0; i < logs.length; i++) { + _processLog(logs[i], bn, logIdx++); + } + } + + function _execAs(address who, bytes memory callData) internal { + vm.recordLogs(); + uint256 bn = block.number; + currentBlockTimestamp = block.timestamp; + + vm.prank(who); + (bool ok, bytes memory ret) = address(midnight).call(callData); + if (!ok) { + assembly ("memory-safe") { revert(add(ret, 32), mload(ret)) } + } + + Vm.Log[] memory logs = vm.getRecordedLogs(); + for (uint256 i = 0; i < logs.length; i++) { + _processLog(logs[i], bn, logIdx++); + } + } + + // ── Log dispatcher + // ──────────────────────────────────────────────────────── + + function _processLog(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 sig = log.topics[0]; + if (sig == SEL_TAKE) _decodeTake(log, bn, idx); + else if (sig == SEL_UPDATE_POS) _decodeUpdatePos(log, bn, idx); + else if (sig == SEL_WITHDRAW) _decodeWithdraw(log, bn, idx); + else if (sig == SEL_REPAY) _decodeRepay(log, bn, idx); + else if (sig == SEL_SUPPLY_COL) _decodeSupplyCol(log, bn, idx); + else if (sig == SEL_WITHDRAW_COL) _decodeWithdrawCol(log, bn, idx); + else if (sig == SEL_MARKET_CREATED) _decodeMarketCreated(log, bn, idx); + else if (sig == SEL_SET_CONSUMED) _decodeSetConsumed(log, bn, idx); + else if (sig == SEL_SET_IS_AUTH) _decodeSetIsAuth(log, bn, idx); + else if (sig == SEL_SET_DEF_TF) _decodeSetDefTf(log, bn, idx); + else if (sig == SEL_SET_MKT_TF) _decodeSetMktTf(log, bn, idx); + else if (sig == SEL_SET_MKT_CF) _decodeSetMktCf(log, bn, idx); + else if (sig == SEL_CLAIM_TF) _decodeClaimTf(log, bn, idx); + else if (sig == SEL_CLAIM_CF) _decodeClaimCf(log, bn, idx); + else if (sig == SEL_CONSTRUCTOR) _decodeConstructor(log, bn, idx); + else if (sig == SEL_SET_FEE_SETTER) _decodeSetFeeSetter(log, bn, idx); + else if (sig == SEL_SET_FEE_CLAIMER) _decodeSetFeeClaimer(log, bn, idx); + else if (sig == SEL_SET_TS_SETTER) _decodeSetTsSetter(log, bn, idx); + else if (sig == SEL_SET_DEF_CF) _decodeSetDefCf(log, bn, idx); + else if (sig == SEL_LIQUIDATE) _decodeLiquidate(log, bn, idx); + } + + // ── Per-event decoders + // ──────────────────────────────────────────────────── + + function _decodeTake(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + address taker = address(uint160(uint256(log.topics[2]))); + address maker = address(uint160(uint256(log.topics[3]))); + // data slots (0-indexed): 0=caller, 1=units, 2=offerIsBuy, 3=group, 4=buyerAssets, + // 5=sellerAssets, 6=consumed, 7=buyerPFI, 8=sellerPFD, 9=buyerCI, 10=sellerCD, 11=receiver, 12=payer + bytes memory d = log.data; + string memory body = string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("maker", _as(maker)), + _c, + _sb("taker", _as(taker)), + _c, + _nb("offerisbuy", _word(d, 2) != 0 ? "true" : "false"), + _c, + _sn("buyerassets", uint256(_word(d, 4))), + _c, + _sn("sellerassets", uint256(_word(d, 5))), + _c, + _sn("units", uint256(_word(d, 1))) + ); + body = string.concat( + body, + _c, + _sb("group", _b32s(_word(d, 3))), + _c, + _sn("consumed", uint256(_word(d, 6))), + _c, + _sn("buyerpendingfeeincrease", uint256(_word(d, 7))), + _c, + _sn("sellerpendingfeedecrease", uint256(_word(d, 8))), + _c, + _sn("buyercreditincrease", uint256(_word(d, 9))), + _c, + _sn("sellercreditdecrease", uint256(_word(d, 10))) + ); + body = string.concat(body, _c, _sn("evt_block_number", bn), _c, _sn("evt_index", idx)); + jTake = _app(jTake, body); + } + + function _decodeUpdatePos(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + address user = address(uint160(uint256(log.topics[2]))); + (uint256 creditDecrease, uint256 pendingFeeDecrease, uint256 accruedFee) = + abi.decode(log.data, (uint256, uint256, uint256)); + string memory body = string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("user", _as(user)), + _c, + _sn("creditdecrease", creditDecrease), + _c, + _sn("pendingfeedecrease", pendingFeeDecrease), + _c, + _sn("accruedfee", accruedFee) + ); + body = string.concat( + body, + _c, + _sn("evt_block_time", currentBlockTimestamp), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ); + jUpdatePos = _app(jUpdatePos, body); + } + + function _decodeWithdraw(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + address onB = address(uint160(uint256(log.topics[2]))); + address recv = address(uint160(uint256(log.topics[3]))); + (, uint256 units, uint256 pfd) = abi.decode(log.data, (address, uint256, uint256)); + jWithdraw = _app( + jWithdraw, + string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("onbehalf", _as(onB)), + _c, + _sb("receiver", _as(recv)), + _c, + _sn("units", units), + _c, + _sn("pendingfeedecrease", pfd), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeRepay(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[2]; + address onB = address(uint160(uint256(log.topics[3]))); + (uint256 units,) = abi.decode(log.data, (uint256, address)); + jRepay = _app( + jRepay, + string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("onbehalf", _as(onB)), + _c, + _sn("units", units), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeSupplyCol(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + address token = address(uint160(uint256(log.topics[2]))); + address onB = address(uint160(uint256(log.topics[3]))); + (, uint256 assets) = abi.decode(log.data, (address, uint256)); + jSupplyCol = _app( + jSupplyCol, + string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("collateral", _as(token)), + _c, + _sb("onbehalf", _as(onB)), + _c, + _sn("assets", assets), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeWithdrawCol(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + address token = address(uint160(uint256(log.topics[2]))); + address onB = address(uint160(uint256(log.topics[3]))); + (, uint256 assets,) = abi.decode(log.data, (address, uint256, address)); + jWithdrawCol = _app( + jWithdrawCol, + string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("collateral", _as(token)), + _c, + _sb("onbehalf", _as(onB)), + _c, + _sn("assets", assets), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + // Reads actual initial fee values from the contract after touchMarket initialises them. + function _decodeMarketCreated(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + Market memory mkt = abi.decode(log.data, (Market)); + string memory body; + // Split into two scoped blocks to keep live variables under the Yul stack limit. + { + uint16 tf0; + uint16 tf1; + uint16 tf2; + uint16 tf3; + (,,,, tf0, tf1, tf2, tf3,,,,,) = midnight.marketState(_id); + body = string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("market_loantoken", _as(mkt.loanToken)), + _c, + _sn("market_maturity", mkt.maturity), + _c, + _sn("market_settlementfeecbp0", tf0), + _c, + _sn("market_settlementfeecbp1", tf1), + _c, + _sn("market_settlementfeecbp2", tf2), + _c, + _sn("market_settlementfeecbp3", tf3) + ); + } + { + uint16 tf4; + uint16 tf5; + uint16 tf6; + uint32 cf; + (,,,,,,,, tf4, tf5, tf6, cf,) = midnight.marketState(_id); + body = string.concat( + body, + _c, + _sn("market_settlementfeecbp4", tf4), + _c, + _sn("market_settlementfeecbp5", tf5), + _c, + _sn("market_settlementfeecbp6", tf6), + _c, + _sn("market_continuousfee", cf), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ); + } + jMarketCreated = _app(jMarketCreated, body); + } + + function _decodeSetConsumed(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 grp = log.topics[2]; + address onB = address(uint160(uint256(log.topics[3]))); + (uint256 amount) = abi.decode(log.data, (uint256)); + jSetConsumed = _app( + jSetConsumed, + string.concat( + _sb("onbehalf", _as(onB)), + _c, + _sb("group", _b32s(grp)), + _c, + _sn("amount", amount), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeSetIsAuth(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address auth = address(uint160(uint256(log.topics[2]))); + address onB = address(uint160(uint256(log.topics[3]))); + (bool val) = abi.decode(log.data, (bool)); + jSetIsAuth = _app( + jSetIsAuth, + string.concat( + _sb("onbehalf", _as(onB)), + _c, + _sb("authorized", _as(auth)), + _c, + _nb("newisauthorized", val ? "true" : "false"), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeSetDefTf(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address lt = address(uint160(uint256(log.topics[1]))); + uint256 ix = uint256(log.topics[2]); + (uint256 fee) = abi.decode(log.data, (uint256)); + jSetDefTf = _app( + jSetDefTf, + string.concat( + _sb("loantoken", _as(lt)), + _c, + _sn("index", ix), + _c, + _sn("newsettlementfee", fee), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeSetMktTf(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + uint256 ix = uint256(log.topics[2]); + (uint256 fee) = abi.decode(log.data, (uint256)); + jSetMktTf = _app( + jSetMktTf, + string.concat( + _sb("id_", _b32s(_id)), + _c, + _sn("index", ix), + _c, + _sn("newsettlementfee", fee), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeSetMktCf(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + (uint256 fee) = abi.decode(log.data, (uint256)); + jSetMktCf = _app( + jSetMktCf, + string.concat( + _sb("id_", _b32s(_id)), + _c, + _sn("newcontinuousfee", fee), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeClaimTf(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address token = address(uint160(uint256(log.topics[2]))); + (uint256 amount) = abi.decode(log.data, (uint256)); + jClaimTf = _app( + jClaimTf, + string.concat( + _sb("token", _as(token)), + _c, + _sn("amount", amount), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeClaimCf(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[2]; + (uint256 amount) = abi.decode(log.data, (uint256)); + jClaimCf = _app( + jClaimCf, + string.concat( + _sb("id_", _b32s(_id)), + _c, + _sn("amount", amount), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeConstructor(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address rs = address(uint160(uint256(log.topics[1]))); + jConstructor = _app( + jConstructor, + string.concat(_sb("rolesetter", _as(rs)), _c, _sn("evt_block_number", bn), _c, _sn("evt_index", idx)) + ); + } + + function _decodeSetFeeSetter(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address a = address(uint160(uint256(log.topics[1]))); + jSetFeeSetter = _app( + jSetFeeSetter, + string.concat(_sb("feesetter", _as(a)), _c, _sn("evt_block_number", bn), _c, _sn("evt_index", idx)) + ); + } + + function _decodeSetFeeClaimer(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address a = address(uint160(uint256(log.topics[1]))); + jSetFeeClaimer = _app( + jSetFeeClaimer, + string.concat(_sb("feeclaimer", _as(a)), _c, _sn("evt_block_number", bn), _c, _sn("evt_index", idx)) + ); + } + + function _decodeSetTsSetter(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address a = address(uint160(uint256(log.topics[1]))); + jSetTsSetter = _app( + jSetTsSetter, + string.concat(_sb("tickspacingsetter", _as(a)), _c, _sn("evt_block_number", bn), _c, _sn("evt_index", idx)) + ); + } + + function _decodeSetDefCf(Vm.Log memory log, uint256 bn, uint256 idx) internal { + address lt = address(uint160(uint256(log.topics[1]))); + (uint256 fee) = abi.decode(log.data, (uint256)); + jSetDefCf = _app( + jSetDefCf, + string.concat( + _sb("loantoken", _as(lt)), + _c, + _sn("newcontinuousfee", fee), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ) + ); + } + + function _decodeLiquidate(Vm.Log memory log, uint256 bn, uint256 idx) internal { + bytes32 _id = log.topics[1]; + address collat = address(uint160(uint256(log.topics[2]))); + address borrowerAddr = address(uint160(uint256(log.topics[3]))); + // data: caller, seizedAssets, repaidUnits, postMaturityMode, receiver, payer, badDebt, latestLossFactor, + // latestContinuousFeeCredit + (, uint256 seized, uint256 repaid, bool pmMode,,, uint256 bad, uint256 latestLF, uint256 latestCFC) = + abi.decode(log.data, (address, uint256, uint256, bool, address, address, uint256, uint256, uint256)); + string memory body = string.concat( + _sb("id_", _b32s(_id)), + _c, + _sb("collateral", _as(collat)), + _c, + _sb("borrower", _as(borrowerAddr)), + _c, + _sn("seizedassets", seized), + _c, + _sn("repaidunits", repaid), + _c, + _nb("postmaturitymode", pmMode ? "true" : "false"), + _c, + _sn("baddebt", bad), + _c, + _sn("latestlossfactor", latestLF) + ); + body = string.concat( + body, + _c, + _sn("latestcontinuousfeecredit", latestCFC), + _c, + _sn("evt_block_number", bn), + _c, + _sn("evt_index", idx) + ); + jLiquidate = _app(jLiquidate, body); + } + + // ── Manually append setUp events (block 1) + // ──────────────────────────────── + + function _appendSetUpEvents() internal { + // Constructor + jConstructor = _app( + jConstructor, + string.concat( + _sb("rolesetter", _as(address(this))), _c, _sn("evt_block_number", 1), _c, _sn("evt_index", logIdx++) + ) + ); + // SetFeeSetter + jSetFeeSetter = _app( + jSetFeeSetter, + string.concat( + _sb("feesetter", _as(address(this))), _c, _sn("evt_block_number", 1), _c, _sn("evt_index", logIdx++) + ) + ); + // SetTickSpacingSetter + jSetTsSetter = _app( + jSetTsSetter, + string.concat( + _sb("tickspacingsetter", _as(address(this))), + _c, + _sn("evt_block_number", 1), + _c, + _sn("evt_index", logIdx++) + ) + ); + // SetIsAuthorized ×4 (each user authorises ecrecoverRatifier in setUp) + address[4] memory users = [borrower, lender, otherBorrower, otherLender]; + for (uint256 i = 0; i < 4; i++) { + jSetIsAuth = _app( + jSetIsAuth, + string.concat( + _sb("onbehalf", _as(users[i])), + _c, + _sb("authorized", _as(address(ecrecoverRatifier))), + _c, + _nb("newisauthorized", "true"), + _c, + _sn("evt_block_number", 1), + _c, + _sn("evt_index", logIdx++) + ) + ); + } + // SetFeeClaimer (called in SqlScenarioTest.setUp) + jSetFeeClaimer = _app( + jSetFeeClaimer, + string.concat( + _sb("feeclaimer", _as(address(this))), _c, _sn("evt_block_number", 1), _c, _sn("evt_index", logIdx++) + ) + ); + } + + // ── Write output files + // ──────────────────────────────────────────────────── + + function _writeOutput() internal { + string memory dir = "sql/test/events/"; + vm.writeFile(string.concat(dir, "take.json"), string.concat(jTake, "]")); + vm.writeFile(string.concat(dir, "updateposition.json"), string.concat(jUpdatePos, "]")); + vm.writeFile(string.concat(dir, "withdraw.json"), string.concat(jWithdraw, "]")); + vm.writeFile(string.concat(dir, "repay.json"), string.concat(jRepay, "]")); + vm.writeFile(string.concat(dir, "supplycollateral.json"), string.concat(jSupplyCol, "]")); + vm.writeFile(string.concat(dir, "withdrawcollateral.json"), string.concat(jWithdrawCol, "]")); + vm.writeFile(string.concat(dir, "marketcreated.json"), string.concat(jMarketCreated, "]")); + vm.writeFile(string.concat(dir, "setconsumed.json"), string.concat(jSetConsumed, "]")); + vm.writeFile(string.concat(dir, "setisauthorized.json"), string.concat(jSetIsAuth, "]")); + vm.writeFile(string.concat(dir, "setdefaultsettlementfee.json"), string.concat(jSetDefTf, "]")); + vm.writeFile(string.concat(dir, "setmarketsettlementfee.json"), string.concat(jSetMktTf, "]")); + vm.writeFile(string.concat(dir, "setmarketcontinuousfee.json"), string.concat(jSetMktCf, "]")); + vm.writeFile(string.concat(dir, "claimsettlementfee.json"), string.concat(jClaimTf, "]")); + vm.writeFile(string.concat(dir, "claimcontinuousfee.json"), string.concat(jClaimCf, "]")); + vm.writeFile(string.concat(dir, "constructor.json"), string.concat(jConstructor, "]")); + vm.writeFile(string.concat(dir, "setfeesetter.json"), string.concat(jSetFeeSetter, "]")); + vm.writeFile(string.concat(dir, "setfeeclaimer.json"), string.concat(jSetFeeClaimer, "]")); + vm.writeFile(string.concat(dir, "settickspacingsetter.json"), string.concat(jSetTsSetter, "]")); + vm.writeFile(string.concat(dir, "setdefaultcontinuousfee.json"), string.concat(jSetDefCf, "]")); + vm.writeFile(string.concat(dir, "liquidate.json"), string.concat(jLiquidate, "]")); + + _writeExpectedState(); + } + + function _writeExpectedState() internal { + string memory j = "{"; + j = string.concat(j, '"lender_addr":"', _as(lender), '",'); + j = string.concat(j, '"borrower_addr":"', _as(borrower), '",'); + j = string.concat(j, '"other_lender_addr":"', _as(otherLender), '",'); + j = string.concat(j, '"other_borrower_addr":"', _as(otherBorrower), '",'); + j = string.concat(j, '"loan_token_addr":"', _as(address(loanToken)), '",'); + j = string.concat(j, '"market_id":"', _b32s(id), '",'); + j = string.concat(j, '"group_a":"', _b32s(GROUP_A), '",'); + j = string.concat(j, '"position_lender":{', _posJson(id, lender), "},"); + j = string.concat(j, '"position_borrower":{', _posJson(id, borrower), "},"); + j = string.concat(j, '"position_other_borrower":{', _posJson(id, otherBorrower), "},"); + j = string.concat(j, '"borrower_collateral_0":', _us(midnight.collateral(id, borrower, 0)), ","); + j = string.concat(j, '"market_state":{', _mktStateJson(), "},"); + j = string.concat(j, '"consumed_borrower_group_a":', _us(midnight.consumed(borrower, GROUP_A)), ","); + bool isAuth = midnight.isAuthorized(lender, otherLender); + j = string.concat(j, '"is_authorized_lender_other_lender":', isAuth ? "true" : "false", ","); + j = string.concat( + j, '"claimable_settlement_fee":', _us(midnight.claimableSettlementFee(address(loanToken))), "," + ); + j = string.concat( + j, '"default_settlement_fee_cbp_3":', _us(midnight.defaultSettlementFeeCbp(address(loanToken), 3)), "," + ); + j = string.concat(j, '"default_continuous_fee":', _us(midnight.defaultContinuousFee(address(loanToken))), ","); + j = string.concat(j, '"role_setter":"', _as(midnight.roleSetter()), '",'); + j = string.concat(j, '"fee_setter":"', _as(midnight.feeSetter()), '",'); + j = string.concat(j, '"fee_claimer":"', _as(midnight.feeClaimer()), '",'); + j = string.concat(j, '"tick_spacing_setter":"', _as(midnight.tickSpacingSetter()), '"'); + j = string.concat(j, "}"); + vm.writeFile("sql/test/expected_state.json", j); + } + + // ── Market-state helpers (split to avoid Yul stack depth limits) ────────── + + function _posJson(bytes32 _id, address user) internal view returns (string memory) { + (uint128 credit, uint128 pendingFee, uint128 lastLF, uint128 lastAccrual, uint128 debt,) = + midnight.position(_id, user); + return string.concat( + _sn("credit", credit), + _c, + _sn("pending_fee", pendingFee), + _c, + _sn("last_loss_factor", lastLF), + _c, + _sn("last_accrual", lastAccrual), + _c, + _sn("debt", debt) + ); + } + + function _mktStateJson() internal view returns (string memory) { + string memory s = string.concat(_mktStateCoreJson(), _c, _mktStateFeesJson()); + return string.concat(s, _c, _mktStateLastJson()); + } + + function _mktStateCoreJson() internal view returns (string memory) { + (uint128 totalUnits, uint128 lossFactor, uint128 withdrawable, uint128 cfc,,,,,,,,,) = midnight.marketState(id); + return string.concat( + _sn("total_units", totalUnits), + _c, + _sn("loss_factor", lossFactor), + _c, + _sn("withdrawable", withdrawable), + _c, + _sn("continuous_fee_credit", cfc) + ); + } + + function _mktStateFeesJson() internal view returns (string memory) { + (,,,, uint16 tf0, uint16 tf1, uint16 tf2, uint16 tf3, uint16 tf4, uint16 tf5, uint16 tf6,,) = + midnight.marketState(id); + string memory s = string.concat( + _sn("settlement_fee_cbp_0", tf0), + _c, + _sn("settlement_fee_cbp_1", tf1), + _c, + _sn("settlement_fee_cbp_2", tf2), + _c, + _sn("settlement_fee_cbp_3", tf3) + ); + return string.concat( + s, + _c, + _sn("settlement_fee_cbp_4", tf4), + _c, + _sn("settlement_fee_cbp_5", tf5), + _c, + _sn("settlement_fee_cbp_6", tf6) + ); + } + + function _mktStateLastJson() internal view returns (string memory) { + (,,,,,,,,,,, uint32 contFee, uint8 tickSpacing) = midnight.marketState(id); + return string.concat(_sn("continuous_fee", contFee), _c, _sn("tick_spacing", tickSpacing)); + } + + // ── JSON helpers + // ────────────────────────────────────────────────────────── + + string internal constant _c = ","; + + // Reads the i-th 32-byte ABI word (0-indexed) from ABI-encoded bytes memory. + function _word(bytes memory d, uint256 i) private pure returns (bytes32 v) { + assembly ("memory-safe") { v := mload(add(add(d, 32), mul(i, 32))) } + } + + function merkleRatifierData(Offer[1] memory offers) internal view returns (bytes memory) { + bytes32 _root = HashLib.hashOffer(offers[0]); + bytes32[] memory _proof = new bytes32[](0); + Signature memory _sig = signature(_root, privateKey[offers[0].maker], offers[0].ratifier, 0); + return abi.encode(_sig, _root, 0, _proof); + } + + function _app(string memory arr, string memory obj) internal pure returns (string memory) { + bool empty = bytes(arr).length == 1; + return string.concat(arr, empty ? "" : ",", "{", obj, "}"); + } + + function _sb(string memory k, string memory v) internal pure returns (string memory) { + return string.concat('"', k, '":"', v, '"'); + } + + function _sn(string memory k, uint256 v) internal pure returns (string memory) { + return string.concat('"', k, '":"', vm.toString(v), '"'); + } + + function _nb(string memory k, string memory v) internal pure returns (string memory) { + return string.concat('"', k, '":', v); + } + + function _as(address a) internal pure returns (string memory) { + return vm.toString(a); + } + + function _b32s(bytes32 b) internal pure returns (string memory) { + return vm.toString(b); + } + + function _us(uint256 v) internal pure returns (string memory) { + return string.concat('"', vm.toString(v), '"'); + } +}