From d1b7a7e06389aa4e7b75865acfc2ea4b9bd895af Mon Sep 17 00:00:00 2001
From: Ruairi Conlon <ruairi.conlon@dataintellect.com>
Date: Tue, 2 Jun 2026 16:17:24 +0100
Subject: [PATCH 01/15] feature/dbwritemodule-Initial-commit

---
 di/dbwrite/dbwrite.md | 248 ++++++++++++++++++++++++++++++++++++++++++
 di/dbwrite/dbwrite.q  | 100 +++++++++++++++++
 di/dbwrite/init.q     |   9 ++
 di/dbwrite/test.csv   | 101 +++++++++++++++++
 4 files changed, 458 insertions(+)
 create mode 100644 di/dbwrite/dbwrite.md
 create mode 100644 di/dbwrite/dbwrite.q
 create mode 100644 di/dbwrite/init.q
 create mode 100644 di/dbwrite/test.csv

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
new file mode 100644
index 00000000..6d0b450b
--- /dev/null
+++ b/di/dbwrite/dbwrite.md
@@ -0,0 +1,248 @@
+# di.dbwrite
+
+Sort, attribute application, save-down manipulation, and garbage-collection utilities for kdb+ processes that persist data to disk (RDB, WDB, TickerLogReplay).
+
+---
+
+## Features
+
+- Sort on-disk table partitions by configured columns using `xasc`
+- Apply kdb+ attributes (`p`, `s`, `g`, `u`) to on-disk columns after sort
+- Register per-table pre-write manipulation functions applied before save-down
+- Run `.Q.gc[]` with before/after memory logging
+- Sort and attribute behaviour driven by a CSV config file; a `default` row acts as a fallback
+- Built-in `defaultparams` provides an out-of-the-box fallback (sort by `time` ascending) when no config file is loaded
+- All errors from sort, attribute application, and manipulation are caught and logged — they do not propagate
+
+---
+
+## Dependencies
+
+| Dependency | Key | Required | Description |
+|---|---|---|---|
+| `di.log` | `` `log `` | yes | Logging functions `info`, `warn`, `error` — each `{[ctx;msg] ...}` |
+
+The `log` dependency must be passed to `init`. The module throws if it is absent or `(::)`.
+
+---
+
+## Sort config CSV
+
+`loadconfig` reads a CSV with four columns:
+
+| Column | Type | Description |
+|---|---|---|
+| `tabname` | symbol | Table name, or `` `default `` as a catch-all fallback |
+| `att` | symbol | kdb+ attribute to apply: `p`, `s`, `g`, `u`, or empty for none |
+| `column` | symbol | Column to sort or attribute; empty means attribute-only (no sort contribution) |
+| `sort` | boolean | `1b` — include in `xasc` sort key; `0b` — attribute only |
+
+Example `sort.csv`:
+
+```
+tabname,att,column,sort
+trade,p,sym,1
+trade,,price,0
+quote,p,sym,1
+default,,time,1
+```
+
+Sorts `trade` by `sym`, applies `p` to `sym`. Tables not listed fall back to `default` and sort by `time`.
+
+---
+
+## Functions
+
+### Summary
+
+| Function | Description |
+|---|---|
+| `init[config;deps]` | Wire injected dependencies; must be called first |
+| `loadconfig[file]` | Load and validate the sort config CSV into module state |
+| `sort[d]` | Sort an on-disk partition and apply attributes per config |
+| `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
+| `manipulate[t;x]` | Apply a registered pre-write manipulation to a table |
+| `postreplay[d;p]` | Post-EOD stub; override to add custom logic |
+| `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
+
+---
+
+### `init[config;deps]`
+
+Wires injected dependencies into the module. Must be called before any other function.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `config` | any | Accepted but unused; pass `(::)` |
+| `deps` | dict | Must contain `` `log `` → `` `info`warn`error!(infofunc;warnfunc;errfunc) `` |
+
+**Returns** — generic null.
+
+Throws with a descriptive message if the `log` dependency is missing or set to `(::)`.
+
+```q
+log:use`di.log
+log.init[logconfig]
+logdep:`info`warn`error!(log.info;log.warn;log.error)
+
+dbwrite:use`di.dbwrite
+dbwrite.init[(::);(enlist`log)!enlist logdep]
+```
+
+---
+
+### `loadconfig[file]`
+
+Loads and validates the sort configuration CSV, storing the result in module state for use by `sort`.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `file` | hsym | Path to the sort config CSV. Passing a null symbol (`` ` ``) uses the module-level `defaultfile` value |
+
+**Returns** — generic null on success; throws on failure.
+
+Validation checks that all four required columns (`tabname`, `att`, `column`, `sort`) are present and that all `att` values are within `` ``p`s`g`u ``. Throws a descriptive error for invalid files or unreadable paths.
+
+```q
+dbwrite.loadconfig[`:config/sort.csv]
+```
+
+> **Note:** `defaultfile` is a module-level variable (default: null symbol). Set it in `init.q` to configure a default sort config path, or always pass the path explicitly. When `params` is empty and `defaultfile` is null, `sort` falls back to the built-in `defaultparams` rather than erroring.
+
+---
+
+### `sort[d]`
+
+Sorts an on-disk table partition and applies configured attributes.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `d` | symbol or list | Table name alone, or `(tabname;dir)`, or `(tabname;list of dirs)` — see below |
+
+`d` forms:
+
+| Form | Example |
+|---|---|
+| Symbol | `` `trade `` |
+| Tabname + single dir | `` (`trade;`:hdb/2024.01.02/trade/) `` |
+| Tabname + dir list | `` (`trade;`:hdb/2024.01.02/trade/ `:hdb/2024.01.03/trade/) `` |
+
+**Returns** — generic null on success; `()` if no sort config is found for the table.
+
+If `params` is empty when `sort` is called:
+- `defaultfile` is set → `loadconfig[defaultfile]` is called to populate `params`.
+- `defaultfile` is null → `params` is populated from the built-in `defaultparams` (sort by `time` ascending, no attribute).
+
+Config lookup order within `params`:
+1. Rows where `tabname` matches — used directly.
+2. Rows where `tabname = \`default` — used with a `warn` log.
+3. No match — warns and returns `()`.
+
+Sort and attribute errors are caught, logged, and swallowed.
+
+```q
+dbwrite.sort[(`trade;`:hdb/2024.01.02/trade/)]
+```
+
+---
+
+### `applyattr[dloc;colname;att]`
+
+Applies a single kdb+ attribute to an on-disk column.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `dloc` | hsym | On-disk partition directory (e.g. `` `:hdb/2024.01.02/trade/ ``) |
+| `colname` | symbol | Column name |
+| `att` | symbol | Attribute to apply: `` `p ``, `` `s ``, `` `g ``, or `` `u `` |
+
+**Returns** — generic null on success.
+
+Logs the attempt before applying. On failure, logs the error and continues — does not throw.
+
+```q
+dbwrite.applyattr[`:hdb/2024.01.02/trade/;`sym;`p]
+```
+
+---
+
+### `manipulate[t;x]`
+
+Applies a registered pre-write manipulation to table `x` of type `t`.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `t` | symbol | Table name used to look up the registered manipulation function |
+| `x` | table | Table data to transform |
+
+**Returns** — modified table, or original table unmodified if no manipulation is registered or the function throws.
+
+Manipulations are registered in the module-internal `savedownmanipulation` dictionary (`` tabname → unary function ``). This dictionary is module-bound and populated by process initialisation code before EOD.
+
+```q
+data:dbwrite.manipulate[`trade;data]
+```
+
+---
+
+### `postreplay[d;p]`
+
+Post-EOD stub called after all tables have been written and sorted.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `d` | hsym | HDB directory |
+| `p` | date | Partition value |
+
+**Returns** — generic null.
+
+This is a no-op by default. Override at the call site to add custom post-replay logic.
+
+```q
+dbwrite.postreplay[`:hdb;2024.01.02]
+```
+
+---
+
+### `gc[]`
+
+Runs `.Q.gc[]` and logs before/after memory statistics.
+
+**Returns** — generic null.
+
+Emits two `info`-level log lines: memory stats before collection, and bytes recovered plus memory stats after.
+
+```q
+dbwrite.gc[]
+```
+
+---
+
+## Running tests
+
+```q
+k4unit:use`di.k4unit
+k4unit.moduletest`di.dbwrite
+```
+
+Tests cover: dependency injection, `loadconfig` validation, `applyattr` on valid and missing paths, `sort` with explicit config / default fallback / no-config skip, `manipulate` pass-through, and `postreplay` stub.
+
+---
+
+## Exported symbols
+
+```q
+export:([init;sort;applyattr;loadconfig;manipulate;postreplay;gc])
+```
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
new file mode 100644
index 00000000..5915529c
--- /dev/null
+++ b/di/dbwrite/dbwrite.q
@@ -0,0 +1,100 @@
+/ sort params table - populated by loadconfig
+params:([] tabname:`symbol$(); att:`symbol$(); column:`symbol$(); sort:`boolean$());
+
+/ save-down manipulation registry: tabname -> unary function
+savedownmanipulation:()!();
+
+/ default sort params - used when params is empty and no config file is set
+defaultparams:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
+
+/ load and validate sort.csv into .z.M.params
+/ file: hsym path; null falls back to .z.M.defaultfile
+loadconfig:{[file]
+  file:hsym file;
+  if[null file;file:.z.M.defaultfile];
+  if[null file;
+    dp:.z.M.defaultparams;
+    .z.m.loginfo[`dbwrite;"no sort config file set; using defaultparams"];
+    @[.z.M;`params;:;dp];
+    :[]];
+  p:@[
+    {.z.m.loginfo[`dbwrite;"retrieving sort settings from ",string x];("SSSB";enlist",")0:x};
+    file;
+    {[f;e]'"failed to open ",string[f],": ",e}[file]
+  ];
+  if[not all spcb:(spc:cols p) in `tabname`att`column`sort;
+    '"unrecognised columns (",(", " sv string spc where not spcb),") in ",string file];
+  if[not all atb:(at:distinct p`att) in ``p`s`g`u;
+    '"unrecognised attribute(s): ",", " sv string at where not atb];
+  @[.z.M;`params;:;p];
+  };
+
+/ apply a single kdb+ attribute to an on-disk column; logs and swallows errors
+applyattr:{[dloc;colname;att]
+  .z.m.loginfo[`dbwrite;"applying ",string[att]," attr to ",string[colname]," in ",string dloc];
+  .[{@[x;y;z#]};(dloc;colname;att);
+    {[dloc;colname;att;e]
+      .z.m.logerr[`dbwrite;"unable to apply ",string[att]," attr to ",string[colname]," in ",string[dloc],": ",e]
+    }[dloc;colname;att]
+  ];
+  };
+
+/ sort an on-disk table partition and apply attributes per sort.csv config
+/ d: tabname | (tabname;dir) | (tabname;list of dirs)
+sort:{[d]
+  if[0=count .z.M.params;.z.M.loadconfig .z.M.defaultfile];
+  .z.m.loginfo[`dbwrite;"sorting ",(st:string t:first d)," table"];
+  sp:$[count tabsp:select from .z.M.params where tabname=t;
+      [.z.m.loginfo[`dbwrite;"sort params found for: ",st];tabsp];
+    count defsp:select from .z.M.params where tabname=`default;
+      [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; using defaults"];defsp];
+    [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; skipping sort"];:()]];
+  {[sp;dloc]
+    if[count sortcols:exec column from sp where sort,not null column;
+      .z.m.loginfo[`dbwrite;"sorting ",string[dloc]," by: ",", " sv string sortcols];
+      .[xasc;(sortcols;dloc);
+        {[sc;dl;e]
+          .z.m.logerr[`dbwrite;"failed to sort ",string[dl]," by ",(", " sv string sc),": ",e]
+        }[sortcols;dloc]]];
+    if[count attrcols:select column,att from sp where not null att;
+      .z.M.applyattr[dloc;;]'[attrcols`column;attrcols`att]];
+  }[sp] each distinct (),last d;
+  .z.m.loginfo[`dbwrite;"finished sorting ",st," table"];
+  };
+
+/ apply registered pre-write manipulation to table x of type t
+/ returns modified table; on error logs and returns original unmodified table
+manipulate:{[t;x]
+  $[t in key .z.M.savedownmanipulation;
+    @[.z.M.savedownmanipulation[t];x;
+      {[x;e].z.m.logerr[`dbwrite;"save-down manipulation failed: ",e];x}[x]];
+    x]
+  };
+
+/ post-EOD hook - called after all tables written and sorted
+/ d: hdb directory (hsym), p: partition value (date)
+/ stub: override at the call site to add custom post-replay logic
+postreplay:{[d;p]};
+
+/ format current process memory stats as a loggable string
+memstats:{"mem stats: ",{"; "sv "=" sv'flip(string key x;(string value x),\:" MB")}`long$.Q.w[]%1048576};
+
+/ run .Q.gc[] and log before/after memory stats
+gc:{
+  .z.m.loginfo[`dbwrite;"starting garbage collect. ",.z.M.memstats[]];
+  r:.Q.gc[];
+  .z.m.loginfo[`dbwrite;"garbage collection returned ",(string `long$r%1048576),"MB. ",.z.M.memstats[]]
+  };
+
+init:{[config;deps]
+  / deps: `log!(logdict)
+  /   `log: `info`warn`error!(infofunc;warnfunc;errfunc) - required
+  / example: dbwrite.init[enlist[`log]!enlist logdep]
+  logdict:$[99h=type deps;$[(`log in key deps) and not (::)~deps`log;deps`log;()!()];()!()];
+  if[not count logdict;
+    '"di.dbwrite: log dependency is required; pass `info`warn`error functions - see di.log or refer to confluence documentation";
+  ];
+  .z.m.loginfo:logdict`info;
+  .z.m.logwarn:logdict`warn;
+  .z.m.logerr:logdict`error;
+  };
\ No newline at end of file
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
new file mode 100644
index 00000000..dcca8127
--- /dev/null
+++ b/di/dbwrite/init.q
@@ -0,0 +1,9 @@
+/ dbwrite module - sort, attribute application, save-down manipulation, and GC utilities
+/ used by processes that persist data to disk (rdb, wdb, tickerlogreplay)
+
+\l ::dbwrite.q
+
+/ default sort config file - set directly or extend init with a config dep
+defaultfile:`;
+
+export:([init;sort;applyattr;loadconfig;manipulate;postreplay;gc])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
new file mode 100644
index 00000000..53265fc1
--- /dev/null
+++ b/di/dbwrite/test.csv
@@ -0,0 +1,101 @@
+action,ms,bytes,lang,code,repeat,minver,comment
+/ Pre-test set-up: load module and mock loggers, and initialise module with mocks
+before,0,0,q,dbwrite:use`di.dbwrite,1,,load di.dbwrite module
+before,0,0,q,logcount:0,1,,initialise log call counter
+before,0,0,q,loginfo:{[c;m] logcount::logcount+1},1,,mock info logger
+before,0,0,q,logwarn:{[c;m] logcount::logcount+1},1,,mock warn logger
+before,0,0,q,logerr:{[c;m] logcount::logcount+1},1,,mock error logger
+before,0,0,q,"logdep:`info`warn`error!(loginfo;logwarn;logerr)",1,,build log dep dict
+before,0,0,q,"deps:(enlist`log)!enlist logdep",1,,wrap in deps dict
+before,0,0,q,dbwrite.init[(::);deps],1,,initialise module with mock loggers
+
+/ Test 1: init wires injected logger into individual loginfo/logwarn/logerr slots
+true,0,0,q,logdep[`info]~.m.di.0dbwrite.loginfo,1,1,injected info function stored
+true,0,0,q,logdep[`warn]~.m.di.0dbwrite.logwarn,1,1,injected warn function stored
+true,0,0,q,logdep[`error]~.m.di.0dbwrite.logerr,1,1,injected error function stored
+
+/ Test 2: init errors when log dep not provided
+fail,0,0,q,dbwrite.init[(::);(::)],1,1,init without log dep throws an error
+fail,0,0,q,dbwrite.init[(::);`log!(::)],1,1,init with log set to (::) throws an error
+
+/ Test 3: manipulate: error cases
+run,0,0,q,"tbl:([]sym:`AAPL`IBM;price:100 200f)",1,,sample table
+true,0,0,q,"tbl~dbwrite.manipulate[`trade;tbl]",1,,unregistered table name
+true,0,0,q,"tbl~dbwrite.manipulate[`;tbl]",1,,null table name
+true,0,0,q,"(0#tbl)~dbwrite.manipulate[`other;0#tbl]",1,,empty table
+
+/ Test 4 postreplay: returns generic null
+true,0,0,q,(::)~dbwrite.postreplay[`:hdb;2024.01.01],1,,stub returns generic null
+
+/ Test 5: sort uses defaultparams (sort by time) when params is empty and defaultfile is null
+run,0,0,q,`:dbwrite_defp_tp/.d set `time`sym,1,,write column order file
+run,0,0,q,`:dbwrite_defp_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
+run,0,0,q,`:dbwrite_defp_tp/sym set `IBM`AAPL,1,,write sym column
+run,0,0,q,"dbwrite.sort[(`anytable;`:dbwrite_defp_tp/)]",1,,sort using defaultparams (empty params + null defaultfile)
+true,0,0,q,(asc exec time from get `:dbwrite_defp_tp/)~exec time from get `:dbwrite_defp_tp/,1,,time column sorted ascending by defaultparams
+run,0,0,q,@[hdel;`:dbwrite_defp_tp/;{}],1,,cleanup test partition
+
+/ Test 6: loadconfig: valid config file is loaded and parsed correctly
+run,0,0,q,cfgfile:`:dbwrite_test.csv,1,,temp sort config path
+run,0,0,q,"cfgfile 0:(""tabname,att,column,sort"";""trade,p,sym,1"";""trade,,price,0"")",1,,write test sort config
+run,0,0,q,dbwrite.loadconfig[cfgfile],1,,loadconfig with valid csv succeeds
+
+/ Test 6 loadconfig - invalid config files
+true,0,0,q,@[dbwrite.loadconfig;`:nonexistent_file_xyz;{1b}],1,,nonexistent file throws
+run,0,0,q,badcols:`:dbwrite_badcols.csv,1,,
+run,0,0,q,"badcols 0:(""wrongcol,att,column,sort"";""trade,p,sym,1"")",1,,csv with unrecognised column name
+true,0,0,q,@[dbwrite.loadconfig;badcols;{1b}],1,,unrecognised column throws
+run,0,0,q,badatt:`:dbwrite_badatt.csv,1,,
+run,0,0,q,"badatt 0:(""tabname,att,column,sort"";""trade,z,sym,1"")",1,,csv with unrecognised attribute
+true,0,0,q,@[dbwrite.loadconfig;badatt;{1b}],1,,unrecognised attribute throws
+
+/ Clean-up config files after loadconfig tests
+run,0,0,q,@[hdel;cfgfile;{}],1,,remove temp sort config
+run,0,0,q,@[hdel;badcols;{}],1,,remove temp bad columns file
+run,0,0,q,@[hdel;badatt;{}],1,,remove temp bad attribute file
+comment,,,,,,,--- manipulate: registered function path not testable (savedownmanipulation in .z.M is module-bound and not settable from test scope) ---
+
+/ Test 7: Applyattr - error cases
+run,0,0,q,logcount:0,1,,reset counter
+run,0,0,q,"dbwrite.applyattr[`:nonexist_attr_dir/;`sym;`p]",1,,apply to non-existent path
+true,0,0,q,2=logcount,1,,loginfo (before attempt) and logerr (on failure) each called once
+
+/ Test 8 - Applyattr - attribute application and logging when config entry exists for table
+run,0,0,q,`:dbwrite_attr_tp/.d set `sym`price,1,,write column order file
+run,0,0,q,`:dbwrite_attr_tp/sym set `IBM`AAPL`MSFT,1,,write sym column
+run,0,0,q,`:dbwrite_attr_tp/price set 200 100 300f,1,,write price column
+run,0,0,q,logcount:0,1,,reset counter
+run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`p]",1,,apply p attr to sym column
+true,0,0,q,1=logcount,1,,loginfo called once (no error)
+true,0,0,q,`p=attr get `:dbwrite_attr_tp/sym,1,,p attribute applied on disk
+run,0,0,q,@[hdel;`:dbwrite_attr_tp/;{}],1,,cleanup test partition
+
+/ Test 9 - sort: error cases
+run,0,0,q,logcount:0,1,,reset counter
+true,0,0,q,()~dbwrite.sort[`no_params_table_xyz],1,,returns () when table not in config
+true,0,0,q,2=logcount,1,,loginfo (sorting start) and logwarn (skip) both called
+
+/ Test 10 - sort: sorting and logging when config entry exists for table
+run,0,0,q,`:dbwrite_sort_tp/.d set `sym`price,1,,write column order file
+run,0,0,q,`:dbwrite_sort_tp/sym set `IBM`AAPL`MSFT,1,,write unsorted sym column
+run,0,0,q,`:dbwrite_sort_tp/price set 200 100 300f,1,,write price column
+run,0,0,q,"dbwrite.sort[(`trade;`:dbwrite_sort_tp/)]",1,,sort trade table in test partition
+true,0,0,q,`AAPL`IBM`MSFT~exec sym from get `:dbwrite_sort_tp/,1,,sorted ascending by sym
+true,0,0,q,`p=attr get `:dbwrite_sort_tp/sym,1,,p attribute applied to sym on disk
+run,0,0,q,@[hdel;`:dbwrite_sort_tp/;{}],1,,cleanup sort test partition
+
+/ Test 11 - sort: sorting with default config entry when no specific entry for table
+run,0,0,q,defcfg:`:dbwrite_defcfg.csv,1,,
+run,0,0,q,"defcfg 0:(""tabname,att,column,sort"";""default,,sym,1"")",1,,config with default entry only
+run,0,0,q,dbwrite.loadconfig[defcfg],1,,load default-only config
+run,0,0,q,`:dbwrite_def_tp/.d set `sym`price,1,,write column order file
+run,0,0,q,`:dbwrite_def_tp/sym set `IBM`AAPL,1,,write unsorted sym column
+run,0,0,q,`:dbwrite_def_tp/price set 200 100f,1,,write price column
+run,0,0,q,logcount:0,1,,reset counter
+run,0,0,q,"dbwrite.sort[(`othertable;`:dbwrite_def_tp/)]",1,,sort using default fallback
+true,0,0,q,`AAPL`IBM~exec sym from get `:dbwrite_def_tp/,1,,sorted ascending by sym via default params
+true,0,0,q,4=logcount,1,,loginfo x2 (sorting start + sort-by) + logwarn x1 (using defaults) + loginfo x1 (finished)
+
+/ Clean-up default config and test partition
+after,0,0,q,@[hdel;`:dbwrite_def_tp/;{}],1,,cleanup test partition
+after,0,0,q,@[hdel;defcfg;{}],1,,remove default config csv
\ No newline at end of file

From dac127aa78b00895d4d4ae37fa5640eacc7d9670 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Wed, 3 Jun 2026 10:40:36 +0100
Subject: [PATCH 02/15] Update dbwrite.md and dbwrite.q

---
 di/dbwrite/dbwrite.md | 10 +++++-----
 di/dbwrite/dbwrite.q  | 11 +++++------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 6d0b450b..4a5d249a 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -111,7 +111,7 @@ Validation checks that all four required columns (`tabname`, `att`, `column`, `s
 dbwrite.loadconfig[`:config/sort.csv]
 ```
 
-> **Note:** `defaultfile` is a module-level variable (default: null symbol). Set it in `init.q` to configure a default sort config path, or always pass the path explicitly. When `params` is empty and `defaultfile` is null, `sort` falls back to the built-in `defaultparams` rather than erroring.
+> **Note:** `defaultfile` is a module-level variable (default: null symbol). Set it in `init.q` to configure a path that `sort` will auto-load on first use. If no explicit `loadconfig` call is made and `defaultfile` is not set, `sort` falls back to `defaultparams` automatically.
 
 ---
 
@@ -135,9 +135,9 @@ Sorts an on-disk table partition and applies configured attributes.
 
 **Returns** — generic null on success; `()` if no sort config is found for the table.
 
-If `params` is empty when `sort` is called:
-- `defaultfile` is set → `loadconfig[defaultfile]` is called to populate `params`.
-- `defaultfile` is null → `params` is populated from the built-in `defaultparams` (sort by `time` ascending, no attribute).
+If `params` is empty when `sort` is called, it is auto-populated before the lookup:
+1. If `defaultfile` is set, `loadconfig[defaultfile]` is attempted (errors are swallowed).
+2. If `params` is still empty after that, the built-in `defaultparams` is used — a single `default` row that sorts by `time` ascending with no attribute.
 
 Config lookup order within `params`:
 1. Rows where `tabname` matches — used directly.
@@ -237,7 +237,7 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-Tests cover: dependency injection, `loadconfig` validation, `applyattr` on valid and missing paths, `sort` with explicit config / default fallback / no-config skip, `manipulate` pass-through, and `postreplay` stub.
+Tests cover: dependency injection, `loadconfig` validation, `applyattr` on valid and missing paths, `sort` with explicit config / `defaultparams` fallback when no config is loaded / `default` row fallback / no-config skip, `manipulate` pass-through, and `postreplay` stub.
 
 ---
 
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 5915529c..113bf4b6 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -12,11 +12,6 @@ defaultparams:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:
 loadconfig:{[file]
   file:hsym file;
   if[null file;file:.z.M.defaultfile];
-  if[null file;
-    dp:.z.M.defaultparams;
-    .z.m.loginfo[`dbwrite;"no sort config file set; using defaultparams"];
-    @[.z.M;`params;:;dp];
-    :[]];
   p:@[
     {.z.m.loginfo[`dbwrite;"retrieving sort settings from ",string x];("SSSB";enlist",")0:x};
     file;
@@ -42,7 +37,11 @@ applyattr:{[dloc;colname;att]
 / sort an on-disk table partition and apply attributes per sort.csv config
 / d: tabname | (tabname;dir) | (tabname;list of dirs)
 sort:{[d]
-  if[0=count .z.M.params;.z.M.loadconfig .z.M.defaultfile];
+  if[0=count select from .z.M.params;
+    @[{.z.M.loadconfig .z.M.defaultfile};`;{[e]:}]];
+  if[0=count select from .z.M.params;
+    dp:select from .z.M.defaultparams;
+    @[.z.M;`params;:;dp]];
   .z.m.loginfo[`dbwrite;"sorting ",(st:string t:first d)," table"];
   sp:$[count tabsp:select from .z.M.params where tabname=t;
       [.z.m.loginfo[`dbwrite;"sort params found for: ",st];tabsp];

From b8b3914e6f155fe1137e7c688728312300c6bc33 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Wed, 3 Jun 2026 14:09:56 +0100
Subject: [PATCH 03/15] Export savedownmanipulation, simplify sort fallback,
 add loadconfig null guard

- Export savedownmanipulation so consumers can register per-table pre-write functions
- loadconfig warns and loads defaultparams when called with a null file rather than silently failing
- sort falls back to defaultparams directly; removes defaultfile indirection from init.q
- Tests: add savedownmanipulation registered function and error recovery cases, loadconfig null case, move all on-disk cleanup to after blocks
- Docs updated throughout to match

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.md | 37 ++++++++++++++++++---------
 di/dbwrite/dbwrite.q  | 12 ++++-----
 di/dbwrite/init.q     |  5 +---
 di/dbwrite/test.csv   | 59 +++++++++++++++++++++++++------------------
 4 files changed, 66 insertions(+), 47 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 4a5d249a..ebb61cd1 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -61,6 +61,7 @@ Sorts `trade` by `sym`, applies `p` to `sym`. Tables not listed fall back to `de
 | `loadconfig[file]` | Load and validate the sort config CSV into module state |
 | `sort[d]` | Sort an on-disk partition and apply attributes per config |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
+| `savedownmanipulation` | Dict mapping table name → unary function; populate before EOD to register pre-write transformations |
 | `manipulate[t;x]` | Apply a registered pre-write manipulation to a table |
 | `postreplay[d;p]` | Post-EOD stub; override to add custom logic |
 | `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
@@ -101,18 +102,18 @@ Loads and validates the sort configuration CSV, storing the result in module sta
 
 | Parameter | Type | Description |
 |---|---|---|
-| `file` | hsym | Path to the sort config CSV. Passing a null symbol (`` ` ``) uses the module-level `defaultfile` value |
+| `file` | hsym | Path to the sort config CSV; pass null (`` ` ``) to warn and load `defaultparams` instead |
 
-**Returns** — generic null on success; throws on failure.
+**Returns** — generic null on success; throws on file/validation failure.
 
 Validation checks that all four required columns (`tabname`, `att`, `column`, `sort`) are present and that all `att` values are within `` ``p`s`g`u ``. Throws a descriptive error for invalid files or unreadable paths.
 
+Passing null warns at `warn` level and loads `defaultparams` — it does not throw.
+
 ```q
 dbwrite.loadconfig[`:config/sort.csv]
 ```
 
-> **Note:** `defaultfile` is a module-level variable (default: null symbol). Set it in `init.q` to configure a path that `sort` will auto-load on first use. If no explicit `loadconfig` call is made and `defaultfile` is not set, `sort` falls back to `defaultparams` automatically.
-
 ---
 
 ### `sort[d]`
@@ -135,11 +136,9 @@ Sorts an on-disk table partition and applies configured attributes.
 
 **Returns** — generic null on success; `()` if no sort config is found for the table.
 
-If `params` is empty when `sort` is called, it is auto-populated before the lookup:
-1. If `defaultfile` is set, `loadconfig[defaultfile]` is attempted (errors are swallowed).
-2. If `params` is still empty after that, the built-in `defaultparams` is used — a single `default` row that sorts by `time` ascending with no attribute.
+If `loadconfig` has not been called before `sort` is first invoked, `sort` automatically uses `defaultparams` — a single `default` row that sorts by `time` ascending with no attribute.
 
-Config lookup order within `params`:
+Config lookup order within the loaded params:
 1. Rows where `tabname` matches — used directly.
 2. Rows where `tabname = \`default` — used with a `warn` log.
 3. No match — warns and returns `()`.
@@ -174,6 +173,19 @@ dbwrite.applyattr[`:hdb/2024.01.02/trade/;`sym;`p]
 
 ---
 
+### `savedownmanipulation`
+
+A dictionary mapping table name (symbol) to a unary manipulation function. Populate this before EOD to register per-table pre-write transformations.
+
+```q
+/ register a manipulation for the trade table
+dbwrite.savedownmanipulation[`trade]:{[x] update sym:`p#sym from x}
+```
+
+Manipulations are called by `manipulate[t;x]`. An empty dict (the default) means no manipulation is applied to any table.
+
+---
+
 ### `manipulate[t;x]`
 
 Applies a registered pre-write manipulation to table `x` of type `t`.
@@ -185,11 +197,12 @@ Applies a registered pre-write manipulation to table `x` of type `t`.
 | `t` | symbol | Table name used to look up the registered manipulation function |
 | `x` | table | Table data to transform |
 
-**Returns** — modified table, or original table unmodified if no manipulation is registered or the function throws.
+**Returns** — modified table, or original table unmodified if no manipulation is registered for `t` or the registered function throws.
 
-Manipulations are registered in the module-internal `savedownmanipulation` dictionary (`` tabname → unary function ``). This dictionary is module-bound and populated by process initialisation code before EOD.
+On error the original table is returned unchanged and the error is logged at `error` level. Register functions in `savedownmanipulation` before calling.
 
 ```q
+dbwrite.savedownmanipulation[`trade]:{[x] update sym:`p#sym from x}
 data:dbwrite.manipulate[`trade;data]
 ```
 
@@ -237,12 +250,12 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-Tests cover: dependency injection, `loadconfig` validation, `applyattr` on valid and missing paths, `sort` with explicit config / `defaultparams` fallback when no config is loaded / `default` row fallback / no-config skip, `manipulate` pass-through, and `postreplay` stub.
+Tests cover: dependency injection, `init` error on missing log dep, `manipulate` pass-through and registered function application and error recovery via `savedownmanipulation`, `postreplay` stub, `sort` with `defaultparams` fallback / explicit config / `default` row fallback / no-match skip, `loadconfig` with null file (warns and loads `defaultparams`) / valid file / unrecognised columns / unrecognised attributes / missing file, `applyattr` on missing and valid paths.
 
 ---
 
 ## Exported symbols
 
 ```q
-export:([init;sort;applyattr;loadconfig;manipulate;postreplay;gc])
+export:([init;sort;applyattr;loadconfig;manipulate;savedownmanipulation;postreplay;gc])
 ```
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 113bf4b6..0f0f9d5b 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -8,10 +8,13 @@ savedownmanipulation:()!();
 defaultparams:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
 
 / load and validate sort.csv into .z.M.params
-/ file: hsym path; null falls back to .z.M.defaultfile
+/ file: hsym path; null warns and loads defaultparams instead
 loadconfig:{[file]
   file:hsym file;
-  if[null file;file:.z.M.defaultfile];
+  if[null file;
+    .z.m.logwarn[`dbwrite;"loadconfig called with no file; using defaultparams"];
+    @[.z.M;`params;:;.z.M.defaultparams];
+    :];
   p:@[
     {.z.m.loginfo[`dbwrite;"retrieving sort settings from ",string x];("SSSB";enlist",")0:x};
     file;
@@ -38,10 +41,7 @@ applyattr:{[dloc;colname;att]
 / d: tabname | (tabname;dir) | (tabname;list of dirs)
 sort:{[d]
   if[0=count select from .z.M.params;
-    @[{.z.M.loadconfig .z.M.defaultfile};`;{[e]:}]];
-  if[0=count select from .z.M.params;
-    dp:select from .z.M.defaultparams;
-    @[.z.M;`params;:;dp]];
+    @[.z.M;`params;:;.z.M.defaultparams]];
   .z.m.loginfo[`dbwrite;"sorting ",(st:string t:first d)," table"];
   sp:$[count tabsp:select from .z.M.params where tabname=t;
       [.z.m.loginfo[`dbwrite;"sort params found for: ",st];tabsp];
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index dcca8127..bd2cd900 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -3,7 +3,4 @@
 
 \l ::dbwrite.q
 
-/ default sort config file - set directly or extend init with a config dep
-defaultfile:`;
-
-export:([init;sort;applyattr;loadconfig;manipulate;postreplay;gc])
\ No newline at end of file
+export:([init;sort;applyattr;loadconfig;manipulate;savedownmanipulation;postreplay;gc])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index 53265fc1..fc1dfee5 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -18,29 +18,40 @@ true,0,0,q,logdep[`error]~.m.di.0dbwrite.logerr,1,1,injected error function stor
 fail,0,0,q,dbwrite.init[(::);(::)],1,1,init without log dep throws an error
 fail,0,0,q,dbwrite.init[(::);`log!(::)],1,1,init with log set to (::) throws an error
 
-/ Test 3: manipulate: error cases
+/ Test 3: manipulate - pass-through when no function registered
 run,0,0,q,"tbl:([]sym:`AAPL`IBM;price:100 200f)",1,,sample table
-true,0,0,q,"tbl~dbwrite.manipulate[`trade;tbl]",1,,unregistered table name
-true,0,0,q,"tbl~dbwrite.manipulate[`;tbl]",1,,null table name
-true,0,0,q,"(0#tbl)~dbwrite.manipulate[`other;0#tbl]",1,,empty table
+true,0,0,q,"tbl~dbwrite.manipulate[`trade;tbl]",1,,unregistered table name returns original
+true,0,0,q,"tbl~dbwrite.manipulate[`;tbl]",1,,null table name returns original
+true,0,0,q,"(0#tbl)~dbwrite.manipulate[`other;0#tbl]",1,,empty table returned unchanged
 
-/ Test 4 postreplay: returns generic null
+/ Test 4: manipulate - registered function is applied via savedownmanipulation
+run,0,0,q,"dbwrite.savedownmanipulation[`trade]:{[x] update price:price*2 from x}",1,,register manipulation function for trade
+true,0,0,q,"(update price:price*2 from tbl)~dbwrite.manipulate[`trade;tbl]",1,,registered function applied to table
+run,0,0,q,"dbwrite.savedownmanipulation[`errortable]:{[x] '`testfail}",1,,register function that throws
+true,0,0,q,"tbl~dbwrite.manipulate[`errortable;tbl]",1,,returns original table when registered function throws
+run,0,0,q,"dbwrite.savedownmanipulation:()!()",1,,clear registered manipulation functions
+
+/ Test 5: postreplay returns generic null
 true,0,0,q,(::)~dbwrite.postreplay[`:hdb;2024.01.01],1,,stub returns generic null
 
-/ Test 5: sort uses defaultparams (sort by time) when params is empty and defaultfile is null
+/ Test 6: sort uses defaultparams (sort by time) when no loadconfig has been called
 run,0,0,q,`:dbwrite_defp_tp/.d set `time`sym,1,,write column order file
 run,0,0,q,`:dbwrite_defp_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
 run,0,0,q,`:dbwrite_defp_tp/sym set `IBM`AAPL,1,,write sym column
 run,0,0,q,"dbwrite.sort[(`anytable;`:dbwrite_defp_tp/)]",1,,sort using defaultparams (empty params + null defaultfile)
 true,0,0,q,(asc exec time from get `:dbwrite_defp_tp/)~exec time from get `:dbwrite_defp_tp/,1,,time column sorted ascending by defaultparams
-run,0,0,q,@[hdel;`:dbwrite_defp_tp/;{}],1,,cleanup test partition
 
-/ Test 6: loadconfig: valid config file is loaded and parsed correctly
+/ Test 7: loadconfig - null file warns and loads defaultparams
+run,0,0,q,logcount:0,1,,reset counter
+run,0,0,q,dbwrite.loadconfig[`],1,,loadconfig with null file
+true,0,0,q,1=logcount,1,,logwarn called once
+
+/ Test 8: loadconfig - valid config file is loaded and parsed correctly
 run,0,0,q,cfgfile:`:dbwrite_test.csv,1,,temp sort config path
 run,0,0,q,"cfgfile 0:(""tabname,att,column,sort"";""trade,p,sym,1"";""trade,,price,0"")",1,,write test sort config
 run,0,0,q,dbwrite.loadconfig[cfgfile],1,,loadconfig with valid csv succeeds
 
-/ Test 6 loadconfig - invalid config files
+/ Test 7 cont: loadconfig - invalid config files
 true,0,0,q,@[dbwrite.loadconfig;`:nonexistent_file_xyz;{1b}],1,,nonexistent file throws
 run,0,0,q,badcols:`:dbwrite_badcols.csv,1,,
 run,0,0,q,"badcols 0:(""wrongcol,att,column,sort"";""trade,p,sym,1"")",1,,csv with unrecognised column name
@@ -49,18 +60,12 @@ run,0,0,q,badatt:`:dbwrite_badatt.csv,1,,
 run,0,0,q,"badatt 0:(""tabname,att,column,sort"";""trade,z,sym,1"")",1,,csv with unrecognised attribute
 true,0,0,q,@[dbwrite.loadconfig;badatt;{1b}],1,,unrecognised attribute throws
 
-/ Clean-up config files after loadconfig tests
-run,0,0,q,@[hdel;cfgfile;{}],1,,remove temp sort config
-run,0,0,q,@[hdel;badcols;{}],1,,remove temp bad columns file
-run,0,0,q,@[hdel;badatt;{}],1,,remove temp bad attribute file
-comment,,,,,,,--- manipulate: registered function path not testable (savedownmanipulation in .z.M is module-bound and not settable from test scope) ---
-
-/ Test 7: Applyattr - error cases
+/ Test 9: applyattr - error cases
 run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:nonexist_attr_dir/;`sym;`p]",1,,apply to non-existent path
 true,0,0,q,2=logcount,1,,loginfo (before attempt) and logerr (on failure) each called once
 
-/ Test 8 - Applyattr - attribute application and logging when config entry exists for table
+/ Test 10: applyattr - attribute application and logging when path exists
 run,0,0,q,`:dbwrite_attr_tp/.d set `sym`price,1,,write column order file
 run,0,0,q,`:dbwrite_attr_tp/sym set `IBM`AAPL`MSFT,1,,write sym column
 run,0,0,q,`:dbwrite_attr_tp/price set 200 100 300f,1,,write price column
@@ -68,23 +73,21 @@ run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`p]",1,,apply p attr to sym column
 true,0,0,q,1=logcount,1,,loginfo called once (no error)
 true,0,0,q,`p=attr get `:dbwrite_attr_tp/sym,1,,p attribute applied on disk
-run,0,0,q,@[hdel;`:dbwrite_attr_tp/;{}],1,,cleanup test partition
 
-/ Test 9 - sort: error cases
+/ Test 11: sort - skip when table not in config
 run,0,0,q,logcount:0,1,,reset counter
 true,0,0,q,()~dbwrite.sort[`no_params_table_xyz],1,,returns () when table not in config
 true,0,0,q,2=logcount,1,,loginfo (sorting start) and logwarn (skip) both called
 
-/ Test 10 - sort: sorting and logging when config entry exists for table
+/ Test 12: sort - sorting and attribute application when config entry exists for table
 run,0,0,q,`:dbwrite_sort_tp/.d set `sym`price,1,,write column order file
 run,0,0,q,`:dbwrite_sort_tp/sym set `IBM`AAPL`MSFT,1,,write unsorted sym column
 run,0,0,q,`:dbwrite_sort_tp/price set 200 100 300f,1,,write price column
 run,0,0,q,"dbwrite.sort[(`trade;`:dbwrite_sort_tp/)]",1,,sort trade table in test partition
 true,0,0,q,`AAPL`IBM`MSFT~exec sym from get `:dbwrite_sort_tp/,1,,sorted ascending by sym
 true,0,0,q,`p=attr get `:dbwrite_sort_tp/sym,1,,p attribute applied to sym on disk
-run,0,0,q,@[hdel;`:dbwrite_sort_tp/;{}],1,,cleanup sort test partition
 
-/ Test 11 - sort: sorting with default config entry when no specific entry for table
+/ Test 13: sort - default config row used as fallback when no specific entry for table
 run,0,0,q,defcfg:`:dbwrite_defcfg.csv,1,,
 run,0,0,q,"defcfg 0:(""tabname,att,column,sort"";""default,,sym,1"")",1,,config with default entry only
 run,0,0,q,dbwrite.loadconfig[defcfg],1,,load default-only config
@@ -96,6 +99,12 @@ run,0,0,q,"dbwrite.sort[(`othertable;`:dbwrite_def_tp/)]",1,,sort using default
 true,0,0,q,`AAPL`IBM~exec sym from get `:dbwrite_def_tp/,1,,sorted ascending by sym via default params
 true,0,0,q,4=logcount,1,,loginfo x2 (sorting start + sort-by) + logwarn x1 (using defaults) + loginfo x1 (finished)
 
-/ Clean-up default config and test partition
-after,0,0,q,@[hdel;`:dbwrite_def_tp/;{}],1,,cleanup test partition
-after,0,0,q,@[hdel;defcfg;{}],1,,remove default config csv
\ No newline at end of file
+/ Clean up all created on-disk files and directories
+after,0,0,q,@[hdel;`:dbwrite_defp_tp/;{}],1,,cleanup defaultparams test partition
+after,0,0,q,@[hdel;cfgfile;{}],1,,remove temp sort config
+after,0,0,q,@[hdel;badcols;{}],1,,remove temp bad columns file
+after,0,0,q,@[hdel;badatt;{}],1,,remove temp bad attribute file
+after,0,0,q,@[hdel;`:dbwrite_attr_tp/;{}],1,,cleanup attr test partition
+after,0,0,q,@[hdel;`:dbwrite_sort_tp/;{}],1,,cleanup sort test partition
+after,0,0,q,@[hdel;`:dbwrite_def_tp/;{}],1,,cleanup default config test partition
+after,0,0,q,@[hdel;defcfg;{}],1,,remove default config csv

From 4c0099e16943188a3e0cbaca6b9e3cc58d734bad Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Wed, 3 Jun 2026 15:28:36 +0100
Subject: [PATCH 04/15] Add null/type guards, comprehensive edge-case tests,
 fix sort empty-input crash

- loadconfig: explicit symbol type check with clear error before null check
- applyattr: guard against invalid att (must be in `p`s`g`u), null colname guard
- sort: type check for d (must be symbol or list); empty list returns () safely
- test.csv: 78 tests covering wrong types, nulls, empty inputs across all functions

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.md |   9 ++--
 di/dbwrite/dbwrite.q  | 103 +++++++++++++++++++++++-------------------
 di/dbwrite/test.csv   |  88 ++++++++++++++++++++++--------------
 3 files changed, 115 insertions(+), 85 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index ebb61cd1..651ee3b2 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -61,7 +61,7 @@ Sorts `trade` by `sym`, applies `p` to `sym`. Tables not listed fall back to `de
 | `loadconfig[file]` | Load and validate the sort config CSV into module state |
 | `sort[d]` | Sort an on-disk partition and apply attributes per config |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
-| `savedownmanipulation` | Dict mapping table name → unary function; populate before EOD to register pre-write transformations |
+| `savedownmanipulation` | Dict mapping table name → unary function; amend to register pre-write transformations |
 | `manipulate[t;x]` | Apply a registered pre-write manipulation to a table |
 | `postreplay[d;p]` | Post-EOD stub; override to add custom logic |
 | `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
@@ -175,14 +175,13 @@ dbwrite.applyattr[`:hdb/2024.01.02/trade/;`sym;`p]
 
 ### `savedownmanipulation`
 
-A dictionary mapping table name (symbol) to a unary manipulation function. Populate this before EOD to register per-table pre-write transformations.
+A dictionary mapping table name (symbol) to a unary manipulation function. Amend this before EOD to register per-table pre-write transformations.
 
 ```q
-/ register a manipulation for the trade table
 dbwrite.savedownmanipulation[`trade]:{[x] update sym:`p#sym from x}
 ```
 
-Manipulations are called by `manipulate[t;x]`. An empty dict (the default) means no manipulation is applied to any table.
+Manipulations are applied by `manipulate[t;x]`. An empty dict (the default) means no manipulation is applied to any table.
 
 ---
 
@@ -250,7 +249,7 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-Tests cover: dependency injection, `init` error on missing log dep, `manipulate` pass-through and registered function application and error recovery via `savedownmanipulation`, `postreplay` stub, `sort` with `defaultparams` fallback / explicit config / `default` row fallback / no-match skip, `loadconfig` with null file (warns and loads `defaultparams`) / valid file / unrecognised columns / unrecognised attributes / missing file, `applyattr` on missing and valid paths.
+Tests cover: dependency injection, `init` error on missing log dep, `manipulate` pass-through and registered function application and error recovery via `savedownmanipulation`, visibility of registrations, `postreplay` stub, `sort` with `defaultparams` fallback / explicit config / `default` row fallback / no-match skip, `loadconfig` with null file (warns and loads `defaultparams`) / valid file / unrecognised columns / unrecognised attributes / missing file, `applyattr` on missing and valid paths.
 
 ---
 
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 0f0f9d5b..fbb1df78 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -1,64 +1,72 @@
-/ sort params table - populated by loadconfig
-params:([] tabname:`symbol$(); att:`symbol$(); column:`symbol$(); sort:`boolean$());
+/ sort params table - default row sorts all tables by time ascending
+params:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
 
 / save-down manipulation registry: tabname -> unary function
 savedownmanipulation:()!();
 
-/ default sort params - used when params is empty and no config file is set
-defaultparams:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
-
 / load and validate sort.csv into .z.M.params
-/ file: hsym path; null warns and loads defaultparams instead
+/ file: hsym path; null warns and resets params to default row
 loadconfig:{[file]
-  file:hsym file;
+  if[not -11h=type file;
+    '"loadconfig: file must be a symbol, got type ",(string type file)];
   if[null file;
-    .z.m.logwarn[`dbwrite;"loadconfig called with no file; using defaultparams"];
-    @[.z.M;`params;:;.z.M.defaultparams];
-    :];
-  p:@[
-    {.z.m.loginfo[`dbwrite;"retrieving sort settings from ",string x];("SSSB";enlist",")0:x};
-    file;
-    {[f;e]'"failed to open ",string[f],": ",e}[file]
-  ];
-  if[not all spcb:(spc:cols p) in `tabname`att`column`sort;
-    '"unrecognised columns (",(", " sv string spc where not spcb),") in ",string file];
-  if[not all atb:(at:distinct p`att) in ``p`s`g`u;
-    '"unrecognised attribute(s): ",", " sv string at where not atb];
-  @[.z.M;`params;:;p];
+    .z.m.logwarn[`dbwrite;"loadconfig called with no file; resetting params to default"];
+    @[.z.M;`params;:;([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b)]];
+  if[not null file;
+    file:hsym file;
+    p:@[
+      {.z.m.loginfo[`dbwrite;"retrieving sort settings from ",string x];("SSSB";enlist",")0:x};
+      file;
+      {[f;e]'"failed to open ",string[f],": ",e}[file]
+    ];
+    if[not all spcb:(spc:cols p) in `tabname`att`column`sort;
+      '"unrecognised columns (",(", " sv string spc where not spcb),") in ",string file];
+    if[not all atb:(at:distinct p`att) in ``p`s`g`u;
+      '"unrecognised attribute(s): ",", " sv string at where not atb];
+    @[.z.M;`params;:;p]];
   };
 
 / apply a single kdb+ attribute to an on-disk column; logs and swallows errors
 applyattr:{[dloc;colname;att]
   .z.m.loginfo[`dbwrite;"applying ",string[att]," attr to ",string[colname]," in ",string dloc];
-  .[{@[x;y;z#]};(dloc;colname;att);
-    {[dloc;colname;att;e]
-      .z.m.logerr[`dbwrite;"unable to apply ",string[att]," attr to ",string[colname]," in ",string[dloc],": ",e]
-    }[dloc;colname;att]
-  ];
+  if[null colname;
+    .z.m.logerr[`dbwrite;"applyattr called with null column name in ",string dloc]];
+  if[not null colname;
+    $[not att in `p`s`g`u;
+      .z.m.logerr[`dbwrite;"applyattr: invalid attribute ",string[att]," for ",string[colname]," in ",string dloc];
+      .[{@[x;y;z#]};(dloc;colname;att);
+        {[dloc;colname;att;e]
+          .z.m.logerr[`dbwrite;"unable to apply ",string[att]," attr to ",string[colname]," in ",string[dloc],": ",e]
+        }[dloc;colname;att]
+      ]]];
   };
 
 / sort an on-disk table partition and apply attributes per sort.csv config
 / d: tabname | (tabname;dir) | (tabname;list of dirs)
 sort:{[d]
-  if[0=count select from .z.M.params;
-    @[.z.M;`params;:;.z.M.defaultparams]];
-  .z.m.loginfo[`dbwrite;"sorting ",(st:string t:first d)," table"];
-  sp:$[count tabsp:select from .z.M.params where tabname=t;
-      [.z.m.loginfo[`dbwrite;"sort params found for: ",st];tabsp];
-    count defsp:select from .z.M.params where tabname=`default;
-      [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; using defaults"];defsp];
-    [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; skipping sort"];:()]];
-  {[sp;dloc]
-    if[count sortcols:exec column from sp where sort,not null column;
-      .z.m.loginfo[`dbwrite;"sorting ",string[dloc]," by: ",", " sv string sortcols];
-      .[xasc;(sortcols;dloc);
-        {[sc;dl;e]
-          .z.m.logerr[`dbwrite;"failed to sort ",string[dl]," by ",(", " sv string sc),": ",e]
-        }[sortcols;dloc]]];
-    if[count attrcols:select column,att from sp where not null att;
-      .z.M.applyattr[dloc;;]'[attrcols`column;attrcols`att]];
-  }[sp] each distinct (),last d;
-  .z.m.loginfo[`dbwrite;"finished sorting ",st," table"];
+  $[not count d;
+    ();
+    not (type d) in -11 0 11h;
+    [.z.m.logerr[`dbwrite;"sort: d must be a symbol or list, got type ",(string type d)];()];
+    [
+      .z.m.loginfo[`dbwrite;"sorting ",(st:string t:first d)," table"];
+      sp:$[count tabsp:select from .z.M.params where tabname=t;
+          [.z.m.loginfo[`dbwrite;"sort params found for: ",st];tabsp];
+        count defsp:select from .z.M.params where tabname=`default;
+          [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; using defaults"];defsp];
+        [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; skipping sort"];:()]];
+      {[sp;dloc]
+        if[count sortcols:exec column from sp where sort,not null column;
+          .z.m.loginfo[`dbwrite;"sorting ",string[dloc]," by: ",", " sv string sortcols];
+          .[xasc;(sortcols;dloc);
+            {[sc;dl;e]
+              .z.m.logerr[`dbwrite;"failed to sort ",string[dl]," by ",(", " sv string sc),": ",e]
+            }[sortcols;dloc]]];
+        if[count attrcols:select column,att from sp where not null att;
+          .z.M.applyattr[dloc;;]'[attrcols`column;attrcols`att]];
+      }[sp] each distinct (),last d;
+      .z.m.loginfo[`dbwrite;"finished sorting ",st," table"]
+    ]]
   };
 
 / apply registered pre-write manipulation to table x of type t
@@ -76,7 +84,7 @@ manipulate:{[t;x]
 postreplay:{[d;p]};
 
 / format current process memory stats as a loggable string
-memstats:{"mem stats: ",{"; "sv "=" sv'flip(string key x;(string value x),\:" MB")}`long$.Q.w[]%1048576};
+memstats:{[]"mem stats: ",{"; "sv "=" sv'flip(string key x;(string value x),\:" MB")}`long$.Q.w[]%1048576};
 
 / run .Q.gc[] and log before/after memory stats
 gc:{
@@ -86,9 +94,10 @@ gc:{
   };
 
 init:{[config;deps]
+  / config: dict with optional keys
+  /   `savedownmanipulation: tabname!function dict of pre-write manipulation functions
   / deps: `log!(logdict)
   /   `log: `info`warn`error!(infofunc;warnfunc;errfunc) - required
-  / example: dbwrite.init[enlist[`log]!enlist logdep]
   logdict:$[99h=type deps;$[(`log in key deps) and not (::)~deps`log;deps`log;()!()];()!()];
   if[not count logdict;
     '"di.dbwrite: log dependency is required; pass `info`warn`error functions - see di.log or refer to confluence documentation";
@@ -96,4 +105,4 @@ init:{[config;deps]
   .z.m.loginfo:logdict`info;
   .z.m.logwarn:logdict`warn;
   .z.m.logerr:logdict`error;
-  };
\ No newline at end of file
+  };
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index fc1dfee5..6eec3123 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -18,68 +18,86 @@ true,0,0,q,logdep[`error]~.m.di.0dbwrite.logerr,1,1,injected error function stor
 fail,0,0,q,dbwrite.init[(::);(::)],1,1,init without log dep throws an error
 fail,0,0,q,dbwrite.init[(::);`log!(::)],1,1,init with log set to (::) throws an error
 
-/ Test 3: manipulate - pass-through when no function registered
+/ Test 3: manipulate - pass-through cases (no functions registered)
 run,0,0,q,"tbl:([]sym:`AAPL`IBM;price:100 200f)",1,,sample table
 true,0,0,q,"tbl~dbwrite.manipulate[`trade;tbl]",1,,unregistered table name returns original
 true,0,0,q,"tbl~dbwrite.manipulate[`;tbl]",1,,null table name returns original
 true,0,0,q,"(0#tbl)~dbwrite.manipulate[`other;0#tbl]",1,,empty table returned unchanged
+true,0,0,q,42~dbwrite.manipulate[`notregistered;42],1,,non-table input returns unchanged when unregistered
 
-/ Test 4: manipulate - registered function is applied via savedownmanipulation
-run,0,0,q,"dbwrite.savedownmanipulation[`trade]:{[x] update price:price*2 from x}",1,,register manipulation function for trade
-true,0,0,q,"(update price:price*2 from tbl)~dbwrite.manipulate[`trade;tbl]",1,,registered function applied to table
-run,0,0,q,"dbwrite.savedownmanipulation[`errortable]:{[x] '`testfail}",1,,register function that throws
-true,0,0,q,"tbl~dbwrite.manipulate[`errortable;tbl]",1,,returns original table when registered function throws
-run,0,0,q,"dbwrite.savedownmanipulation:()!()",1,,clear registered manipulation functions
-
-/ Test 5: postreplay returns generic null
+/ Test 4: postreplay returns generic null
 true,0,0,q,(::)~dbwrite.postreplay[`:hdb;2024.01.01],1,,stub returns generic null
+true,0,0,q,(::)~dbwrite.postreplay[`;0Nd],1,,null args return generic null
 
-/ Test 6: sort uses defaultparams (sort by time) when no loadconfig has been called
+/ Test 5: sort uses default row when table name not in config
 run,0,0,q,`:dbwrite_defp_tp/.d set `time`sym,1,,write column order file
 run,0,0,q,`:dbwrite_defp_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
 run,0,0,q,`:dbwrite_defp_tp/sym set `IBM`AAPL,1,,write sym column
-run,0,0,q,"dbwrite.sort[(`anytable;`:dbwrite_defp_tp/)]",1,,sort using defaultparams (empty params + null defaultfile)
-true,0,0,q,(asc exec time from get `:dbwrite_defp_tp/)~exec time from get `:dbwrite_defp_tp/,1,,time column sorted ascending by defaultparams
+run,0,0,q,"dbwrite.sort[(`anytable;`:dbwrite_defp_tp/)]",1,,sort using default row
+true,0,0,q,(asc exec time from get `:dbwrite_defp_tp/)~exec time from get `:dbwrite_defp_tp/,1,,time column sorted ascending by default row
 
-/ Test 7: loadconfig - null file warns and loads defaultparams
-run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,dbwrite.loadconfig[`],1,,loadconfig with null file
-true,0,0,q,1=logcount,1,,logwarn called once
+/ Test 6: loadconfig null resets params to default row
+run,0,0,q,dbwrite.loadconfig[`],1,,call loadconfig with null file
+true,0,0,q,.m.di.0dbwrite.params~([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b),1,,params reset to default row
 
-/ Test 8: loadconfig - valid config file is loaded and parsed correctly
+/ Test 7: loadconfig - valid config file loaded and parsed correctly
 run,0,0,q,cfgfile:`:dbwrite_test.csv,1,,temp sort config path
 run,0,0,q,"cfgfile 0:(""tabname,att,column,sort"";""trade,p,sym,1"";""trade,,price,0"")",1,,write test sort config
 run,0,0,q,dbwrite.loadconfig[cfgfile],1,,loadconfig with valid csv succeeds
+true,0,0,q,2=count .m.di.0dbwrite.params,1,,two rows loaded into params
+true,0,0,q,all `trade=exec tabname from .m.di.0dbwrite.params,1,,both rows are for trade table
 
-/ Test 7 cont: loadconfig - invalid config files
+/ Test 7 cont: loadconfig - invalid and edge-case inputs
 true,0,0,q,@[dbwrite.loadconfig;`:nonexistent_file_xyz;{1b}],1,,nonexistent file throws
+true,0,0,q,@[dbwrite.loadconfig;42;{1b}],1,,non-symbol arg throws
 run,0,0,q,badcols:`:dbwrite_badcols.csv,1,,
 run,0,0,q,"badcols 0:(""wrongcol,att,column,sort"";""trade,p,sym,1"")",1,,csv with unrecognised column name
 true,0,0,q,@[dbwrite.loadconfig;badcols;{1b}],1,,unrecognised column throws
 run,0,0,q,badatt:`:dbwrite_badatt.csv,1,,
 run,0,0,q,"badatt 0:(""tabname,att,column,sort"";""trade,z,sym,1"")",1,,csv with unrecognised attribute
 true,0,0,q,@[dbwrite.loadconfig;badatt;{1b}],1,,unrecognised attribute throws
+run,0,0,q,emptycfg:`:dbwrite_empty.csv,1,,
+run,0,0,q,"emptycfg 0:enlist""tabname,att,column,sort""",1,,csv with header only
+run,0,0,q,dbwrite.loadconfig[emptycfg],1,,loadconfig with header-only csv
+true,0,0,q,0=count .m.di.0dbwrite.params,1,,params is empty after header-only csv
 
-/ Test 9: applyattr - error cases
+/ Test 8: applyattr - error cases
 run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:nonexist_attr_dir/;`sym;`p]",1,,apply to non-existent path
 true,0,0,q,2=logcount,1,,loginfo (before attempt) and logerr (on failure) each called once
 
-/ Test 10: applyattr - attribute application and logging when path exists
+/ Test 8 cont: applyattr - null column name logs error and does not throw
 run,0,0,q,`:dbwrite_attr_tp/.d set `sym`price,1,,write column order file
 run,0,0,q,`:dbwrite_attr_tp/sym set `IBM`AAPL`MSFT,1,,write sym column
 run,0,0,q,`:dbwrite_attr_tp/price set 200 100 300f,1,,write price column
 run,0,0,q,logcount:0,1,,reset counter
+run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`;`p]",1,,apply with null column name
+true,0,0,q,2=logcount,1,,loginfo and logerr both called for null column
+
+/ Test 8 cont: applyattr - invalid att logs error
+run,0,0,q,logcount:0,1,,reset counter
+run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`z]",1,,apply with invalid attribute
+true,0,0,q,2=logcount,1,,loginfo and logerr both called for invalid att
+
+/ Test 9: applyattr - attribute applied and logged when path exists
+run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`p]",1,,apply p attr to sym column
 true,0,0,q,1=logcount,1,,loginfo called once (no error)
 true,0,0,q,`p=attr get `:dbwrite_attr_tp/sym,1,,p attribute applied on disk
 
-/ Test 11: sort - skip when table not in config
+/ Test 10: sort - skip when table not in config and no default row
+run,0,0,q,dbwrite.loadconfig[cfgfile],1,,reload trade-only config (no default row)
 run,0,0,q,logcount:0,1,,reset counter
 true,0,0,q,()~dbwrite.sort[`no_params_table_xyz],1,,returns () when table not in config
 true,0,0,q,2=logcount,1,,loginfo (sorting start) and logwarn (skip) both called
+true,0,0,q,()~dbwrite.sort[()],1,,empty list returns ()
+
+/ Test 10 cont: sort - wrong type logs error and returns ()
+run,0,0,q,logcount:0,1,,reset counter
+true,0,0,q,()~dbwrite.sort[42],1,,wrong type (long) returns ()
+true,0,0,q,1=logcount,1,,logerr called once for wrong type
 
-/ Test 12: sort - sorting and attribute application when config entry exists for table
+/ Test 11: sort - sorting and attribute application when config entry exists for table
 run,0,0,q,`:dbwrite_sort_tp/.d set `sym`price,1,,write column order file
 run,0,0,q,`:dbwrite_sort_tp/sym set `IBM`AAPL`MSFT,1,,write unsorted sym column
 run,0,0,q,`:dbwrite_sort_tp/price set 200 100 300f,1,,write price column
@@ -87,24 +105,28 @@ run,0,0,q,"dbwrite.sort[(`trade;`:dbwrite_sort_tp/)]",1,,sort trade table in tes
 true,0,0,q,`AAPL`IBM`MSFT~exec sym from get `:dbwrite_sort_tp/,1,,sorted ascending by sym
 true,0,0,q,`p=attr get `:dbwrite_sort_tp/sym,1,,p attribute applied to sym on disk
 
-/ Test 13: sort - default config row used as fallback when no specific entry for table
-run,0,0,q,defcfg:`:dbwrite_defcfg.csv,1,,
-run,0,0,q,"defcfg 0:(""tabname,att,column,sort"";""default,,sym,1"")",1,,config with default entry only
-run,0,0,q,dbwrite.loadconfig[defcfg],1,,load default-only config
-run,0,0,q,`:dbwrite_def_tp/.d set `sym`price,1,,write column order file
-run,0,0,q,`:dbwrite_def_tp/sym set `IBM`AAPL,1,,write unsorted sym column
-run,0,0,q,`:dbwrite_def_tp/price set 200 100f,1,,write price column
+/ Test 12: sort - default row used after resetting params with loadconfig null
+run,0,0,q,dbwrite.loadconfig[`],1,,reset params to default row via null loadconfig
+run,0,0,q,`:dbwrite_reset_tp/.d set `time`sym,1,,write column order file
+run,0,0,q,`:dbwrite_reset_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
+run,0,0,q,`:dbwrite_reset_tp/sym set `IBM`AAPL,1,,write sym column
+run,0,0,q,`:dbwrite_reset_tp/price set 200 100f,1,,write price column
 run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,"dbwrite.sort[(`othertable;`:dbwrite_def_tp/)]",1,,sort using default fallback
-true,0,0,q,`AAPL`IBM~exec sym from get `:dbwrite_def_tp/,1,,sorted ascending by sym via default params
+run,0,0,q,"dbwrite.sort[(`othertable;`:dbwrite_reset_tp/)]",1,,sort using default row
+true,0,0,q,(asc exec time from get `:dbwrite_reset_tp/)~exec time from get `:dbwrite_reset_tp/,1,,sorted ascending by time via default row
 true,0,0,q,4=logcount,1,,loginfo x2 (sorting start + sort-by) + logwarn x1 (using defaults) + loginfo x1 (finished)
 
+/ Test 13: gc calls loginfo twice and does not throw
+run,0,0,q,logcount:0,1,,reset counter
+run,0,0,q,dbwrite.gc[],1,,run garbage collect
+true,0,0,q,2=logcount,1,,loginfo called twice (start and end)
+
 / Clean up all created on-disk files and directories
 after,0,0,q,@[hdel;`:dbwrite_defp_tp/;{}],1,,cleanup defaultparams test partition
 after,0,0,q,@[hdel;cfgfile;{}],1,,remove temp sort config
 after,0,0,q,@[hdel;badcols;{}],1,,remove temp bad columns file
 after,0,0,q,@[hdel;badatt;{}],1,,remove temp bad attribute file
+after,0,0,q,@[hdel;emptycfg;{}],1,,remove empty config file
 after,0,0,q,@[hdel;`:dbwrite_attr_tp/;{}],1,,cleanup attr test partition
 after,0,0,q,@[hdel;`:dbwrite_sort_tp/;{}],1,,cleanup sort test partition
-after,0,0,q,@[hdel;`:dbwrite_def_tp/;{}],1,,cleanup default config test partition
-after,0,0,q,@[hdel;defcfg;{}],1,,remove default config csv
+after,0,0,q,@[hdel;`:dbwrite_reset_tp/;{}],1,,cleanup reset test partition

From c2ac97181ff43a0e6532dbc99d71acea8dab5718 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Wed, 3 Jun 2026 15:40:32 +0100
Subject: [PATCH 05/15] Update dbwrite.md: fix stale defaultparams references,
 add mock-logging test setup

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.md | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 651ee3b2..1b84c9c0 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -11,7 +11,7 @@ Sort, attribute application, save-down manipulation, and garbage-collection util
 - Register per-table pre-write manipulation functions applied before save-down
 - Run `.Q.gc[]` with before/after memory logging
 - Sort and attribute behaviour driven by a CSV config file; a `default` row acts as a fallback
-- Built-in `defaultparams` provides an out-of-the-box fallback (sort by `time` ascending) when no config file is loaded
+- A built-in `default` row in `params` provides an out-of-the-box fallback (sort by `time` ascending) when no config file is loaded
 - All errors from sort, attribute application, and manipulation are caught and logged — they do not propagate
 
 ---
@@ -102,13 +102,13 @@ Loads and validates the sort configuration CSV, storing the result in module sta
 
 | Parameter | Type | Description |
 |---|---|---|
-| `file` | hsym | Path to the sort config CSV; pass null (`` ` ``) to warn and load `defaultparams` instead |
+| `file` | hsym | Path to the sort config CSV; pass null (`` ` ``) to warn and reset `params` to the default row |
 
 **Returns** — generic null on success; throws on file/validation failure.
 
 Validation checks that all four required columns (`tabname`, `att`, `column`, `sort`) are present and that all `att` values are within `` ``p`s`g`u ``. Throws a descriptive error for invalid files or unreadable paths.
 
-Passing null warns at `warn` level and loads `defaultparams` — it does not throw.
+Passing null warns at `warn` level and resets `params` to the default row — it does not throw.
 
 ```q
 dbwrite.loadconfig[`:config/sort.csv]
@@ -136,7 +136,7 @@ Sorts an on-disk table partition and applies configured attributes.
 
 **Returns** — generic null on success; `()` if no sort config is found for the table.
 
-If `loadconfig` has not been called before `sort` is first invoked, `sort` automatically uses `defaultparams` — a single `default` row that sorts by `time` ascending with no attribute.
+If `loadconfig` has not been called before `sort` is first invoked, `sort` automatically uses the built-in `default` row in `params` — sorts by `time` ascending with no attribute.
 
 Config lookup order within the loaded params:
 1. Rows where `tabname` matches — used directly.
@@ -249,7 +249,20 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-Tests cover: dependency injection, `init` error on missing log dep, `manipulate` pass-through and registered function application and error recovery via `savedownmanipulation`, visibility of registrations, `postreplay` stub, `sort` with `defaultparams` fallback / explicit config / `default` row fallback / no-match skip, `loadconfig` with null file (warns and loads `defaultparams`) / valid file / unrecognised columns / unrecognised attributes / missing file, `applyattr` on missing and valid paths.
+The test suite uses mock logging (no `di.log` dependency required). The mock wires up three no-op counters so log call counts can be asserted:
+
+```q
+dbwrite:use`di.dbwrite
+logcount:0
+loginfo:{[c;m] logcount::logcount+1}
+logwarn:{[c;m] logcount::logcount+1}
+logerr:{[c;m] logcount::logcount+1}
+logdep:`info`warn`error!(loginfo;logwarn;logerr)
+deps:(enlist`log)!enlist logdep
+dbwrite.init[(::);deps]
+```
+
+Tests cover: dependency injection, `init` error on missing log dep, `manipulate` pass-through and registered function application and error recovery via `savedownmanipulation`, `postreplay` stub, `sort` with default row fallback / explicit config / `default` row fallback / no-match skip / empty input / wrong type, `loadconfig` with null file (warns and resets to default row) / valid file / unrecognised columns / unrecognised attributes / missing file / header-only file, `applyattr` on missing path / null column / invalid attribute / valid path, `gc` log count.
 
 ---
 

From 931db34d75eb6f0e1070af301140d857e184e740 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Fri, 19 Jun 2026 11:51:18 +0100
Subject: [PATCH 06/15] Add savedown and upsert, remove TorQ-specific features

- Add savedown[dir;part;tabname;data]: enumerates syms, applies p#sym,
  writes partition via .Q.par + set, then calls sort
- Add upsert[dir;part;tabname;data]: appends to existing partition via
  functional amend, then re-sorts
- Remove manipulate, savedownmanipulation, postreplay (TorQ-influenced)
- Update export dict, init comment, tests, and docs accordingly

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.md | 121 +++++++++++++++++++-----------------------
 di/dbwrite/dbwrite.q  |  38 +++++++------
 di/dbwrite/init.q     |   2 +-
 di/dbwrite/test.csv   |  61 ++++++++++++---------
 4 files changed, 113 insertions(+), 109 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 1b84c9c0..eaea7ed1 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -1,18 +1,19 @@
 # di.dbwrite
 
-Sort, attribute application, save-down manipulation, and garbage-collection utilities for kdb+ processes that persist data to disk (RDB, WDB, TickerLogReplay).
+Write, sort, and attribute utilities for kdb+ processes that persist data to disk.
 
 ---
 
 ## Features
 
+- Write in-memory tables to a date-partitioned HDB with `savedown` — enumerates syms, applies `p#` to `sym`, writes, then sorts
+- Append rows to an existing partition with `upsert` — enumerates syms, appends, then re-sorts
 - Sort on-disk table partitions by configured columns using `xasc`
 - Apply kdb+ attributes (`p`, `s`, `g`, `u`) to on-disk columns after sort
-- Register per-table pre-write manipulation functions applied before save-down
-- Run `.Q.gc[]` with before/after memory logging
 - Sort and attribute behaviour driven by a CSV config file; a `default` row acts as a fallback
 - A built-in `default` row in `params` provides an out-of-the-box fallback (sort by `time` ascending) when no config file is loaded
-- All errors from sort, attribute application, and manipulation are caught and logged — they do not propagate
+- Run `.Q.gc[]` with before/after memory logging
+- All errors from sort, attribute application, and write operations are either caught-and-logged (sort, applyattr) or propagated to the caller (savedown, upsert)
 
 ---
 
@@ -34,7 +35,7 @@ The `log` dependency must be passed to `init`. The module throws if it is absent
 |---|---|---|
 | `tabname` | symbol | Table name, or `` `default `` as a catch-all fallback |
 | `att` | symbol | kdb+ attribute to apply: `p`, `s`, `g`, `u`, or empty for none |
-| `column` | symbol | Column to sort or attribute; empty means attribute-only (no sort contribution) |
+| `column` | symbol | Column to sort or attribute |
 | `sort` | boolean | `1b` — include in `xasc` sort key; `0b` — attribute only |
 
 Example `sort.csv`:
@@ -58,12 +59,11 @@ Sorts `trade` by `sym`, applies `p` to `sym`. Tables not listed fall back to `de
 | Function | Description |
 |---|---|
 | `init[config;deps]` | Wire injected dependencies; must be called first |
+| `savedown[dir;part;tabname;data]` | Write in-memory table to HDB partition, enumerate syms, apply `p#sym`, then sort |
+| `upsert[dir;part;tabname;data]` | Append rows to existing partition, enumerate syms, then re-sort |
 | `loadconfig[file]` | Load and validate the sort config CSV into module state |
 | `sort[d]` | Sort an on-disk partition and apply attributes per config |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
-| `savedownmanipulation` | Dict mapping table name → unary function; amend to register pre-write transformations |
-| `manipulate[t;x]` | Apply a registered pre-write manipulation to a table |
-| `postreplay[d;p]` | Post-EOD stub; override to add custom logic |
 | `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
 
 ---
@@ -94,6 +94,50 @@ dbwrite.init[(::);(enlist`log)!enlist logdep]
 
 ---
 
+### `savedown[dir;part;tabname;data]`
+
+Writes an in-memory table to a date-partitioned HDB. Enumerates symbol columns against the HDB sym file, applies `p#` to `sym` if present, writes the partition, then calls `sort` to sort and apply attributes per the loaded config.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `dir` | hsym | HDB root directory (e.g. `` `:hdb ``) |
+| `part` | date/month/int | Partition value |
+| `tabname` | symbol | Table name — determines the partition subdirectory |
+| `data` | table | In-memory table to write |
+
+**Returns** — generic null on success; throws on write failure.
+
+If `loadconfig` has not been called, the built-in default row applies (sort by `time` ascending). If the table has no `sym` column, enumeration and `p#` are skipped.
+
+```q
+dbwrite.savedown[`:hdb;2024.01.02;`trade;data]
+```
+
+---
+
+### `upsert[dir;part;tabname;data]`
+
+Appends rows to an existing on-disk partition then re-sorts. Enumerates symbol columns before appending. The partition must already exist — use `savedown` for the initial write.
+
+**Parameters**
+
+| Parameter | Type | Description |
+|---|---|---|
+| `dir` | hsym | HDB root directory |
+| `part` | date/month/int | Partition value |
+| `tabname` | symbol | Table name |
+| `data` | table | Rows to append |
+
+**Returns** — generic null on success; throws if the partition does not exist or on write failure.
+
+```q
+dbwrite.upsert[`:hdb;2024.01.02;`trade;latedata]
+```
+
+---
+
 ### `loadconfig[file]`
 
 Loads and validates the sort configuration CSV, storing the result in module state for use by `sort`.
@@ -136,8 +180,6 @@ Sorts an on-disk table partition and applies configured attributes.
 
 **Returns** — generic null on success; `()` if no sort config is found for the table.
 
-If `loadconfig` has not been called before `sort` is first invoked, `sort` automatically uses the built-in `default` row in `params` — sorts by `time` ascending with no attribute.
-
 Config lookup order within the loaded params:
 1. Rows where `tabname` matches — used directly.
 2. Rows where `tabname = \`default` — used with a `warn` log.
@@ -173,61 +215,6 @@ dbwrite.applyattr[`:hdb/2024.01.02/trade/;`sym;`p]
 
 ---
 
-### `savedownmanipulation`
-
-A dictionary mapping table name (symbol) to a unary manipulation function. Amend this before EOD to register per-table pre-write transformations.
-
-```q
-dbwrite.savedownmanipulation[`trade]:{[x] update sym:`p#sym from x}
-```
-
-Manipulations are applied by `manipulate[t;x]`. An empty dict (the default) means no manipulation is applied to any table.
-
----
-
-### `manipulate[t;x]`
-
-Applies a registered pre-write manipulation to table `x` of type `t`.
-
-**Parameters**
-
-| Parameter | Type | Description |
-|---|---|---|
-| `t` | symbol | Table name used to look up the registered manipulation function |
-| `x` | table | Table data to transform |
-
-**Returns** — modified table, or original table unmodified if no manipulation is registered for `t` or the registered function throws.
-
-On error the original table is returned unchanged and the error is logged at `error` level. Register functions in `savedownmanipulation` before calling.
-
-```q
-dbwrite.savedownmanipulation[`trade]:{[x] update sym:`p#sym from x}
-data:dbwrite.manipulate[`trade;data]
-```
-
----
-
-### `postreplay[d;p]`
-
-Post-EOD stub called after all tables have been written and sorted.
-
-**Parameters**
-
-| Parameter | Type | Description |
-|---|---|---|
-| `d` | hsym | HDB directory |
-| `p` | date | Partition value |
-
-**Returns** — generic null.
-
-This is a no-op by default. Override at the call site to add custom post-replay logic.
-
-```q
-dbwrite.postreplay[`:hdb;2024.01.02]
-```
-
----
-
 ### `gc[]`
 
 Runs `.Q.gc[]` and logs before/after memory statistics.
@@ -262,12 +249,12 @@ deps:(enlist`log)!enlist logdep
 dbwrite.init[(::);deps]
 ```
 
-Tests cover: dependency injection, `init` error on missing log dep, `manipulate` pass-through and registered function application and error recovery via `savedownmanipulation`, `postreplay` stub, `sort` with default row fallback / explicit config / `default` row fallback / no-match skip / empty input / wrong type, `loadconfig` with null file (warns and resets to default row) / valid file / unrecognised columns / unrecognised attributes / missing file / header-only file, `applyattr` on missing path / null column / invalid attribute / valid path, `gc` log count.
+Tests cover: dependency injection, `init` error on missing log dep, `savedown` write and sort, `savedown` without sym column, `upsert` append and re-sort, `upsert` error on non-existent partition, `sort` with default row fallback / explicit config / `default` row fallback / no-match skip / empty input / wrong type, `loadconfig` with null file / valid file / unrecognised columns / unrecognised attributes / missing file / header-only file, `applyattr` on missing path / null column / invalid attribute / valid path, `gc` log count.
 
 ---
 
 ## Exported symbols
 
 ```q
-export:([init;sort;applyattr;loadconfig;manipulate;savedownmanipulation;postreplay;gc])
+export:([init;savedown;upsert;sort;applyattr;loadconfig;gc])
 ```
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index fbb1df78..95d05d3b 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -1,8 +1,6 @@
 / sort params table - default row sorts all tables by time ascending
 params:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
 
-/ save-down manipulation registry: tabname -> unary function
-savedownmanipulation:()!();
 
 / load and validate sort.csv into .z.M.params
 / file: hsym path; null warns and resets params to default row
@@ -69,19 +67,27 @@ sort:{[d]
     ]]
   };
 
-/ apply registered pre-write manipulation to table x of type t
-/ returns modified table; on error logs and returns original unmodified table
-manipulate:{[t;x]
-  $[t in key .z.M.savedownmanipulation;
-    @[.z.M.savedownmanipulation[t];x;
-      {[x;e].z.m.logerr[`dbwrite;"save-down manipulation failed: ",e];x}[x]];
-    x]
+
+/ write table to a date-partitioned hdb: enumerate syms, apply p# to sym if present, write, then sort
+/ dir: hdb root (hsym); part: partition value (date/month/int); tabname: symbol; data: in-memory table
+savedown:{[dir;part;tabname;data]
+  .z.m.loginfo[`dbwrite;"saving ",string[tabname]," partition ",string[part]," to ",string dir];
+  path:.Q.par[dir;part;tabname];
+  data:.Q.en[dir;data];
+  path set $[`sym in cols data;@[data;`sym;{`p#x}];data];
+  sort[(tabname;path)];
+  .z.m.loginfo[`dbwrite;"finished saving ",string tabname];
   };
 
-/ post-EOD hook - called after all tables written and sorted
-/ d: hdb directory (hsym), p: partition value (date)
-/ stub: override at the call site to add custom post-replay logic
-postreplay:{[d;p]};
+/ upsert data into an existing on-disk partition and re-sort
+/ dir: hdb root (hsym); part: partition value; tabname: symbol; data: in-memory table
+upsert:{[dir;part;tabname;data]
+  .z.m.loginfo[`dbwrite;"upserting ",string[tabname]," partition ",string[part]," in ",string dir];
+  path:.Q.par[dir;part;tabname];
+  .[path;();,;.Q.en[dir;data]];
+  sort[(tabname;path)];
+  .z.m.loginfo[`dbwrite;"finished upserting ",string tabname];
+  };
 
 / format current process memory stats as a loggable string
 memstats:{[]"mem stats: ",{"; "sv "=" sv'flip(string key x;(string value x),\:" MB")}`long$.Q.w[]%1048576};
@@ -94,10 +100,8 @@ gc:{
   };
 
 init:{[config;deps]
-  / config: dict with optional keys
-  /   `savedownmanipulation: tabname!function dict of pre-write manipulation functions
-  / deps: `log!(logdict)
-  /   `log: `info`warn`error!(infofunc;warnfunc;errfunc) - required
+  / config: unused, pass (::)
+  / deps: `log!(logdict) - `info`warn`error!(infofunc;warnfunc;errfunc) - required
   logdict:$[99h=type deps;$[(`log in key deps) and not (::)~deps`log;deps`log;()!()];()!()];
   if[not count logdict;
     '"di.dbwrite: log dependency is required; pass `info`warn`error functions - see di.log or refer to confluence documentation";
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index bd2cd900..d69bd034 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -3,4 +3,4 @@
 
 \l ::dbwrite.q
 
-export:([init;sort;applyattr;loadconfig;manipulate;savedownmanipulation;postreplay;gc])
\ No newline at end of file
+export:([init;savedown;upsert;sort;applyattr;loadconfig;gc])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index 6eec3123..047a3ca8 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -18,36 +18,25 @@ true,0,0,q,logdep[`error]~.m.di.0dbwrite.logerr,1,1,injected error function stor
 fail,0,0,q,dbwrite.init[(::);(::)],1,1,init without log dep throws an error
 fail,0,0,q,dbwrite.init[(::);`log!(::)],1,1,init with log set to (::) throws an error
 
-/ Test 3: manipulate - pass-through cases (no functions registered)
-run,0,0,q,"tbl:([]sym:`AAPL`IBM;price:100 200f)",1,,sample table
-true,0,0,q,"tbl~dbwrite.manipulate[`trade;tbl]",1,,unregistered table name returns original
-true,0,0,q,"tbl~dbwrite.manipulate[`;tbl]",1,,null table name returns original
-true,0,0,q,"(0#tbl)~dbwrite.manipulate[`other;0#tbl]",1,,empty table returned unchanged
-true,0,0,q,42~dbwrite.manipulate[`notregistered;42],1,,non-table input returns unchanged when unregistered
-
-/ Test 4: postreplay returns generic null
-true,0,0,q,(::)~dbwrite.postreplay[`:hdb;2024.01.01],1,,stub returns generic null
-true,0,0,q,(::)~dbwrite.postreplay[`;0Nd],1,,null args return generic null
-
-/ Test 5: sort uses default row when table name not in config
+/ Test 3: sort uses default row when table name not in config
 run,0,0,q,`:dbwrite_defp_tp/.d set `time`sym,1,,write column order file
 run,0,0,q,`:dbwrite_defp_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
 run,0,0,q,`:dbwrite_defp_tp/sym set `IBM`AAPL,1,,write sym column
 run,0,0,q,"dbwrite.sort[(`anytable;`:dbwrite_defp_tp/)]",1,,sort using default row
 true,0,0,q,(asc exec time from get `:dbwrite_defp_tp/)~exec time from get `:dbwrite_defp_tp/,1,,time column sorted ascending by default row
 
-/ Test 6: loadconfig null resets params to default row
+/ Test 4: loadconfig null resets params to default row
 run,0,0,q,dbwrite.loadconfig[`],1,,call loadconfig with null file
 true,0,0,q,.m.di.0dbwrite.params~([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b),1,,params reset to default row
 
-/ Test 7: loadconfig - valid config file loaded and parsed correctly
+/ Test 5: loadconfig - valid config file loaded and parsed correctly
 run,0,0,q,cfgfile:`:dbwrite_test.csv,1,,temp sort config path
 run,0,0,q,"cfgfile 0:(""tabname,att,column,sort"";""trade,p,sym,1"";""trade,,price,0"")",1,,write test sort config
 run,0,0,q,dbwrite.loadconfig[cfgfile],1,,loadconfig with valid csv succeeds
 true,0,0,q,2=count .m.di.0dbwrite.params,1,,two rows loaded into params
 true,0,0,q,all `trade=exec tabname from .m.di.0dbwrite.params,1,,both rows are for trade table
 
-/ Test 7 cont: loadconfig - invalid and edge-case inputs
+/ Test 5 cont: loadconfig - invalid and edge-case inputs
 true,0,0,q,@[dbwrite.loadconfig;`:nonexistent_file_xyz;{1b}],1,,nonexistent file throws
 true,0,0,q,@[dbwrite.loadconfig;42;{1b}],1,,non-symbol arg throws
 run,0,0,q,badcols:`:dbwrite_badcols.csv,1,,
@@ -61,12 +50,12 @@ run,0,0,q,"emptycfg 0:enlist""tabname,att,column,sort""",1,,csv with header only
 run,0,0,q,dbwrite.loadconfig[emptycfg],1,,loadconfig with header-only csv
 true,0,0,q,0=count .m.di.0dbwrite.params,1,,params is empty after header-only csv
 
-/ Test 8: applyattr - error cases
+/ Test 6: applyattr - error cases
 run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:nonexist_attr_dir/;`sym;`p]",1,,apply to non-existent path
 true,0,0,q,2=logcount,1,,loginfo (before attempt) and logerr (on failure) each called once
 
-/ Test 8 cont: applyattr - null column name logs error and does not throw
+/ Test 6 cont: applyattr - null column name logs error and does not throw
 run,0,0,q,`:dbwrite_attr_tp/.d set `sym`price,1,,write column order file
 run,0,0,q,`:dbwrite_attr_tp/sym set `IBM`AAPL`MSFT,1,,write sym column
 run,0,0,q,`:dbwrite_attr_tp/price set 200 100 300f,1,,write price column
@@ -74,30 +63,30 @@ run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`;`p]",1,,apply with null column name
 true,0,0,q,2=logcount,1,,loginfo and logerr both called for null column
 
-/ Test 8 cont: applyattr - invalid att logs error
+/ Test 6 cont: applyattr - invalid att logs error
 run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`z]",1,,apply with invalid attribute
 true,0,0,q,2=logcount,1,,loginfo and logerr both called for invalid att
 
-/ Test 9: applyattr - attribute applied and logged when path exists
+/ Test 7: applyattr - attribute applied and logged when path exists
 run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`p]",1,,apply p attr to sym column
 true,0,0,q,1=logcount,1,,loginfo called once (no error)
 true,0,0,q,`p=attr get `:dbwrite_attr_tp/sym,1,,p attribute applied on disk
 
-/ Test 10: sort - skip when table not in config and no default row
+/ Test 8: sort - skip when table not in config and no default row
 run,0,0,q,dbwrite.loadconfig[cfgfile],1,,reload trade-only config (no default row)
 run,0,0,q,logcount:0,1,,reset counter
 true,0,0,q,()~dbwrite.sort[`no_params_table_xyz],1,,returns () when table not in config
 true,0,0,q,2=logcount,1,,loginfo (sorting start) and logwarn (skip) both called
 true,0,0,q,()~dbwrite.sort[()],1,,empty list returns ()
 
-/ Test 10 cont: sort - wrong type logs error and returns ()
+/ Test 8 cont: sort - wrong type logs error and returns ()
 run,0,0,q,logcount:0,1,,reset counter
 true,0,0,q,()~dbwrite.sort[42],1,,wrong type (long) returns ()
 true,0,0,q,1=logcount,1,,logerr called once for wrong type
 
-/ Test 11: sort - sorting and attribute application when config entry exists for table
+/ Test 9: sort - sorting and attribute application when config entry exists for table
 run,0,0,q,`:dbwrite_sort_tp/.d set `sym`price,1,,write column order file
 run,0,0,q,`:dbwrite_sort_tp/sym set `IBM`AAPL`MSFT,1,,write unsorted sym column
 run,0,0,q,`:dbwrite_sort_tp/price set 200 100 300f,1,,write price column
@@ -105,7 +94,7 @@ run,0,0,q,"dbwrite.sort[(`trade;`:dbwrite_sort_tp/)]",1,,sort trade table in tes
 true,0,0,q,`AAPL`IBM`MSFT~exec sym from get `:dbwrite_sort_tp/,1,,sorted ascending by sym
 true,0,0,q,`p=attr get `:dbwrite_sort_tp/sym,1,,p attribute applied to sym on disk
 
-/ Test 12: sort - default row used after resetting params with loadconfig null
+/ Test 10: sort - default row used after resetting params with loadconfig null
 run,0,0,q,dbwrite.loadconfig[`],1,,reset params to default row via null loadconfig
 run,0,0,q,`:dbwrite_reset_tp/.d set `time`sym,1,,write column order file
 run,0,0,q,`:dbwrite_reset_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
@@ -116,11 +105,34 @@ run,0,0,q,"dbwrite.sort[(`othertable;`:dbwrite_reset_tp/)]",1,,sort using defaul
 true,0,0,q,(asc exec time from get `:dbwrite_reset_tp/)~exec time from get `:dbwrite_reset_tp/,1,,sorted ascending by time via default row
 true,0,0,q,4=logcount,1,,loginfo x2 (sorting start + sort-by) + logwarn x1 (using defaults) + loginfo x1 (finished)
 
-/ Test 13: gc calls loginfo twice and does not throw
+/ Test 11: gc calls loginfo twice and does not throw
 run,0,0,q,logcount:0,1,,reset counter
 run,0,0,q,dbwrite.gc[],1,,run garbage collect
 true,0,0,q,2=logcount,1,,loginfo called twice (start and end)
 
+/ Test 12: savedown - write table to hdb partition
+run,0,0,q,dbwrite.loadconfig[`],1,,ensure default sort params (by time)
+run,0,0,q,"sdtbl:([]time:2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000;sym:`IBM`AAPL;price:100 200f)",1,,unsorted test table
+run,0,0,q,"dbwrite.savedown[`:dbwrite_sd_hdb;2024.01.01;`trade;sdtbl]",1,,write to hdb partition
+run,0,0,q,"sdpath:.Q.par[`:dbwrite_sd_hdb;2024.01.01;`trade]",1,,partition path for assertions
+true,0,0,q,2=count get sdpath,1,,two rows written to partition
+true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,,rows sorted by time ascending after sort
+true,0,0,q,0<count get `:dbwrite_sd_hdb/sym,1,,sym enumeration file created in hdb root
+
+/ Test 12 cont: savedown - table without sym column
+run,0,0,q,"nosymtbl:([]time:enlist 2024.01.02D10:00:00.000000000;price:enlist 50f)",1,,table with no sym column
+run,0,0,q,"dbwrite.savedown[`:dbwrite_sd_hdb;2024.01.02;`nosymtrade;nosymtbl]",1,,savedown table without sym column
+true,0,0,q,"1=count get .Q.par[`:dbwrite_sd_hdb;2024.01.02;`nosymtrade]",1,,row written correctly for table without sym
+
+/ Test 13: upsert - append rows to existing partition and re-sort
+run,0,0,q,"usttbl:([]time:enlist 2024.01.01D07:00:00.000000000;sym:enlist`MSFT;price:enlist 300f)",1,,row with earlier time than existing data
+run,0,0,q,"dbwrite.upsert[`:dbwrite_sd_hdb;2024.01.01;`trade;usttbl]",1,,upsert into existing partition
+true,0,0,q,3=count get sdpath,1,,three rows on disk after upsert
+true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,,still sorted by time after upsert
+
+/ Test 13 cont: upsert into non-existent partition throws
+true,0,0,q,"@[dbwrite.upsert[`:dbwrite_sd_hdb;2099.01.01;`nonexist;];sdtbl;{1b}]",1,,upsert into non-existent partition throws
+
 / Clean up all created on-disk files and directories
 after,0,0,q,@[hdel;`:dbwrite_defp_tp/;{}],1,,cleanup defaultparams test partition
 after,0,0,q,@[hdel;cfgfile;{}],1,,remove temp sort config
@@ -130,3 +142,4 @@ after,0,0,q,@[hdel;emptycfg;{}],1,,remove empty config file
 after,0,0,q,@[hdel;`:dbwrite_attr_tp/;{}],1,,cleanup attr test partition
 after,0,0,q,@[hdel;`:dbwrite_sort_tp/;{}],1,,cleanup sort test partition
 after,0,0,q,@[hdel;`:dbwrite_reset_tp/;{}],1,,cleanup reset test partition
+after,0,0,q,@[hdel;`:dbwrite_sd_hdb;{}],1,,cleanup savedown/upsert test hdb

From 7b0a7871251f665d791bc63b9abaabac801c9d66 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Fri, 19 Jun 2026 14:40:17 +0100
Subject: [PATCH 07/15] Fix upsert naming, path construction, and add
 appenddown existence guard

- Rename upsert to appenddown (upsert is a reserved word in kdb+)
- Remove auto-sort from appenddown; caller sorts explicitly when done
- Fix savedown/appenddown to use trailing-slash path via ` sv (.Q.par[...];`)
  so xasc treats it as a splayed table directory (without slash, set writes
  a binary file and xasc fails with "Not a directory")
- Add existence guard to appenddown: throws if partition path does not exist
- Update init.q export dict, tests, and docs accordingly

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.md | 20 +++++++++++++-------
 di/dbwrite/dbwrite.q  | 16 +++++++++-------
 di/dbwrite/init.q     |  2 +-
 di/dbwrite/test.csv   | 23 ++++++++++++++---------
 4 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index eaea7ed1..c9986da3 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -7,7 +7,7 @@ Write, sort, and attribute utilities for kdb+ processes that persist data to dis
 ## Features
 
 - Write in-memory tables to a date-partitioned HDB with `savedown` — enumerates syms, applies `p#` to `sym`, writes, then sorts
-- Append rows to an existing partition with `upsert` — enumerates syms, appends, then re-sorts
+- Append rows to an existing partition with `appenddown` — enumerates syms and appends; sort separately when the partition is complete
 - Sort on-disk table partitions by configured columns using `xasc`
 - Apply kdb+ attributes (`p`, `s`, `g`, `u`) to on-disk columns after sort
 - Sort and attribute behaviour driven by a CSV config file; a `default` row acts as a fallback
@@ -60,7 +60,7 @@ Sorts `trade` by `sym`, applies `p` to `sym`. Tables not listed fall back to `de
 |---|---|
 | `init[config;deps]` | Wire injected dependencies; must be called first |
 | `savedown[dir;part;tabname;data]` | Write in-memory table to HDB partition, enumerate syms, apply `p#sym`, then sort |
-| `upsert[dir;part;tabname;data]` | Append rows to existing partition, enumerate syms, then re-sort |
+| `appenddown[dir;part;tabname;data]` | Append rows to existing partition and enumerate syms; does not sort |
 | `loadconfig[file]` | Load and validate the sort config CSV into module state |
 | `sort[d]` | Sort an on-disk partition and apply attributes per config |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
@@ -117,9 +117,11 @@ dbwrite.savedown[`:hdb;2024.01.02;`trade;data]
 
 ---
 
-### `upsert[dir;part;tabname;data]`
+### `appenddown[dir;part;tabname;data]`
 
-Appends rows to an existing on-disk partition then re-sorts. Enumerates symbol columns before appending. The partition must already exist — use `savedown` for the initial write.
+Appends rows to an existing on-disk partition. Enumerates symbol columns then appends; does not sort. Call `sort` explicitly when the partition is complete.
+
+Keeping sort separate allows multiple intraday appends without the cost of re-sorting a growing partition on each call.
 
 **Parameters**
 
@@ -133,7 +135,11 @@ Appends rows to an existing on-disk partition then re-sorts. Enumerates symbol c
 **Returns** — generic null on success; throws if the partition does not exist or on write failure.
 
 ```q
-dbwrite.upsert[`:hdb;2024.01.02;`trade;latedata]
+/ intraday: append each batch as it arrives
+dbwrite.appenddown[`:hdb;2024.01.02;`trade;batch]
+
+/ end-of-day: sort once when done
+dbwrite.sort[(`trade;.Q.par[`:hdb;2024.01.02;`trade])]
 ```
 
 ---
@@ -249,12 +255,12 @@ deps:(enlist`log)!enlist logdep
 dbwrite.init[(::);deps]
 ```
 
-Tests cover: dependency injection, `init` error on missing log dep, `savedown` write and sort, `savedown` without sym column, `upsert` append and re-sort, `upsert` error on non-existent partition, `sort` with default row fallback / explicit config / `default` row fallback / no-match skip / empty input / wrong type, `loadconfig` with null file / valid file / unrecognised columns / unrecognised attributes / missing file / header-only file, `applyattr` on missing path / null column / invalid attribute / valid path, `gc` log count.
+Tests cover: dependency injection, `init` error on missing log dep, `savedown` write and sort, `savedown` without sym column, `appenddown` append without sort, explicit `sort` after `appenddown`, `appenddown` error on non-existent partition, `sort` with default row fallback / explicit config / `default` row fallback / no-match skip / empty input / wrong type, `loadconfig` with null file / valid file / unrecognised columns / unrecognised attributes / missing file / header-only file, `applyattr` on missing path / null column / invalid attribute / valid path, `gc` log count.
 
 ---
 
 ## Exported symbols
 
 ```q
-export:([init;savedown;upsert;sort;applyattr;loadconfig;gc])
+export:([init;savedown;appenddown;sort;applyattr;loadconfig;gc])
 ```
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 95d05d3b..1e3fdc9c 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -72,21 +72,23 @@ sort:{[d]
 / dir: hdb root (hsym); part: partition value (date/month/int); tabname: symbol; data: in-memory table
 savedown:{[dir;part;tabname;data]
   .z.m.loginfo[`dbwrite;"saving ",string[tabname]," partition ",string[part]," to ",string dir];
-  path:.Q.par[dir;part;tabname];
+  path:` sv (.Q.par[dir;part;tabname];`);
   data:.Q.en[dir;data];
   path set $[`sym in cols data;@[data;`sym;{`p#x}];data];
   sort[(tabname;path)];
   .z.m.loginfo[`dbwrite;"finished saving ",string tabname];
   };
 
-/ upsert data into an existing on-disk partition and re-sort
+/ append data to an existing on-disk partition; enumerate syms but do not sort
+/ call sort separately when the partition is complete
 / dir: hdb root (hsym); part: partition value; tabname: symbol; data: in-memory table
-upsert:{[dir;part;tabname;data]
-  .z.m.loginfo[`dbwrite;"upserting ",string[tabname]," partition ",string[part]," in ",string dir];
-  path:.Q.par[dir;part;tabname];
+appenddown:{[dir;part;tabname;data]
+  .z.m.loginfo[`dbwrite;"appending ",string[tabname]," partition ",string[part]," in ",string dir];
+  path:` sv (.Q.par[dir;part;tabname];`);
+  if[not count @[key;path;{`$()}];
+    '"appenddown: partition does not exist at ",string path];
   .[path;();,;.Q.en[dir;data]];
-  sort[(tabname;path)];
-  .z.m.loginfo[`dbwrite;"finished upserting ",string tabname];
+  .z.m.loginfo[`dbwrite;"finished appending ",string tabname];
   };
 
 / format current process memory stats as a loggable string
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index d69bd034..917d5dd6 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -3,4 +3,4 @@
 
 \l ::dbwrite.q
 
-export:([init;savedown;upsert;sort;applyattr;loadconfig;gc])
\ No newline at end of file
+export:([init;savedown;appenddown;sort;applyattr;loadconfig;gc])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index 047a3ca8..c742d5ff 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -114,7 +114,7 @@ true,0,0,q,2=logcount,1,,loginfo called twice (start and end)
 run,0,0,q,dbwrite.loadconfig[`],1,,ensure default sort params (by time)
 run,0,0,q,"sdtbl:([]time:2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000;sym:`IBM`AAPL;price:100 200f)",1,,unsorted test table
 run,0,0,q,"dbwrite.savedown[`:dbwrite_sd_hdb;2024.01.01;`trade;sdtbl]",1,,write to hdb partition
-run,0,0,q,"sdpath:.Q.par[`:dbwrite_sd_hdb;2024.01.01;`trade]",1,,partition path for assertions
+run,0,0,q,"sdpath:` sv (.Q.par[`:dbwrite_sd_hdb;2024.01.01;`trade];`)",1,,partition path with trailing slash for xasc compatibility
 true,0,0,q,2=count get sdpath,1,,two rows written to partition
 true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,,rows sorted by time ascending after sort
 true,0,0,q,0<count get `:dbwrite_sd_hdb/sym,1,,sym enumeration file created in hdb root
@@ -122,16 +122,21 @@ true,0,0,q,0<count get `:dbwrite_sd_hdb/sym,1,,sym enumeration file created in h
 / Test 12 cont: savedown - table without sym column
 run,0,0,q,"nosymtbl:([]time:enlist 2024.01.02D10:00:00.000000000;price:enlist 50f)",1,,table with no sym column
 run,0,0,q,"dbwrite.savedown[`:dbwrite_sd_hdb;2024.01.02;`nosymtrade;nosymtbl]",1,,savedown table without sym column
-true,0,0,q,"1=count get .Q.par[`:dbwrite_sd_hdb;2024.01.02;`nosymtrade]",1,,row written correctly for table without sym
+true,0,0,q,"1=count get ` sv (.Q.par[`:dbwrite_sd_hdb;2024.01.02;`nosymtrade];`)",1,,row written correctly for table without sym
 
-/ Test 13: upsert - append rows to existing partition and re-sort
-run,0,0,q,"usttbl:([]time:enlist 2024.01.01D07:00:00.000000000;sym:enlist`MSFT;price:enlist 300f)",1,,row with earlier time than existing data
-run,0,0,q,"dbwrite.upsert[`:dbwrite_sd_hdb;2024.01.01;`trade;usttbl]",1,,upsert into existing partition
-true,0,0,q,3=count get sdpath,1,,three rows on disk after upsert
-true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,,still sorted by time after upsert
+/ Test 13: appenddown - append rows to existing partition without sorting
+run,0,0,q,"apptbl:([]time:enlist 2024.01.01D07:00:00.000000000;sym:enlist`MSFT;price:enlist 300f)",1,,row with earlier time than existing data
+run,0,0,q,"dbwrite.appenddown[`:dbwrite_sd_hdb;2024.01.01;`trade;apptbl]",1,,append to existing partition
+true,0,0,q,3=count get sdpath,1,,three rows on disk after appenddown
+true,0,0,q,not (asc exec time from get sdpath)~exec time from get sdpath,1,,rows not yet sorted (no auto-sort on appenddown)
+
+/ Test 13 cont: sort explicitly after appenddown
+run,0,0,q,"dbwrite.sort[(`trade;sdpath)]",1,,sort partition after appending
+true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,,sorted by time after explicit sort call
+
+/ Test 13 cont: appenddown into non-existent partition throws
+true,0,0,q,"@[dbwrite.appenddown[`:dbwrite_sd_hdb;2099.01.01;`nonexist;];sdtbl;{1b}]",1,,appenddown into non-existent partition throws
 
-/ Test 13 cont: upsert into non-existent partition throws
-true,0,0,q,"@[dbwrite.upsert[`:dbwrite_sd_hdb;2099.01.01;`nonexist;];sdtbl;{1b}]",1,,upsert into non-existent partition throws
 
 / Clean up all created on-disk files and directories
 after,0,0,q,@[hdel;`:dbwrite_defp_tp/;{}],1,,cleanup defaultparams test partition

From c2a2f0022fec9ca9febafd0a85c394d00703d575 Mon Sep 17 00:00:00 2001
From: ascottDI <ascott@homer.aquaq.co.uk>
Date: Mon, 22 Jun 2026 09:09:11 +0100
Subject: [PATCH 08/15] merging sort modules functionality into dbwrite

---
 di/dbwrite/dbwrite.md | 220 ++++++++++----------------
 di/dbwrite/dbwrite.q  | 298 +++++++++++++++++++++++------------
 di/dbwrite/init.q     |   2 +-
 di/dbwrite/test.csv   | 355 ++++++++++++++++++++++++------------------
 4 files changed, 492 insertions(+), 383 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index c9986da3..677937a3 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -1,19 +1,22 @@
 # di.dbwrite
 
-Write, sort, and attribute utilities for kdb+ processes that persist data to disk.
+Write, sort, and attribute utilities for kdb+ processes that persist data to disk (rdb, wdb, tickerlogreplay).
+
+The sorting/attribute engine is `di.sort` baked in directly: a **config table** (`tabname`,`att`,`column`,`sort`) drives which columns are sorted and which attributes are applied per table. You pass that config straight to `sort`/`savedown`; if it lives in a CSV, `readcsv` reads it into the right shape. A table with no explicit entry falls back to a `default` row, and `(::)` selects a built-in default (sort every table by `time` ascending).
+
+`di.dbwrite` holds no sort state of its own — each call takes the config it should use, so you can build that config by hand, from a query, or from a CSV and reuse or vary it freely.
 
 ---
 
 ## Features
 
-- Write in-memory tables to a date-partitioned HDB with `savedown` — enumerates syms, applies `p#` to `sym`, writes, then sorts
+- Write an in-memory table to a date-partitioned HDB with `savedown` — enumerates syms, applies `p#` to `sym`, writes, then sorts per config
 - Append rows to an existing partition with `appenddown` — enumerates syms and appends; sort separately when the partition is complete
-- Sort on-disk table partitions by configured columns using `xasc`
-- Apply kdb+ attributes (`p`, `s`, `g`, `u`) to on-disk columns after sort
-- Sort and attribute behaviour driven by a CSV config file; a `default` row acts as a fallback
-- A built-in `default` row in `params` provides an out-of-the-box fallback (sort by `time` ascending) when no config file is loaded
+- Sort on-disk table partitions by configured columns using `xasc`, then apply kdb+ attributes (`p`,`s`,`g`,`u`)
+- Config supplied as an in-memory table or read from a CSV (`readcsv`); a `default` row or `(::)` provides a fallback
+- Apply a registered pre-write `manipulate` transformation per table
 - Run `.Q.gc[]` with before/after memory logging
-- All errors from sort, attribute application, and write operations are either caught-and-logged (sort, applyattr) or propagated to the caller (savedown, upsert)
+- Sort and attribute errors are caught-and-logged (a single partition failure does not halt the run); config and write errors are raised to the caller with a `di.dbwrite:` prefix
 
 ---
 
@@ -21,217 +24,173 @@ Write, sort, and attribute utilities for kdb+ processes that persist data to dis
 
 | Dependency | Key | Required | Description |
 |---|---|---|---|
-| `di.log` | `` `log `` | yes | Logging functions `info`, `warn`, `error` — each `{[ctx;msg] ...}` |
+| logger | `` `log `` | yes | Functions `info`,`warn`,`error` — each `{[ctx;msg] ...}` |
 
-The `log` dependency must be passed to `init`. The module throws if it is absent or `(::)`.
+The `log` dependency must be passed to `init`. The module throws if it is absent, `(::)`, or missing any of the three keys. `di.log` satisfies the contract:
 
----
+```q
+log:use`di.log
+logdep:`info`warn`error!(log.info;log.warn;log.error)
+dbwrite:use`di.dbwrite
+dbwrite.init[(::); enlist[`log]!enlist logdep]
+```
 
-## Sort config CSV
+---
 
-`loadconfig` reads a CSV with four columns:
+## The config table
 
 | Column | Type | Description |
 |---|---|---|
 | `tabname` | symbol | Table name, or `` `default `` as a catch-all fallback |
-| `att` | symbol | kdb+ attribute to apply: `p`, `s`, `g`, `u`, or empty for none |
-| `column` | symbol | Column to sort or attribute |
-| `sort` | boolean | `1b` — include in `xasc` sort key; `0b` — attribute only |
+| `att` | symbol | Attribute applied after sort: `p`,`s`,`g`,`u`, or empty (`` ` ``) for none |
+| `column` | symbol | Column to sort and/or attribute |
+| `sort` | boolean | `1b` — include in the `xasc` sort key; `0b` — attribute only |
 
-Example `sort.csv`:
+Build it directly, or read it from a CSV with `readcsv`. A CSV must have exactly the four columns `tabname,att,column,sort`, in **any** order (the result is normalised to canonical order); a missing, extra, or misnamed column raises a clear `di.dbwrite:` error rather than silently mis-parsing.
 
 ```
 tabname,att,column,sort
 trade,p,sym,1
 trade,,price,0
-quote,p,sym,1
 default,,time,1
 ```
 
-Sorts `trade` by `sym`, applies `p` to `sym`. Tables not listed fall back to `default` and sort by `time`.
-
 ---
 
 ## Functions
 
-### Summary
-
 | Function | Description |
 |---|---|
-| `init[config;deps]` | Wire injected dependencies; must be called first |
-| `savedown[dir;part;tabname;data]` | Write in-memory table to HDB partition, enumerate syms, apply `p#sym`, then sort |
-| `appenddown[dir;part;tabname;data]` | Append rows to existing partition and enumerate syms; does not sort |
-| `loadconfig[file]` | Load and validate the sort config CSV into module state |
-| `sort[d]` | Sort an on-disk partition and apply attributes per config |
+| `init[config;deps]` | Wire injected dependencies (and optional `savedownmanipulation`); must be called first |
+| `readcsv[file]` | Read a config CSV and return it as a table |
+| `sort[config;tabname;dirs]` | Sort on-disk partition(s) for a table and apply attributes per config |
+| `savedown[config;dir;part;tabname;data]` | Write an in-memory table to an HDB partition, then sort it per config |
+| `appenddown[dir;part;tabname;data]` | Append rows to an existing partition (no sort) |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
+| `manipulate[t;x]` | Apply the registered pre-write manipulation for table `t` |
+| `postreplay[d;p]` | Post-EOD override hook (stub) |
 | `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
 
 ---
 
 ### `init[config;deps]`
 
-Wires injected dependencies into the module. Must be called before any other function.
+Wire injected dependencies. Must be called before any other function.
 
-**Parameters**
-
-| Parameter | Type | Description |
+| Arg | Type | Description |
 |---|---|---|
-| `config` | any | Accepted but unused; pass `(::)` |
-| `deps` | dict | Must contain `` `log `` → `` `info`warn`error!(infofunc;warnfunc;errfunc) `` |
-
-**Returns** — generic null.
+| `config` | dict (or `::`) | Optional. May contain `savedownmanipulation` — a `tabname!unary-function` dict applied by `manipulate`. `::` or a dict without the key means no manipulations. |
+| `deps` | dict | Must contain `` `log `` → `` `info`warn`error!(infofn;warnfn;errfn) ``. |
 
-Throws with a descriptive message if the `log` dependency is missing or set to `(::)`.
-
-```q
-log:use`di.log
-log.init[logconfig]
-logdep:`info`warn`error!(log.info;log.warn;log.error)
-
-dbwrite:use`di.dbwrite
-dbwrite.init[(::);(enlist`log)!enlist logdep]
-```
+Throws (prefixed `di.dbwrite:`) if `deps` is not a dict, `log` is missing, or the log dict lacks any required key.
 
 ---
 
-### `savedown[dir;part;tabname;data]`
+### `readcsv[file]`
 
-Writes an in-memory table to a date-partitioned HDB. Enumerates symbol columns against the HDB sym file, applies `p#` to `sym` if present, writes the partition, then calls `sort` to sort and apply attributes per the loaded config.
-
-**Parameters**
+Read a config CSV and **return** it as a table (does not store it) — pass the result to `sort` or `savedown`.
 
 | Parameter | Type | Description |
 |---|---|---|
-| `dir` | hsym | HDB root directory (e.g. `` `:hdb ``) |
-| `part` | date/month/int | Partition value |
-| `tabname` | symbol | Table name — determines the partition subdirectory |
-| `data` | table | In-memory table to write |
-
-**Returns** — generic null on success; throws on write failure.
+| `file` | symbol/hsym | Path to the CSV; coerced with `hsym`. Errors (`di.dbwrite:`) if not a symbol. |
 
-If `loadconfig` has not been called, the built-in default row applies (sort by `time` ascending). If the table has no `sym` column, enumeration and `p#` are skipped.
+The header is validated as it is read; a wrong shape errors clearly. Attribute-value validation happens later in `sort`.
 
 ```q
-dbwrite.savedown[`:hdb;2024.01.02;`trade;data]
+config:dbwrite.readcsv `:config/sort.csv
 ```
 
 ---
 
-### `appenddown[dir;part;tabname;data]`
-
-Appends rows to an existing on-disk partition. Enumerates symbol columns then appends; does not sort. Call `sort` explicitly when the partition is complete.
+### `sort[config;tabname;dirs]`
 
-Keeping sort separate allows multiple intraday appends without the cost of re-sorting a growing partition on each call.
-
-**Parameters**
+Sort and apply attributes to the on-disk partition(s) for one table.
 
 | Parameter | Type | Description |
 |---|---|---|
-| `dir` | hsym | HDB root directory |
-| `part` | date/month/int | Partition value |
-| `tabname` | symbol | Table name |
-| `data` | table | Rows to append |
+| `config` | table, or `::` | Config table (built directly or via `readcsv`). `::` uses the built-in default config. |
+| `tabname` | symbol | Table name. Errors (`di.dbwrite:`) if not a symbol. |
+| `dirs` | hsym, or list of hsyms | Partition directory (or directories). |
 
-**Returns** — generic null on success; throws if the partition does not exist or on write failure.
+`config` is validated first (errors `di.dbwrite:` if not a table, has unknown/missing columns, a non-boolean `sort`, or an `att` outside `` ` `p`s`g`u ``). Row lookup: the table's own rows → the `default` row → otherwise a warn is logged and `()` returned. Each partition is processed independently; a failure on one is logged and does not halt the rest.
 
 ```q
-/ intraday: append each batch as it arrives
-dbwrite.appenddown[`:hdb;2024.01.02;`trade;batch]
-
-/ end-of-day: sort once when done
-dbwrite.sort[(`trade;.Q.par[`:hdb;2024.01.02;`trade])]
+dbwrite.sort[config; `trade; `:/hdb/2024.01.02/trade`:/hdb/2024.01.03/trade]
 ```
 
 ---
 
-### `loadconfig[file]`
-
-Loads and validates the sort configuration CSV, storing the result in module state for use by `sort`.
+### `savedown[config;dir;part;tabname;data]`
 
-**Parameters**
+Write an in-memory table to a date-partitioned HDB partition, then sort it per `config`. Enumerates symbol columns against the HDB sym file and applies `p#` to `sym` (if present) before writing.
 
 | Parameter | Type | Description |
 |---|---|---|
-| `file` | hsym | Path to the sort config CSV; pass null (`` ` ``) to warn and reset `params` to the default row |
-
-**Returns** — generic null on success; throws on file/validation failure.
-
-Validation checks that all four required columns (`tabname`, `att`, `column`, `sort`) are present and that all `att` values are within `` ``p`s`g`u ``. Throws a descriptive error for invalid files or unreadable paths.
+| `config` | table, or `::` | Config passed through to `sort` (`::` for the default). |
+| `dir` | hsym | HDB root directory (e.g. `` `:hdb ``). |
+| `part` | date/month/int | Partition value. |
+| `tabname` | symbol | Table name — determines the partition subdirectory. |
+| `data` | table | In-memory table to write. |
 
-Passing null warns at `warn` level and resets `params` to the default row — it does not throw.
+Throws on write failure. If the table has no `sym` column, enumeration and `p#` are skipped.
 
 ```q
-dbwrite.loadconfig[`:config/sort.csv]
+dbwrite.savedown[config; `:hdb; 2024.01.02; `trade; data]
 ```
 
 ---
 
-### `sort[d]`
-
-Sorts an on-disk table partition and applies configured attributes.
+### `appenddown[dir;part;tabname;data]`
 
-**Parameters**
+Append rows to an existing on-disk partition (enumerates syms); does **not** sort. Keeping sort separate allows multiple intraday appends without re-sorting a growing partition on each call — sort once when the partition is complete.
 
 | Parameter | Type | Description |
 |---|---|---|
-| `d` | symbol or list | Table name alone, or `(tabname;dir)`, or `(tabname;list of dirs)` — see below |
-
-`d` forms:
+| `dir` | hsym | HDB root directory. |
+| `part` | date/month/int | Partition value. |
+| `tabname` | symbol | Table name. |
+| `data` | table | Rows to append. |
 
-| Form | Example |
-|---|---|
-| Symbol | `` `trade `` |
-| Tabname + single dir | `` (`trade;`:hdb/2024.01.02/trade/) `` |
-| Tabname + dir list | `` (`trade;`:hdb/2024.01.02/trade/ `:hdb/2024.01.03/trade/) `` |
-
-**Returns** — generic null on success; `()` if no sort config is found for the table.
-
-Config lookup order within the loaded params:
-1. Rows where `tabname` matches — used directly.
-2. Rows where `tabname = \`default` — used with a `warn` log.
-3. No match — warns and returns `()`.
-
-Sort and attribute errors are caught, logged, and swallowed.
+Throws (prefixed `di.dbwrite:`) if the partition does not exist, or on write failure.
 
 ```q
-dbwrite.sort[(`trade;`:hdb/2024.01.02/trade/)]
+/ intraday: append each batch as it arrives
+dbwrite.appenddown[`:hdb; 2024.01.02; `trade; batch]
+/ end-of-day: sort once when done
+dbwrite.sort[config; `trade; .Q.par[`:hdb;2024.01.02;`trade]]
 ```
 
 ---
 
 ### `applyattr[dloc;colname;att]`
 
-Applies a single kdb+ attribute to an on-disk column.
+Apply a single kdb+ attribute to one on-disk column (best-effort: logs and swallows errors so a run continues). A non-attribute `att` (the empty sentinel or any value outside `` `p`s`g`u ``) is a silent no-op.
 
-**Parameters**
+```q
+dbwrite.applyattr[`:/hdb/2024.01.02/trade; `sym; `p]
+```
 
-| Parameter | Type | Description |
-|---|---|---|
-| `dloc` | hsym | On-disk partition directory (e.g. `` `:hdb/2024.01.02/trade/ ``) |
-| `colname` | symbol | Column name |
-| `att` | symbol | Attribute to apply: `` `p ``, `` `s ``, `` `g ``, or `` `u `` |
+---
 
-**Returns** — generic null on success.
+### `manipulate[t;x]`
 
-Logs the attempt before applying. On failure, logs the error and continues — does not throw.
+Apply the registered pre-write manipulation for table `t` to `x`, returning the result. Unchanged if none is registered; if the manipulation throws, the failure is logged and `x` is returned unchanged.
 
 ```q
-dbwrite.applyattr[`:hdb/2024.01.02/trade/;`sym;`p]
+data:dbwrite.manipulate[`trade; data]
 ```
 
 ---
 
-### `gc[]`
+### `postreplay[d;p]`
 
-Runs `.Q.gc[]` and logs before/after memory statistics.
+Post-EOD hook, called after all tables are written and sorted. A stub by default (`{[d;p]}`) — override at the call site to add custom post-replay logic.
 
-**Returns** — generic null.
+---
 
-Emits two `info`-level log lines: memory stats before collection, and bytes recovered plus memory stats after.
+### `gc[]`
 
-```q
-dbwrite.gc[]
-```
+Run `.Q.gc[]`, logging memory stats before and after.
 
 ---
 
@@ -242,25 +201,12 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-The test suite uses mock logging (no `di.log` dependency required). The mock wires up three no-op counters so log call counts can be asserted:
-
-```q
-dbwrite:use`di.dbwrite
-logcount:0
-loginfo:{[c;m] logcount::logcount+1}
-logwarn:{[c;m] logcount::logcount+1}
-logerr:{[c;m] logcount::logcount+1}
-logdep:`info`warn`error!(loginfo;logwarn;logerr)
-deps:(enlist`log)!enlist logdep
-dbwrite.init[(::);deps]
-```
-
-Tests cover: dependency injection, `init` error on missing log dep, `savedown` write and sort, `savedown` without sym column, `appenddown` append without sort, explicit `sort` after `appenddown`, `appenddown` error on non-existent partition, `sort` with default row fallback / explicit config / `default` row fallback / no-match skip / empty input / wrong type, `loadconfig` with null file / valid file / unrecognised columns / unrecognised attributes / missing file / header-only file, `applyattr` on missing path / null column / invalid attribute / valid path, `gc` log count.
+The suite injects mock loggers: a no-op logger, and a capturing logger that records `(level;ctx;msg)` so log behaviour can be asserted. On-disk behaviour (sort, attributes, `savedown`/`appenddown`) is exercised against real splayed partitions and cleaned up afterwards. It covers: dependency-injection validation; `readcsv` returns / column-order independence / header-validation failures; `sort` validation / edge cases / resolution / on-disk results / multi-dir / non-fatal partition failure; `savedown` write+sort (default and explicit config, and a table without `sym`); `appenddown` append-without-sort then explicit sort, and the non-existent-partition error; `applyattr`; `manipulate`; `postreplay`; `gc`; and the logging contract.
 
 ---
 
 ## Exported symbols
 
 ```q
-export:([init;savedown;appenddown;sort;applyattr;loadconfig;gc])
+export:([init;readcsv;sort;applyattr;savedown;appenddown;manipulate;postreplay;gc])
 ```
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 1e3fdc9c..35c4de4d 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -1,114 +1,220 @@
-/ sort params table - default row sorts all tables by time ascending
-params:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
+/ dbwrite - sort, attribute application, save-down manipulation, and GC utilities for on-disk data
+/ used by processes that persist data to disk (rdb, wdb, tickerlogreplay)
 
+/ attributes that may legitimately appear in a config (empty leaves a column unattributed)
+validatts:``p`s`g`u;
 
-/ load and validate sort.csv into .z.M.params
-/ file: hsym path; null warns and resets params to default row
-loadconfig:{[file]
+/ built-in fallback config - sort every table by time ascending when no config is supplied
+defaultparams:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
+
+init:{[config;deps]
+  / wire injected deps and optional module config
+  / config: dict with optional `savedownmanipulation (tabname!unary fn applied pre-write)
+  / deps: `log!(logdict) where logdict is `info`warn`error!(infofn;warnfn;errfn) - required
+  if[99h<>type deps;
+    '"di.dbwrite: deps must be a dict with a `log key; see di.log for a default logger"];
+  if[not `log in key deps;
+    '"di.dbwrite: log dependency is required; pass `info`warn`error functions keyed on `log"];
+  if[99h<>type deps`log;
+    '"di.dbwrite: log value must be a dict of `info`warn`error functions"];
+  if[not all (`info`warn`error) in key deps`log;
+    '"di.dbwrite: log dict must have `info`warn`error keys; got: ",(", " sv string key deps`log)];
+  .z.m.log:deps`log;
+  .z.m.savedownmanipulation:()!();
+  if[99h=type config;
+    if[`savedownmanipulation in key config;
+      .z.m.savedownmanipulation:config`savedownmanipulation]];
+  };
+
+readcsv:{[file]
+  / read a sort-config csv and return it as a table (does not store it); pass the result to sort
+  / the csv must have the columns tabname,att,column,sort (in any order)
   if[not -11h=type file;
-    '"loadconfig: file must be a symbol, got type ",(string type file)];
-  if[null file;
-    .z.m.logwarn[`dbwrite;"loadconfig called with no file; resetting params to default"];
-    @[.z.M;`params;:;([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b)]];
-  if[not null file;
-    file:hsym file;
-    p:@[
-      {.z.m.loginfo[`dbwrite;"retrieving sort settings from ",string x];("SSSB";enlist",")0:x};
-      file;
-      {[f;e]'"failed to open ",string[f],": ",e}[file]
-    ];
-    if[not all spcb:(spc:cols p) in `tabname`att`column`sort;
-      '"unrecognised columns (",(", " sv string spc where not spcb),") in ",string file];
-    if[not all atb:(at:distinct p`att) in ``p`s`g`u;
-      '"unrecognised attribute(s): ",", " sv string at where not atb];
-    @[.z.M;`params;:;p]];
-  };
-
-/ apply a single kdb+ attribute to an on-disk column; logs and swallows errors
+    '"di.dbwrite: readcsv file must be a symbol, got type ",string type file];
+  file:hsym file;
+  t:parsecsv @[readfile; file; readerr[file]];
+  .z.m.log[`info][`dbwrite;"read ",(string count t)," sort config row(s) from ",string file];
+  :t;
+  };
+
+/ internal - protected file read; only the i/o so a genuine read failure gets the readerr message
+readfile:{[file]
+  / returns the raw csv lines; header validation and parsing happen in parsecsv
+  .z.m.log[`info][`dbwrite;"reading sort config from ",string file];
+  :read0 file;
+  };
+
+/ internal - log and rethrow a csv read failure
+readerr:{[file;e]
+  / build the message once, surface it under the dbwrite context, then rethrow it to the caller
+  m:"failed to read ",string[file],": ",e;
+  .z.m.log[`error][`dbwrite;m];
+  'm;
+  };
+
+/ internal - validate the header and parse csv lines into a config table
+parsecsv:{[lines]
+  / map types by column name so the csv column order does not matter; reject any other shape
+  / outside the readfile i/o trap so a bad header surfaces as a clear di.dbwrite: error
+  if[0=count lines;
+    '"di.dbwrite: csv has no header row"];
+  hdr:`$"," vs first lines;
+  if[not (asc distinct hdr)~`att`column`sort`tabname;
+    '"di.dbwrite: csv header must be exactly tabname,att,column,sort; got: ",", " sv string hdr];
+  types:{$[x=`sort;"B";"S"]} each hdr;
+  :`tabname`att`column`sort#(types;enlist",") 0: lines;
+  };
+
+/ internal - validate a sort-config table, signalling a clear error if it is malformed
+checkconfig:{[t]
+  / guards every sort call so a hand-built or csv-derived table is rejected early if wrong
+  if[98h<>type t;
+    '"di.dbwrite: config must be a table with columns `tabname`att`column`sort"];
+  c:cols t;
+  badcols:c where not c in `tabname`att`column`sort;
+  if[count badcols;
+    '"di.dbwrite: unrecognised config column(s): ",", " sv string badcols];
+  missingcols:(`tabname`att`column`sort) where not (`tabname`att`column`sort) in c;
+  if[count missingcols;
+    '"di.dbwrite: missing required config column(s): ",", " sv string missingcols];
+  if[not 1h=type t`sort;
+    '"di.dbwrite: the sort column must be boolean"];
+  badatts:at where not (at:distinct t`att) in validatts;
+  if[count badatts;
+    '"di.dbwrite: unrecognised attribute(s) in att column: ",", " sv string badatts];
+  };
+
+sort:{[config;tabname;dirs]
+  / sort and apply attributes to the on-disk partition dirs for one table using config
+  / config: a sort-config table (build it directly or via readcsv); (::) uses the default config
+  / tabname: symbol table name; dirs: hsym or list of hsyms (partition directories)
+  config:$[(::)~config; defaultparams; config];
+  checkconfig config;
+  if[not -11h=type tabname;
+    '"di.dbwrite: tabname must be a symbol, got type ",string type tabname];
+  st:string tabname;
+  .z.m.log[`info][`dbwrite;"sorting the ",st," table"];
+  sp:getsortparams[config;tabname;st];
+  if[not count sp; :()];
+  sortdir[sp] each distinct (),dirs;
+  .z.m.log[`info][`dbwrite;"finished sorting the ",st," table"];
+  };
+
+/ internal - log a sort message then return the resolved rows
+logreturn:{[lvl;msg;rows]
+  / keeps each branch body in getsortparams to a single statement
+  .z.m.log[lvl][`dbwrite;msg];
+  :rows;
+  };
+
+/ internal - resolve which config rows apply to a table
+getsortparams:{[config;tab;st]
+  / tab is the table-name symbol (NOT a table); named to avoid clashing with the tabname column
+  / a table uses its own rows; unlisted tables fall back to the default row, else are skipped
+  if[count tabsp:select from config where tabname=tab;
+    :logreturn[`info;"sort params found for: ",st;tabsp]];
+  if[count defsp:select from config where tabname=`default;
+    :logreturn[`info;"no sort params for: ",st,"; using defaults";defsp]];
+  :logreturn[`warn;"no sort params for: ",st,"; skipping sort";0#config];
+  };
+
+/ internal - log a sort failure without rethrowing so remaining partitions still run
+sorterr:{[sc;dl;e]
+  / a single partition failure should not halt the whole run
+  .z.m.log[`error][`dbwrite;"failed to sort ",string[dl]," by ",(", " sv string sc),": ",e];
+  :();
+  };
+
+/ internal - sort one partition directory by the given columns
+sortcolumns:{[dloc;sortcols]
+  / split out of sortdir so the conditional body there stays a single statement
+  .z.m.log[`info][`dbwrite;"sorting ",string[dloc]," by: ",", " sv string sortcols];
+  .[xasc;(sortcols;dloc);
+    sorterr[sortcols;dloc]];
+  };
+
+/ internal - sort columns and apply attributes for a single on-disk partition directory
+sortdir:{[sp;dloc]
+  / sort by the columns flagged sort=1b, then attribute the columns that request one
+  sortcols:exec column from sp where sort, not null column;
+  if[count sortcols; sortcolumns[dloc;sortcols]];
+  attrcols:select column,att from sp where att in `p`s`g`u;
+  if[count attrcols; applyattr[dloc;;]'[attrcols`column;attrcols`att]];
+  };
+
+/ internal - log an attribute application failure without rethrowing
+attrerr:{[dl;cn;at;e]
+  / logs failure and continues so other columns and partitions still get processed
+  .z.m.log[`error][`dbwrite;"unable to apply ",string[at]," attr to ",string[cn]," in ",string[dl],": ",e];
+  :();
+  };
+
 applyattr:{[dloc;colname;att]
-  .z.m.loginfo[`dbwrite;"applying ",string[att]," attr to ",string[colname]," in ",string dloc];
-  if[null colname;
-    .z.m.logerr[`dbwrite;"applyattr called with null column name in ",string dloc]];
-  if[not null colname;
-    $[not att in `p`s`g`u;
-      .z.m.logerr[`dbwrite;"applyattr: invalid attribute ",string[att]," for ",string[colname]," in ",string dloc];
-      .[{@[x;y;z#]};(dloc;colname;att);
-        {[dloc;colname;att;e]
-          .z.m.logerr[`dbwrite;"unable to apply ",string[att]," attr to ",string[colname]," in ",string[dloc],": ",e]
-        }[dloc;colname;att]
-      ]]];
-  };
-
-/ sort an on-disk table partition and apply attributes per sort.csv config
-/ d: tabname | (tabname;dir) | (tabname;list of dirs)
-sort:{[d]
-  $[not count d;
-    ();
-    not (type d) in -11 0 11h;
-    [.z.m.logerr[`dbwrite;"sort: d must be a symbol or list, got type ",(string type d)];()];
-    [
-      .z.m.loginfo[`dbwrite;"sorting ",(st:string t:first d)," table"];
-      sp:$[count tabsp:select from .z.M.params where tabname=t;
-          [.z.m.loginfo[`dbwrite;"sort params found for: ",st];tabsp];
-        count defsp:select from .z.M.params where tabname=`default;
-          [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; using defaults"];defsp];
-        [.z.m.logwarn[`dbwrite;"no sort params for: ",st,"; skipping sort"];:()]];
-      {[sp;dloc]
-        if[count sortcols:exec column from sp where sort,not null column;
-          .z.m.loginfo[`dbwrite;"sorting ",string[dloc]," by: ",", " sv string sortcols];
-          .[xasc;(sortcols;dloc);
-            {[sc;dl;e]
-              .z.m.logerr[`dbwrite;"failed to sort ",string[dl]," by ",(", " sv string sc),": ",e]
-            }[sortcols;dloc]]];
-        if[count attrcols:select column,att from sp where not null att;
-          .z.M.applyattr[dloc;;]'[attrcols`column;attrcols`att]];
-      }[sp] each distinct (),last d;
-      .z.m.loginfo[`dbwrite;"finished sorting ",st," table"]
-    ]]
-  };
-
-
-/ write table to a date-partitioned hdb: enumerate syms, apply p# to sym if present, write, then sort
-/ dir: hdb root (hsym); part: partition value (date/month/int); tabname: symbol; data: in-memory table
-savedown:{[dir;part;tabname;data]
-  .z.m.loginfo[`dbwrite;"saving ",string[tabname]," partition ",string[part]," to ",string dir];
+  / apply a single kdb+ attribute to an on-disk column; logs and swallows errors so a run continues
+  / skip anything that is not a real attribute - covers the empty none-sentinel and any bad value
+  if[not att in `p`s`g`u; :()];
+  .z.m.log[`info][`dbwrite;"applying ",string[att]," attr to ",string[colname]," in ",string dloc];
+  .[{@[x;y;z#]};
+    (dloc;colname;att);
+    attrerr[dloc;colname;att]];
+  };
+
+savedown:{[config;dir;part;tabname;data]
+  / write an in-memory table to a date-partitioned hdb partition, then sort it per config
+  / config: a sort-config table, or (::) for the built-in default; dir: hdb root (hsym)
+  / part: partition value (date/month/int); tabname: symbol; data: in-memory table
+  / enumerates syms against the hdb sym file and applies p# to sym before writing
+  .z.m.log[`info][`dbwrite;"saving ",string[tabname]," partition ",string[part]," to ",string dir];
   path:` sv (.Q.par[dir;part;tabname];`);
   data:.Q.en[dir;data];
-  path set $[`sym in cols data;@[data;`sym;{`p#x}];data];
-  sort[(tabname;path)];
-  .z.m.loginfo[`dbwrite;"finished saving ",string tabname];
+  path set $[`sym in cols data; @[data;`sym;{`p#x}]; data];
+  sort[config;tabname;path];
+  .z.m.log[`info][`dbwrite;"finished saving ",string tabname];
   };
 
-/ append data to an existing on-disk partition; enumerate syms but do not sort
-/ call sort separately when the partition is complete
-/ dir: hdb root (hsym); part: partition value; tabname: symbol; data: in-memory table
 appenddown:{[dir;part;tabname;data]
-  .z.m.loginfo[`dbwrite;"appending ",string[tabname]," partition ",string[part]," in ",string dir];
+  / append rows to an existing on-disk partition (enumerates syms); does not sort
+  / call sort separately once the partition is complete, to avoid re-sorting on every append
+  / dir: hdb root (hsym); part: partition value; tabname: symbol; data: in-memory table
+  .z.m.log[`info][`dbwrite;"appending ",string[tabname]," partition ",string[part]," in ",string dir];
   path:` sv (.Q.par[dir;part;tabname];`);
   if[not count @[key;path;{`$()}];
-    '"appenddown: partition does not exist at ",string path];
+    '"di.dbwrite: appenddown partition does not exist at ",string path];
   .[path;();,;.Q.en[dir;data]];
-  .z.m.loginfo[`dbwrite;"finished appending ",string tabname];
+  .z.m.log[`info][`dbwrite;"finished appending ",string tabname];
   };
 
-/ format current process memory stats as a loggable string
-memstats:{[]"mem stats: ",{"; "sv "=" sv'flip(string key x;(string value x),\:" MB")}`long$.Q.w[]%1048576};
+manipulate:{[t;x]
+  / apply registered pre-write manipulation to table x of type t; unchanged if none registered
+  if[not t in key .z.m.savedownmanipulation; :x];
+  :@[.z.m.savedownmanipulation[t]; x; maniperr[x]];
+  };
 
-/ run .Q.gc[] and log before/after memory stats
-gc:{
-  .z.m.loginfo[`dbwrite;"starting garbage collect. ",.z.M.memstats[]];
-  r:.Q.gc[];
-  .z.m.loginfo[`dbwrite;"garbage collection returned ",(string `long$r%1048576),"MB. ",.z.M.memstats[]]
+/ internal - log a failed save-down manipulation and fall back to the unmodified table
+maniperr:{[x;e]
+  / called as the manipulate error handler; the unmodified table is the safe fallback
+  .z.m.log[`error][`dbwrite;"save-down manipulation failed: ",e];
+  :x;
   };
 
-init:{[config;deps]
-  / config: unused, pass (::)
-  / deps: `log!(logdict) - `info`warn`error!(infofunc;warnfunc;errfunc) - required
-  logdict:$[99h=type deps;$[(`log in key deps) and not (::)~deps`log;deps`log;()!()];()!()];
-  if[not count logdict;
-    '"di.dbwrite: log dependency is required; pass `info`warn`error functions - see di.log or refer to confluence documentation";
-  ];
-  .z.m.loginfo:logdict`info;
-  .z.m.logwarn:logdict`warn;
-  .z.m.logerr:logdict`error;
+/ post-EOD hook - called after all tables written and sorted; override at the call site
+postreplay:{[d;p]};
+
+/ internal - render a memory-usage dict as a "key=val MB; ..." string
+fmtmem:{[m]
+  / m: dict of MB values keyed by .Q.w field name
+  :"; " sv "=" sv' flip (string key m; (string value m),\:" MB");
+  };
+
+/ format current process memory stats as a loggable string
+memstats:{[]
+  / convert .Q.w[] (bytes) to MB and render it via fmtmem
+  :"mem stats: ",fmtmem `long$.Q.w[]%1048576;
+  };
+
+gc:{[]
+  / run .Q.gc[] and log before/after memory stats
+  .z.m.log[`info][`dbwrite;"starting garbage collect. ",memstats[]];
+  r:.Q.gc[];
+  .z.m.log[`info][`dbwrite;"garbage collection returned ",(string `long$r%1048576),"MB. ",memstats[]];
   };
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index 917d5dd6..c402f0a4 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -3,4 +3,4 @@
 
 \l ::dbwrite.q
 
-export:([init;savedown;appenddown;sort;applyattr;loadconfig;gc])
\ No newline at end of file
+export:([init;readcsv;sort;applyattr;savedown;appenddown;manipulate;postreplay;gc])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index c742d5ff..32830167 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -1,150 +1,207 @@
 action,ms,bytes,lang,code,repeat,minver,comment
-/ Pre-test set-up: load module and mock loggers, and initialise module with mocks
-before,0,0,q,dbwrite:use`di.dbwrite,1,,load di.dbwrite module
-before,0,0,q,logcount:0,1,,initialise log call counter
-before,0,0,q,loginfo:{[c;m] logcount::logcount+1},1,,mock info logger
-before,0,0,q,logwarn:{[c;m] logcount::logcount+1},1,,mock warn logger
-before,0,0,q,logerr:{[c;m] logcount::logcount+1},1,,mock error logger
-before,0,0,q,"logdep:`info`warn`error!(loginfo;logwarn;logerr)",1,,build log dep dict
-before,0,0,q,"deps:(enlist`log)!enlist logdep",1,,wrap in deps dict
-before,0,0,q,dbwrite.init[(::);deps],1,,initialise module with mock loggers
-
-/ Test 1: init wires injected logger into individual loginfo/logwarn/logerr slots
-true,0,0,q,logdep[`info]~.m.di.0dbwrite.loginfo,1,1,injected info function stored
-true,0,0,q,logdep[`warn]~.m.di.0dbwrite.logwarn,1,1,injected warn function stored
-true,0,0,q,logdep[`error]~.m.di.0dbwrite.logerr,1,1,injected error function stored
-
-/ Test 2: init errors when log dep not provided
-fail,0,0,q,dbwrite.init[(::);(::)],1,1,init without log dep throws an error
-fail,0,0,q,dbwrite.init[(::);`log!(::)],1,1,init with log set to (::) throws an error
-
-/ Test 3: sort uses default row when table name not in config
-run,0,0,q,`:dbwrite_defp_tp/.d set `time`sym,1,,write column order file
-run,0,0,q,`:dbwrite_defp_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
-run,0,0,q,`:dbwrite_defp_tp/sym set `IBM`AAPL,1,,write sym column
-run,0,0,q,"dbwrite.sort[(`anytable;`:dbwrite_defp_tp/)]",1,,sort using default row
-true,0,0,q,(asc exec time from get `:dbwrite_defp_tp/)~exec time from get `:dbwrite_defp_tp/,1,,time column sorted ascending by default row
-
-/ Test 4: loadconfig null resets params to default row
-run,0,0,q,dbwrite.loadconfig[`],1,,call loadconfig with null file
-true,0,0,q,.m.di.0dbwrite.params~([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b),1,,params reset to default row
-
-/ Test 5: loadconfig - valid config file loaded and parsed correctly
-run,0,0,q,cfgfile:`:dbwrite_test.csv,1,,temp sort config path
-run,0,0,q,"cfgfile 0:(""tabname,att,column,sort"";""trade,p,sym,1"";""trade,,price,0"")",1,,write test sort config
-run,0,0,q,dbwrite.loadconfig[cfgfile],1,,loadconfig with valid csv succeeds
-true,0,0,q,2=count .m.di.0dbwrite.params,1,,two rows loaded into params
-true,0,0,q,all `trade=exec tabname from .m.di.0dbwrite.params,1,,both rows are for trade table
-
-/ Test 5 cont: loadconfig - invalid and edge-case inputs
-true,0,0,q,@[dbwrite.loadconfig;`:nonexistent_file_xyz;{1b}],1,,nonexistent file throws
-true,0,0,q,@[dbwrite.loadconfig;42;{1b}],1,,non-symbol arg throws
-run,0,0,q,badcols:`:dbwrite_badcols.csv,1,,
-run,0,0,q,"badcols 0:(""wrongcol,att,column,sort"";""trade,p,sym,1"")",1,,csv with unrecognised column name
-true,0,0,q,@[dbwrite.loadconfig;badcols;{1b}],1,,unrecognised column throws
-run,0,0,q,badatt:`:dbwrite_badatt.csv,1,,
-run,0,0,q,"badatt 0:(""tabname,att,column,sort"";""trade,z,sym,1"")",1,,csv with unrecognised attribute
-true,0,0,q,@[dbwrite.loadconfig;badatt;{1b}],1,,unrecognised attribute throws
-run,0,0,q,emptycfg:`:dbwrite_empty.csv,1,,
-run,0,0,q,"emptycfg 0:enlist""tabname,att,column,sort""",1,,csv with header only
-run,0,0,q,dbwrite.loadconfig[emptycfg],1,,loadconfig with header-only csv
-true,0,0,q,0=count .m.di.0dbwrite.params,1,,params is empty after header-only csv
-
-/ Test 6: applyattr - error cases
-run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,"dbwrite.applyattr[`:nonexist_attr_dir/;`sym;`p]",1,,apply to non-existent path
-true,0,0,q,2=logcount,1,,loginfo (before attempt) and logerr (on failure) each called once
-
-/ Test 6 cont: applyattr - null column name logs error and does not throw
-run,0,0,q,`:dbwrite_attr_tp/.d set `sym`price,1,,write column order file
-run,0,0,q,`:dbwrite_attr_tp/sym set `IBM`AAPL`MSFT,1,,write sym column
-run,0,0,q,`:dbwrite_attr_tp/price set 200 100 300f,1,,write price column
-run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`;`p]",1,,apply with null column name
-true,0,0,q,2=logcount,1,,loginfo and logerr both called for null column
-
-/ Test 6 cont: applyattr - invalid att logs error
-run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`z]",1,,apply with invalid attribute
-true,0,0,q,2=logcount,1,,loginfo and logerr both called for invalid att
-
-/ Test 7: applyattr - attribute applied and logged when path exists
-run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,"dbwrite.applyattr[`:dbwrite_attr_tp/;`sym;`p]",1,,apply p attr to sym column
-true,0,0,q,1=logcount,1,,loginfo called once (no error)
-true,0,0,q,`p=attr get `:dbwrite_attr_tp/sym,1,,p attribute applied on disk
-
-/ Test 8: sort - skip when table not in config and no default row
-run,0,0,q,dbwrite.loadconfig[cfgfile],1,,reload trade-only config (no default row)
-run,0,0,q,logcount:0,1,,reset counter
-true,0,0,q,()~dbwrite.sort[`no_params_table_xyz],1,,returns () when table not in config
-true,0,0,q,2=logcount,1,,loginfo (sorting start) and logwarn (skip) both called
-true,0,0,q,()~dbwrite.sort[()],1,,empty list returns ()
-
-/ Test 8 cont: sort - wrong type logs error and returns ()
-run,0,0,q,logcount:0,1,,reset counter
-true,0,0,q,()~dbwrite.sort[42],1,,wrong type (long) returns ()
-true,0,0,q,1=logcount,1,,logerr called once for wrong type
-
-/ Test 9: sort - sorting and attribute application when config entry exists for table
-run,0,0,q,`:dbwrite_sort_tp/.d set `sym`price,1,,write column order file
-run,0,0,q,`:dbwrite_sort_tp/sym set `IBM`AAPL`MSFT,1,,write unsorted sym column
-run,0,0,q,`:dbwrite_sort_tp/price set 200 100 300f,1,,write price column
-run,0,0,q,"dbwrite.sort[(`trade;`:dbwrite_sort_tp/)]",1,,sort trade table in test partition
-true,0,0,q,`AAPL`IBM`MSFT~exec sym from get `:dbwrite_sort_tp/,1,,sorted ascending by sym
-true,0,0,q,`p=attr get `:dbwrite_sort_tp/sym,1,,p attribute applied to sym on disk
-
-/ Test 10: sort - default row used after resetting params with loadconfig null
-run,0,0,q,dbwrite.loadconfig[`],1,,reset params to default row via null loadconfig
-run,0,0,q,`:dbwrite_reset_tp/.d set `time`sym,1,,write column order file
-run,0,0,q,`:dbwrite_reset_tp/time set 2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000,1,,write unsorted time column
-run,0,0,q,`:dbwrite_reset_tp/sym set `IBM`AAPL,1,,write sym column
-run,0,0,q,`:dbwrite_reset_tp/price set 200 100f,1,,write price column
-run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,"dbwrite.sort[(`othertable;`:dbwrite_reset_tp/)]",1,,sort using default row
-true,0,0,q,(asc exec time from get `:dbwrite_reset_tp/)~exec time from get `:dbwrite_reset_tp/,1,,sorted ascending by time via default row
-true,0,0,q,4=logcount,1,,loginfo x2 (sorting start + sort-by) + logwarn x1 (using defaults) + loginfo x1 (finished)
-
-/ Test 11: gc calls loginfo twice and does not throw
-run,0,0,q,logcount:0,1,,reset counter
-run,0,0,q,dbwrite.gc[],1,,run garbage collect
-true,0,0,q,2=logcount,1,,loginfo called twice (start and end)
-
-/ Test 12: savedown - write table to hdb partition
-run,0,0,q,dbwrite.loadconfig[`],1,,ensure default sort params (by time)
-run,0,0,q,"sdtbl:([]time:2024.01.01D09:00:00.000000000 2024.01.01D08:00:00.000000000;sym:`IBM`AAPL;price:100 200f)",1,,unsorted test table
-run,0,0,q,"dbwrite.savedown[`:dbwrite_sd_hdb;2024.01.01;`trade;sdtbl]",1,,write to hdb partition
-run,0,0,q,"sdpath:` sv (.Q.par[`:dbwrite_sd_hdb;2024.01.01;`trade];`)",1,,partition path with trailing slash for xasc compatibility
-true,0,0,q,2=count get sdpath,1,,two rows written to partition
-true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,,rows sorted by time ascending after sort
-true,0,0,q,0<count get `:dbwrite_sd_hdb/sym,1,,sym enumeration file created in hdb root
-
-/ Test 12 cont: savedown - table without sym column
-run,0,0,q,"nosymtbl:([]time:enlist 2024.01.02D10:00:00.000000000;price:enlist 50f)",1,,table with no sym column
-run,0,0,q,"dbwrite.savedown[`:dbwrite_sd_hdb;2024.01.02;`nosymtrade;nosymtbl]",1,,savedown table without sym column
-true,0,0,q,"1=count get ` sv (.Q.par[`:dbwrite_sd_hdb;2024.01.02;`nosymtrade];`)",1,,row written correctly for table without sym
-
-/ Test 13: appenddown - append rows to existing partition without sorting
-run,0,0,q,"apptbl:([]time:enlist 2024.01.01D07:00:00.000000000;sym:enlist`MSFT;price:enlist 300f)",1,,row with earlier time than existing data
-run,0,0,q,"dbwrite.appenddown[`:dbwrite_sd_hdb;2024.01.01;`trade;apptbl]",1,,append to existing partition
-true,0,0,q,3=count get sdpath,1,,three rows on disk after appenddown
-true,0,0,q,not (asc exec time from get sdpath)~exec time from get sdpath,1,,rows not yet sorted (no auto-sort on appenddown)
-
-/ Test 13 cont: sort explicitly after appenddown
-run,0,0,q,"dbwrite.sort[(`trade;sdpath)]",1,,sort partition after appending
-true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,,sorted by time after explicit sort call
-
-/ Test 13 cont: appenddown into non-existent partition throws
-true,0,0,q,"@[dbwrite.appenddown[`:dbwrite_sd_hdb;2099.01.01;`nonexist;];sdtbl;{1b}]",1,,appenddown into non-existent partition throws
-
-
-/ Clean up all created on-disk files and directories
-after,0,0,q,@[hdel;`:dbwrite_defp_tp/;{}],1,,cleanup defaultparams test partition
-after,0,0,q,@[hdel;cfgfile;{}],1,,remove temp sort config
-after,0,0,q,@[hdel;badcols;{}],1,,remove temp bad columns file
-after,0,0,q,@[hdel;badatt;{}],1,,remove temp bad attribute file
-after,0,0,q,@[hdel;emptycfg;{}],1,,remove empty config file
-after,0,0,q,@[hdel;`:dbwrite_attr_tp/;{}],1,,cleanup attr test partition
-after,0,0,q,@[hdel;`:dbwrite_sort_tp/;{}],1,,cleanup sort test partition
-after,0,0,q,@[hdel;`:dbwrite_reset_tp/;{}],1,,cleanup reset test partition
-after,0,0,q,@[hdel;`:dbwrite_sd_hdb;{}],1,,cleanup savedown/upsert test hdb
+before,0,0,q,dbwrite:use`di.dbwrite,1,1,load module
+before,0,0,q,mylog:`info`warn`error!({[c;m]};{[c;m]};{[c;m]}),1,1,define no-op mock logger
+before,0,0,q,"caplog:([] fn:`symbol$();ctx:`symbol$();msg:())",1,1,initialise log capture table
+before,0,0,q,logcap:`info`warn`error!({[c;m] `caplog upsert(`info;c;m)};{[c;m] `caplog upsert(`warn;c;m)};{[c;m] `caplog upsert(`error;c;m)}),1,1,define capturing mock logger
+before,0,0,q,"rmrf:{[d] if[11h=type k:key d; rmrf each ` sv/:d,/:k]; @[hdel;d;{}]}",1,1,recursive delete helper for on-disk cleanup
+before,0,0,q,"`:tmp_dbw_valid.csv 0: (""tabname,att,column,sort"";""trade,p,sym,1"";""trade,,time,0"";""default,p,sym,1"")",1,1,write valid sort csv
+before,0,0,q,"`:tmp_dbw_badcols.csv 0: (""tabname,attr,col,sort"";""trade,p,sym,1"")",1,1,write csv with bad column names
+before,0,0,q,"`:tmp_dbw_badatt.csv 0: (""tabname,att,column,sort"";""trade,z,sym,1"")",1,1,write csv with invalid attribute value
+before,0,0,q,"`:tmp_dbw_3col.csv 0: (""tabname,att,sort"";""trade,p,1"")",1,1,write csv missing a column
+before,0,0,q,"`:tmp_dbw_empty.csv 0: enlist ""tabname,att,column,sort""",1,1,write header-only sort csv
+before,0,0,q,"`:tmp_dbw_nodfl.csv 0: (""tabname,att,column,sort"";""trade,p,sym,1"")",1,1,write sort csv with no default row
+before,0,0,q,"`:tmp_dbw_5col.csv 0: (""tabname,att,column,sort,extra"";""trade,p,sym,1,foo"")",1,1,write csv with an extra column
+before,0,0,q,"`:tmp_dbw_reorder.csv 0: (""tabname,att,sort,column"";""trade,p,1,sym"";""trade,,0,time"";""default,p,1,sym"")",1,1,write valid csv with columns reordered
+before,0,0,q,`:tmp_dbw_nohdr.csv 0: (),1,1,write a zero-line csv (no header)
+before,0,0,q,dbwrite.init[(::); enlist[`log]!enlist mylog],1,1,init with no-op logger
+before,0,0,q,cfg:([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,reusable valid config table (hand-built)
+before,0,0,q,nodflcfg:dbwrite.readcsv `:tmp_dbw_nodfl.csv,1,1,reusable config with a trade row but no default
+
+comment,,,,,,,init - dependency injection validation (type errors / empty / null)
+fail,0,0,q,dbwrite.init[(::);(::)],1,1,init rejects :: as deps
+fail,0,0,q,dbwrite.init[(::);42],1,1,init rejects a non-dict deps value
+fail,0,0,q,dbwrite.init[(::);()!()],1,1,init rejects empty dict
+fail,0,0,q,dbwrite.init[(::);enlist[`other]!enlist mylog],1,1,init rejects dict missing log key
+fail,0,0,q,dbwrite.init[(::);enlist[`log]!enlist(::)],1,1,init rejects null log dep
+fail,0,0,q,dbwrite.init[(::);enlist[`log]!enlist 42],1,1,init rejects non-dict log value
+fail,0,0,q,dbwrite.init[(::);enlist[`log]!enlist(`info`warn!(mylog`info;mylog`warn))],1,1,init rejects log dict missing required key
+true,0,0,q,"""di.dbwrite:""~11#@[{dbwrite.init[(::);(::)]};(::);{x}]",1,1,error message is prefixed di.dbwrite
+run,0,0,q,dbwrite.init[(::); enlist[`log]!enlist mylog],1,1,re-init with valid log dep
+
+comment,,,,,,,readcsv - returns a config table from a csv
+true,0,0,q,3=count dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv returns a 3-row config table
+true,0,0,q,`tabname`att`column`sort~cols dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv table has the correct column schema
+true,0,0,q,(dbwrite.readcsv `:tmp_dbw_valid.csv)~([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,readcsv round-trips the csv content
+true,0,0,q,0=count dbwrite.readcsv `:tmp_dbw_empty.csv,1,1,readcsv returns an empty table for a header-only csv
+true,0,0,q,(dbwrite.readcsv `:tmp_dbw_reorder.csv)~dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv is column-order independent
+true,0,0,q,`tabname`att`column`sort~cols dbwrite.readcsv `:tmp_dbw_reorder.csv,1,1,readcsv normalises reordered columns to canonical order
+
+comment,,,,,,,readcsv - input and header validation failures
+fail,0,0,q,dbwrite.readcsv `:nonexistent_file.csv,1,1,readcsv errors on a missing file
+fail,0,0,q,dbwrite.readcsv 42,1,1,readcsv errors on a non-symbol file argument
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_badcols.csv,1,1,readcsv rejects a csv with wrong column names
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_3col.csv,1,1,readcsv rejects a csv missing a required column
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_5col.csv,1,1,readcsv rejects a csv with an extra column
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_nohdr.csv,1,1,readcsv rejects a csv with no header row
+
+comment,,,,,,,sort - happy path via both a hand-built table and a csv
+run,0,0,q,dbwrite.sort[cfg;`trade;()],1,1,sort accepts a hand-built config table
+run,0,0,q,dbwrite.sort[dbwrite.readcsv `:tmp_dbw_valid.csv;`trade;()],1,1,sort accepts a config table read from a csv
+
+comment,,,,,,,sort - config validation (type errors / bad content)
+fail,0,0,q,dbwrite.sort[42;`trade;enlist`:/],1,1,sort rejects a non-table config
+fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; bad:enlist`p; column:enlist`sym; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects unrecognised config columns
+fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`sym);`trade;enlist`:/],1,1,sort rejects a missing required config column
+fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`z; column:enlist`sym; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects unknown attribute values
+fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`sym; sort:enlist 1);`trade;enlist`:/],1,1,sort rejects a non-boolean sort column
+fail,0,0,q,dbwrite.sort[dbwrite.readcsv `:tmp_dbw_badatt.csv;`trade;enlist`:/],1,1,sort rejects csv config with invalid attributes
+
+comment,,,,,,,sort - edge cases (empty config / null att / empty dirs / atom dir / type error)
+true,0,0,q,()~dbwrite.sort[([] tabname:`symbol$();att:`symbol$();column:`symbol$();sort:`boolean$());`trade;enlist`:/],1,1,empty config yields no work and returns ()
+run,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`; column:enlist`sym; sort:enlist 0b);`trade;enlist`:/],1,1,sort accepts a null (no) attribute row
+run,0,0,q,dbwrite.sort[cfg;`trade;()],1,1,sort handles an empty partition list without error
+run,0,0,q,dbwrite.sort[cfg;`trade;`:/],1,1,sort accepts a single atom hsym dir not just a list
+fail,0,0,q,dbwrite.sort[cfg;42;enlist`:/],1,1,sort errors on a non-symbol table name
+true,0,0,q,"""di.dbwrite:""~11#@[{dbwrite.sort[cfg;42;enlist`:/]};(::);{x}]",1,1,non-symbol table name gives a di.dbwrite:-prefixed error
+
+comment,,,,,,,sort - params resolution (table-specific / default / none / (::) fallback)
+run,0,0,q,dbwrite.sort[cfg;`trade;enlist`:/],1,1,sort uses the table-specific params row
+run,0,0,q,dbwrite.sort[cfg;`other;enlist`:/],1,1,sort falls back to the default params row
+true,0,0,q,()~dbwrite.sort[nodflcfg;`other;enlist`:/],1,1,sort returns () when no params found and no default row
+run,0,0,q,dbwrite.sort[(::);`anytable;()],1,1,sort with (::) config uses the built-in default config
+
+comment,,,,,,,sort - on-disk end-to-end (explicit config sorts by sym and applies p#)
+run,0,0,q,`:dbw_sort_tp/.d set `sym`price,1,1,write column order file
+run,0,0,q,`:dbw_sort_tp/sym set `IBM`AAPL`MSFT,1,1,write unsorted sym column
+run,0,0,q,`:dbw_sort_tp/price set 200 100 300f,1,1,write price column
+run,0,0,q,dbwrite.sort[cfg;`trade;`:dbw_sort_tp/],1,1,sort the trade partition by its config
+true,0,0,q,`AAPL`IBM`MSFT~exec sym from get `:dbw_sort_tp/,1,1,sym sorted ascending on disk
+true,0,0,q,`p=attr get `:dbw_sort_tp/sym,1,1,p attribute applied to sym on disk
+
+comment,,,,,,,sort - on-disk end-to-end ((::) default config sorts by time)
+run,0,0,q,`:dbw_def_tp/.d set `time`sym,1,1,write column order file
+run,0,0,q,`:dbw_def_tp/time set 2024.01.01D09:00 2024.01.01D08:00,1,1,write unsorted time column
+run,0,0,q,`:dbw_def_tp/sym set `IBM`AAPL,1,1,write sym column
+run,0,0,q,dbwrite.sort[(::);`anytable;`:dbw_def_tp/],1,1,sort using the built-in default config
+true,0,0,q,(asc exec time from get `:dbw_def_tp/)~exec time from get `:dbw_def_tp/,1,1,time sorted ascending by the default config
+
+comment,,,,,,,sort - multiple partition dirs in one call, and a non-fatal partition failure
+run,0,0,q,`:dbw_md1/.d set `sym`px,1,1,write md1 column order
+run,0,0,q,`:dbw_md1/sym set `c`a`b,1,1,write md1 unsorted sym
+run,0,0,q,`:dbw_md1/px set 1 2 3f,1,1,write md1 price
+run,0,0,q,`:dbw_md2/.d set `sym`px,1,1,write md2 column order
+run,0,0,q,`:dbw_md2/sym set `b`c`a,1,1,write md2 unsorted sym
+run,0,0,q,`:dbw_md2/px set 1 2 3f,1,1,write md2 price
+run,0,0,q,dbwrite.sort[cfg;`trade;(`:dbw_md1/;`:dbw_md2/)],1,1,sort two partition dirs in one call
+true,0,0,q,`a`b`c~exec sym from get `:dbw_md1/,1,1,first partition sorted
+true,0,0,q,`a`b`c~exec sym from get `:dbw_md2/,1,1,second partition sorted
+run,0,0,q,`:dbw_good/.d set `sym`px,1,1,write good-partition column order
+run,0,0,q,`:dbw_good/sym set `c`a`b,1,1,write good-partition unsorted sym
+run,0,0,q,`:dbw_good/px set 1 2 3f,1,1,write good-partition price
+run,0,0,q,dbwrite.sort[cfg;`trade;(`:dbw_good/;`:/)],1,1,sort a good dir alongside a bad one
+true,0,0,q,`a`b`c~exec sym from get `:dbw_good/,1,1,good partition still sorted despite the bad one failing
+
+comment,,,,,,,applyattr - applies an attribute on disk and is a no-op for non-attributes
+run,0,0,q,`:dbw_attr_tp/.d set `sym`price,1,1,write column order file
+run,0,0,q,`:dbw_attr_tp/sym set `IBM`AAPL`MSFT,1,1,write sym column
+run,0,0,q,`:dbw_attr_tp/price set 200 100 300f,1,1,write price column
+run,0,0,q,dbwrite.applyattr[`:dbw_attr_tp/;`sym;`p],1,1,apply p attr to sym
+true,0,0,q,`p=attr get `:dbw_attr_tp/sym,1,1,p attribute applied to sym on disk
+run,0,0,q,dbwrite.applyattr[`:dbw_attr_tp/;`price;`z],1,1,applyattr is a silent no-op for an invalid attribute
+true,0,0,q,`=attr get `:dbw_attr_tp/price,1,1,invalid attribute leaves the column unattributed
+
+comment,,,,,,,savedown - write an in-memory table to an hdb partition then sort it
+run,0,0,q,"sdtbl:([]time:2024.01.01D09:00 2024.01.01D08:00;sym:`IBM`AAPL;price:100 200f)",1,1,unsorted test table
+run,0,0,q,dbwrite.savedown[(::);`:dbw_hdb;2024.01.01;`trade;sdtbl],1,1,savedown with the default config (sort by time)
+run,0,0,q,"sdpath:` sv (.Q.par[`:dbw_hdb;2024.01.01;`trade];`)",1,1,resolve the partition path
+true,0,0,q,2=count get sdpath,1,1,two rows written to the partition
+true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,1,partition sorted by time after savedown
+true,0,0,q,0<count key `:dbw_hdb/sym,1,1,sym enumeration file created in the hdb root
+run,0,0,q,"dbwrite.savedown[(::);`:dbw_hdb;2024.01.02;`nosym;([]time:enlist 2024.01.02D10:00;price:enlist 50f)]",1,1,savedown a table without a sym column
+true,0,0,q,"1=count get ` sv (.Q.par[`:dbw_hdb;2024.01.02;`nosym];`)",1,1,row written for a table without sym
+run,0,0,q,"sdtbl2:([]time:2024.01.03D09:00 2024.01.03D08:00;sym:`IBM`AAPL;price:300 100f)",1,1,table with descending price for an explicit-config savedown
+run,0,0,q,"pcfg:([] tabname:enlist`trade; att:enlist`; column:enlist`price; sort:enlist 1b)",1,1,explicit config sorting trade by price
+run,0,0,q,"dbwrite.savedown[pcfg;`:dbw_hdb;2024.01.03;`trade;sdtbl2]",1,1,savedown with an explicit (non-default) config
+run,0,0,q,"sd3:get ` sv (.Q.par[`:dbw_hdb;2024.01.03;`trade];`)",1,1,read the explicit-config partition
+true,0,0,q,(asc exec price from sd3)~exec price from sd3,1,1,partition sorted by price per the explicit config
+
+comment,,,,,,,appenddown - append to a partition without sorting then sort explicitly
+run,0,0,q,"dbwrite.appenddown[`:dbw_hdb;2024.01.01;`trade;([]time:enlist 2024.01.01D07:00;sym:enlist`MSFT;price:enlist 300f)]",1,1,append an earlier-time row to an existing partition
+true,0,0,q,3=count get sdpath,1,1,three rows on disk after appenddown
+true,0,0,q,not (asc exec time from get sdpath)~exec time from get sdpath,1,1,rows not auto-sorted by appenddown
+run,0,0,q,dbwrite.sort[(::);`trade;sdpath],1,1,sort the partition explicitly after appending
+true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,1,sorted by time after the explicit sort
+fail,0,0,q,dbwrite.appenddown[`:dbw_hdb;2099.01.01;`nope;sdtbl],1,1,appenddown into a non-existent partition errors
+
+comment,,,,,,,manipulate - registered / passthrough / error fallback
+run,0,0,q,mymanip:enlist[`trade]!enlist {[x] update done:1b from x},1,1,define a save-down manipulation
+run,0,0,q,dbwrite.init[enlist[`savedownmanipulation]!enlist mymanip; enlist[`log]!enlist mylog],1,1,init wiring savedownmanipulation from config
+true,0,0,q,`done in cols dbwrite.manipulate[`trade; ([] sym:enlist`a; px:enlist 1.)],1,1,registered manipulation is applied
+true,0,0,q,not `done in cols dbwrite.manipulate[`quote; ([] sym:enlist`a)],1,1,unregistered table passes through unchanged
+run,0,0,q,"badmanip:enlist[`bad]!enlist {[x] '""boom""}",1,1,define a throwing manipulation
+run,0,0,q,dbwrite.init[enlist[`savedownmanipulation]!enlist badmanip; enlist[`log]!enlist mylog],1,1,init wiring the throwing manipulation
+true,0,0,q,([] a:enlist 1)~dbwrite.manipulate[`bad; ([] a:enlist 1)],1,1,manipulation error falls back to the unmodified table
+run,0,0,q,dbwrite.init[(::); enlist[`log]!enlist mylog],1,1,re-init with no manipulations
+
+comment,,,,,,,postreplay - exported override hook (dbwrite never calls it internally); confirm it is a callable no-op
+true,0,0,q,(::)~dbwrite.postreplay[`:hdb;2024.01.01],1,1,postreplay is callable and returns generic null
+
+comment,,,,,,,log call verification - switch to capturing logger
+run,0,0,q,dbwrite.init[(::); enlist[`log]!enlist logcap],1,1,re-init with capturing logger
+
+comment,,,,,,,log call verification - readcsv info logging on successful read
+run,0,0,q,caplog:0#caplog,1,1,reset log capture
+run,0,0,q,dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,call readcsv with capturing logger
+true,0,0,q,2=count select from caplog where fn=`info,1,1,readcsv emits exactly two info entries
+true,0,0,q,"any caplog[`msg] like ""*reading sort config*""",1,1,readcsv logs read start message
+true,0,0,q,"any caplog[`msg] like ""*read 3 sort config*""",1,1,readcsv logs the row count read
+true,0,0,q,all (select from caplog where fn=`info)[`ctx]=`dbwrite,1,1,readcsv logs under the dbwrite context symbol
+true,0,0,q,not any caplog[`fn]=`error,1,1,readcsv logs no errors on success
+
+comment,,,,,,,log call verification - readcsv error logging on file read failure
+run,0,0,q,caplog:0#caplog,1,1,reset log capture
+run,0,0,q,@[dbwrite.readcsv;`:nonexistent_file.csv;{x}],1,1,trigger file read error
+true,0,0,q,1=count select from caplog where fn=`error,1,1,readcsv logs exactly one error on file failure
+true,0,0,q,"any caplog[`msg] like ""*failed to read*""",1,1,error log message mentions failed to read
+
+comment,,,,,,,log call verification - sort info and partition-failure logging
+run,0,0,q,caplog:0#caplog,1,1,reset log capture
+run,0,0,q,dbwrite.sort[cfg;`trade;enlist`:/],1,1,sort with table-specific params against a bad dir
+true,0,0,q,"any caplog[`msg] like ""*sorting the trade table*""",1,1,sort logs sort start
+true,0,0,q,"any caplog[`msg] like ""*sort params found for: trade*""",1,1,sort logs the params lookup
+true,0,0,q,"any caplog[`msg] like ""*finished sorting the trade table*""",1,1,sort logs completion
+true,0,0,q,"any caplog[`msg] like ""*failed to sort*""",1,1,sort logs the partition sort failure
+true,0,0,q,"any caplog[`msg] like ""*unable to apply*""",1,1,sort logs the attribute application failure
+true,0,0,q,all (select from caplog where fn in `info`warn`error)[`ctx]=`dbwrite,1,1,all log entries use the dbwrite context symbol
+
+comment,,,,,,,log call verification - default fallback and skip warning
+run,0,0,q,caplog:0#caplog,1,1,reset log capture
+run,0,0,q,dbwrite.sort[cfg;`other;enlist`:/],1,1,sort falls back to the default params row
+true,0,0,q,"any caplog[`msg] like ""*using defaults*""",1,1,sort logs the default params fallback
+run,0,0,q,caplog:0#caplog,1,1,reset log capture
+run,0,0,q,dbwrite.sort[nodflcfg;`other;enlist`:/],1,1,sort with no matching params and no default
+true,0,0,q,any caplog[`fn]=`warn,1,1,sort emits a warn when no config applies
+true,0,0,q,"any caplog[`msg] like ""*skipping sort*""",1,1,warn message indicates the table is skipped
+
+comment,,,,,,,gc - runs and logs before/after
+run,0,0,q,caplog:0#caplog,1,1,reset log capture
+run,0,0,q,dbwrite.gc[],1,1,run garbage collect
+true,0,0,q,2=count select from caplog where fn=`info,1,1,gc logs two info entries (start and end)
+true,0,0,q,"any caplog[`msg] like ""*garbage collect*""",1,1,gc logs a garbage-collect message
+
+after,0,0,q,hdel`:tmp_dbw_valid.csv,1,1,remove valid csv
+after,0,0,q,hdel`:tmp_dbw_badcols.csv,1,1,remove bad cols csv
+after,0,0,q,hdel`:tmp_dbw_badatt.csv,1,1,remove bad att csv
+after,0,0,q,hdel`:tmp_dbw_3col.csv,1,1,remove 3-col csv
+after,0,0,q,hdel`:tmp_dbw_empty.csv,1,1,remove empty csv
+after,0,0,q,hdel`:tmp_dbw_nodfl.csv,1,1,remove no-default csv
+after,0,0,q,hdel`:tmp_dbw_5col.csv,1,1,remove 5-col csv
+after,0,0,q,hdel`:tmp_dbw_reorder.csv,1,1,remove reordered csv
+after,0,0,q,hdel`:tmp_dbw_nohdr.csv,1,1,remove no-header csv
+after,0,0,q,rmrf`:dbw_sort_tp,1,1,cleanup sort partition
+after,0,0,q,rmrf`:dbw_def_tp,1,1,cleanup default partition
+after,0,0,q,rmrf`:dbw_md1,1,1,cleanup md1 partition
+after,0,0,q,rmrf`:dbw_md2,1,1,cleanup md2 partition
+after,0,0,q,rmrf`:dbw_good,1,1,cleanup good partition
+after,0,0,q,rmrf`:dbw_attr_tp,1,1,cleanup attr partition
+after,0,0,q,rmrf`:dbw_hdb,1,1,cleanup savedown hdb tree

From f8cfc198897722a4ffe05476c782d950929d56b7 Mon Sep 17 00:00:00 2001
From: ascottDI <ascott@homer.aquaq.co.uk>
Date: Mon, 22 Jun 2026 13:31:12 +0100
Subject: [PATCH 09/15] updating to contain changes made by Ruairi and general
 bug fixes

---
 di/dbwrite/dbwrite.md | 32 +++++----------------------
 di/dbwrite/dbwrite.q  | 51 ++++++++++++++++---------------------------
 di/dbwrite/init.q     |  4 ++--
 di/dbwrite/test.csv   | 46 +++++++++++++++++++-------------------
 4 files changed, 49 insertions(+), 84 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 677937a3..e58a40f0 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -14,7 +14,6 @@ The sorting/attribute engine is `di.sort` baked in directly: a **config table**
 - Append rows to an existing partition with `appenddown` — enumerates syms and appends; sort separately when the partition is complete
 - Sort on-disk table partitions by configured columns using `xasc`, then apply kdb+ attributes (`p`,`s`,`g`,`u`)
 - Config supplied as an in-memory table or read from a CSV (`readcsv`); a `default` row or `(::)` provides a fallback
-- Apply a registered pre-write `manipulate` transformation per table
 - Run `.Q.gc[]` with before/after memory logging
 - Sort and attribute errors are caught-and-logged (a single partition failure does not halt the run); config and write errors are raised to the caller with a `di.dbwrite:` prefix
 
@@ -32,7 +31,7 @@ The `log` dependency must be passed to `init`. The module throws if it is absent
 log:use`di.log
 logdep:`info`warn`error!(log.info;log.warn;log.error)
 dbwrite:use`di.dbwrite
-dbwrite.init[(::); enlist[`log]!enlist logdep]
+dbwrite.init[enlist[`log]!enlist logdep]
 ```
 
 ---
@@ -61,25 +60,22 @@ default,,time,1
 
 | Function | Description |
 |---|---|
-| `init[config;deps]` | Wire injected dependencies (and optional `savedownmanipulation`); must be called first |
+| `init[deps]` | Wire injected dependencies; must be called first |
 | `readcsv[file]` | Read a config CSV and return it as a table |
 | `sort[config;tabname;dirs]` | Sort on-disk partition(s) for a table and apply attributes per config |
 | `savedown[config;dir;part;tabname;data]` | Write an in-memory table to an HDB partition, then sort it per config |
 | `appenddown[dir;part;tabname;data]` | Append rows to an existing partition (no sort) |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
-| `manipulate[t;x]` | Apply the registered pre-write manipulation for table `t` |
-| `postreplay[d;p]` | Post-EOD override hook (stub) |
 | `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
 
 ---
 
-### `init[config;deps]`
+### `init[deps]`
 
 Wire injected dependencies. Must be called before any other function.
 
 | Arg | Type | Description |
 |---|---|---|
-| `config` | dict (or `::`) | Optional. May contain `savedownmanipulation` — a `tabname!unary-function` dict applied by `manipulate`. `::` or a dict without the key means no manipulations. |
 | `deps` | dict | Must contain `` `log `` → `` `info`warn`error!(infofn;warnfn;errfn) ``. |
 
 Throws (prefixed `di.dbwrite:`) if `deps` is not a dict, `log` is missing, or the log dict lacks any required key.
@@ -94,7 +90,7 @@ Read a config CSV and **return** it as a table (does not store it) — pass the
 |---|---|---|
 | `file` | symbol/hsym | Path to the CSV; coerced with `hsym`. Errors (`di.dbwrite:`) if not a symbol. |
 
-The header is validated as it is read; a wrong shape errors clearly. Attribute-value validation happens later in `sort`.
+The CSV is parsed field-by-field and validated as it is read — it does **not** silently pad, truncate, or coerce malformed rows. A clear `di.dbwrite:` error is raised if: the header is not exactly `tabname,att,column,sort` (any order); any data row does not have exactly four fields; or any `sort` value is not `0` or `1`. Column order is normalised to canonical. Attribute-value validation happens later in `sort`.
 
 ```q
 config:dbwrite.readcsv `:config/sort.csv
@@ -172,22 +168,6 @@ dbwrite.applyattr[`:/hdb/2024.01.02/trade; `sym; `p]
 
 ---
 
-### `manipulate[t;x]`
-
-Apply the registered pre-write manipulation for table `t` to `x`, returning the result. Unchanged if none is registered; if the manipulation throws, the failure is logged and `x` is returned unchanged.
-
-```q
-data:dbwrite.manipulate[`trade; data]
-```
-
----
-
-### `postreplay[d;p]`
-
-Post-EOD hook, called after all tables are written and sorted. A stub by default (`{[d;p]}`) — override at the call site to add custom post-replay logic.
-
----
-
 ### `gc[]`
 
 Run `.Q.gc[]`, logging memory stats before and after.
@@ -201,12 +181,12 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-The suite injects mock loggers: a no-op logger, and a capturing logger that records `(level;ctx;msg)` so log behaviour can be asserted. On-disk behaviour (sort, attributes, `savedown`/`appenddown`) is exercised against real splayed partitions and cleaned up afterwards. It covers: dependency-injection validation; `readcsv` returns / column-order independence / header-validation failures; `sort` validation / edge cases / resolution / on-disk results / multi-dir / non-fatal partition failure; `savedown` write+sort (default and explicit config, and a table without `sym`); `appenddown` append-without-sort then explicit sort, and the non-existent-partition error; `applyattr`; `manipulate`; `postreplay`; `gc`; and the logging contract.
+The suite injects mock loggers: a no-op logger, and a capturing logger that records `(level;ctx;msg)` so log behaviour can be asserted. On-disk behaviour (sort, attributes, `savedown`/`appenddown`) is exercised against real splayed partitions and cleaned up afterwards. It covers: dependency-injection validation; `readcsv` returns / column-order independence / header-validation failures; `sort` validation / edge cases / resolution / on-disk results / multi-dir / non-fatal partition failure; `savedown` write+sort (default and explicit config, and a table without `sym`); `appenddown` append-without-sort then explicit sort, and the non-existent-partition error; `applyattr`; `gc`; and the logging contract.
 
 ---
 
 ## Exported symbols
 
 ```q
-export:([init;readcsv;sort;applyattr;savedown;appenddown;manipulate;postreplay;gc])
+export:([init;readcsv;sort;applyattr;savedown;appenddown;gc])
 ```
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 35c4de4d..c0c2b3fe 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -1,4 +1,4 @@
-/ dbwrite - sort, attribute application, save-down manipulation, and GC utilities for on-disk data
+/ dbwrite - write, sort, and attribute utilities for on-disk data
 / used by processes that persist data to disk (rdb, wdb, tickerlogreplay)
 
 / attributes that may legitimately appear in a config (empty leaves a column unattributed)
@@ -7,9 +7,8 @@ validatts:``p`s`g`u;
 / built-in fallback config - sort every table by time ascending when no config is supplied
 defaultparams:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:enlist 1b);
 
-init:{[config;deps]
-  / wire injected deps and optional module config
-  / config: dict with optional `savedownmanipulation (tabname!unary fn applied pre-write)
+init:{[deps]
+  / wire the injected log dependency
   / deps: `log!(logdict) where logdict is `info`warn`error!(infofn;warnfn;errfn) - required
   if[99h<>type deps;
     '"di.dbwrite: deps must be a dict with a `log key; see di.log for a default logger"];
@@ -20,10 +19,6 @@ init:{[config;deps]
   if[not all (`info`warn`error) in key deps`log;
     '"di.dbwrite: log dict must have `info`warn`error keys; got: ",(", " sv string key deps`log)];
   .z.m.log:deps`log;
-  .z.m.savedownmanipulation:()!();
-  if[99h=type config;
-    if[`savedownmanipulation in key config;
-      .z.m.savedownmanipulation:config`savedownmanipulation]];
   };
 
 readcsv:{[file]
@@ -52,17 +47,22 @@ readerr:{[file;e]
   'm;
   };
 
-/ internal - validate the header and parse csv lines into a config table
+/ internal - validate the header and data rows, then parse csv lines into a config table
 parsecsv:{[lines]
-  / map types by column name so the csv column order does not matter; reject any other shape
-  / outside the readfile i/o trap so a bad header surfaces as a clear di.dbwrite: error
+  / parse field-by-field rather than via 0:, which silently pads/truncates/coerces malformed rows
+  / map columns by header name so order does not matter; runs outside the readfile i/o trap
   if[0=count lines;
     '"di.dbwrite: csv has no header row"];
   hdr:`$"," vs first lines;
   if[not (asc distinct hdr)~`att`column`sort`tabname;
     '"di.dbwrite: csv header must be exactly tabname,att,column,sort; got: ",", " sv string hdr];
-  types:{$[x=`sort;"B";"S"]} each hdr;
-  :`tabname`att`column`sort#(types;enlist",") 0: lines;
+  rows:"," vs/: 1 _ lines;
+  if[count bad:where (count each rows)<>count hdr;
+    '"di.dbwrite: csv data row(s) ",(", " sv string 1+bad)," must have ",(string count hdr)," fields"];
+  c:hdr!$[count rows; flip rows; (count hdr)#enlist ()];
+  if[not all (c`sort) in enlist each "01";
+    '"di.dbwrite: the sort column must contain only 0 or 1"];
+  :([] tabname:`$c`tabname; att:`$c`att; column:`$c`column; sort:"B"$c`sort);
   };
 
 / internal - validate a sort-config table, signalling a clear error if it is malformed
@@ -77,6 +77,10 @@ checkconfig:{[t]
   missingcols:(`tabname`att`column`sort) where not (`tabname`att`column`sort) in c;
   if[count missingcols;
     '"di.dbwrite: missing required config column(s): ",", " sv string missingcols];
+  if[any null t`tabname;
+    '"di.dbwrite: config tabname must not be null"];
+  if[any null t`column;
+    '"di.dbwrite: config column must not be null"];
   if[not 1h=type t`sort;
     '"di.dbwrite: the sort column must be boolean"];
   badatts:at where not (at:distinct t`att) in validatts;
@@ -135,11 +139,10 @@ sortcolumns:{[dloc;sortcols]
 
 / internal - sort columns and apply attributes for a single on-disk partition directory
 sortdir:{[sp;dloc]
-  / sort by the columns flagged sort=1b, then attribute the columns that request one
+  / sort by the columns flagged sort=1b, then hand every row to applyattr (it skips non-attributes)
   sortcols:exec column from sp where sort, not null column;
   if[count sortcols; sortcolumns[dloc;sortcols]];
-  attrcols:select column,att from sp where att in `p`s`g`u;
-  if[count attrcols; applyattr[dloc;;]'[attrcols`column;attrcols`att]];
+  applyattr[dloc;;]'[sp`column;sp`att];
   };
 
 / internal - log an attribute application failure without rethrowing
@@ -184,22 +187,6 @@ appenddown:{[dir;part;tabname;data]
   .z.m.log[`info][`dbwrite;"finished appending ",string tabname];
   };
 
-manipulate:{[t;x]
-  / apply registered pre-write manipulation to table x of type t; unchanged if none registered
-  if[not t in key .z.m.savedownmanipulation; :x];
-  :@[.z.m.savedownmanipulation[t]; x; maniperr[x]];
-  };
-
-/ internal - log a failed save-down manipulation and fall back to the unmodified table
-maniperr:{[x;e]
-  / called as the manipulate error handler; the unmodified table is the safe fallback
-  .z.m.log[`error][`dbwrite;"save-down manipulation failed: ",e];
-  :x;
-  };
-
-/ post-EOD hook - called after all tables written and sorted; override at the call site
-postreplay:{[d;p]};
-
 / internal - render a memory-usage dict as a "key=val MB; ..." string
 fmtmem:{[m]
   / m: dict of MB values keyed by .Q.w field name
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index c402f0a4..53f5c44e 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -1,6 +1,6 @@
-/ dbwrite module - sort, attribute application, save-down manipulation, and GC utilities
+/ dbwrite module - write, sort, and attribute utilities for on-disk data
 / used by processes that persist data to disk (rdb, wdb, tickerlogreplay)
 
 \l ::dbwrite.q
 
-export:([init;readcsv;sort;applyattr;savedown;appenddown;manipulate;postreplay;gc])
\ No newline at end of file
+export:([init;readcsv;sort;applyattr;savedown;appenddown;gc])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index 32830167..641f866c 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -13,20 +13,23 @@ before,0,0,q,"`:tmp_dbw_nodfl.csv 0: (""tabname,att,column,sort"";""trade,p,sym,
 before,0,0,q,"`:tmp_dbw_5col.csv 0: (""tabname,att,column,sort,extra"";""trade,p,sym,1,foo"")",1,1,write csv with an extra column
 before,0,0,q,"`:tmp_dbw_reorder.csv 0: (""tabname,att,sort,column"";""trade,p,1,sym"";""trade,,0,time"";""default,p,1,sym"")",1,1,write valid csv with columns reordered
 before,0,0,q,`:tmp_dbw_nohdr.csv 0: (),1,1,write a zero-line csv (no header)
-before,0,0,q,dbwrite.init[(::); enlist[`log]!enlist mylog],1,1,init with no-op logger
+before,0,0,q,"`:tmp_dbw_short.csv 0: (""tabname,att,column,sort"";""trade,p,sym"")",1,1,write csv with a too-short data row
+before,0,0,q,"`:tmp_dbw_long.csv 0: (""tabname,att,column,sort"";""trade,p,sym,1,extra"")",1,1,write csv with a too-long data row
+before,0,0,q,"`:tmp_dbw_badbool.csv 0: (""tabname,att,column,sort"";""trade,p,sym,x"")",1,1,write csv with a non-boolean sort value
+before,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,init with no-op logger
 before,0,0,q,cfg:([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,reusable valid config table (hand-built)
 before,0,0,q,nodflcfg:dbwrite.readcsv `:tmp_dbw_nodfl.csv,1,1,reusable config with a trade row but no default
 
 comment,,,,,,,init - dependency injection validation (type errors / empty / null)
-fail,0,0,q,dbwrite.init[(::);(::)],1,1,init rejects :: as deps
-fail,0,0,q,dbwrite.init[(::);42],1,1,init rejects a non-dict deps value
-fail,0,0,q,dbwrite.init[(::);()!()],1,1,init rejects empty dict
-fail,0,0,q,dbwrite.init[(::);enlist[`other]!enlist mylog],1,1,init rejects dict missing log key
-fail,0,0,q,dbwrite.init[(::);enlist[`log]!enlist(::)],1,1,init rejects null log dep
-fail,0,0,q,dbwrite.init[(::);enlist[`log]!enlist 42],1,1,init rejects non-dict log value
-fail,0,0,q,dbwrite.init[(::);enlist[`log]!enlist(`info`warn!(mylog`info;mylog`warn))],1,1,init rejects log dict missing required key
-true,0,0,q,"""di.dbwrite:""~11#@[{dbwrite.init[(::);(::)]};(::);{x}]",1,1,error message is prefixed di.dbwrite
-run,0,0,q,dbwrite.init[(::); enlist[`log]!enlist mylog],1,1,re-init with valid log dep
+fail,0,0,q,dbwrite.init[(::)],1,1,init rejects :: as deps
+fail,0,0,q,dbwrite.init[42],1,1,init rejects a non-dict deps value
+fail,0,0,q,dbwrite.init[()!()],1,1,init rejects empty dict
+fail,0,0,q,dbwrite.init[enlist[`other]!enlist mylog],1,1,init rejects dict missing log key
+fail,0,0,q,dbwrite.init[enlist[`log]!enlist(::)],1,1,init rejects null log dep
+fail,0,0,q,dbwrite.init[enlist[`log]!enlist 42],1,1,init rejects non-dict log value
+fail,0,0,q,dbwrite.init[enlist[`log]!enlist(`info`warn!(mylog`info;mylog`warn))],1,1,init rejects log dict missing required key
+true,0,0,q,"""di.dbwrite:""~11#@[{dbwrite.init[(::)]};(::);{x}]",1,1,error message is prefixed di.dbwrite
+run,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,re-init with valid log dep
 
 comment,,,,,,,readcsv - returns a config table from a csv
 true,0,0,q,3=count dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv returns a 3-row config table
@@ -43,6 +46,9 @@ fail,0,0,q,dbwrite.readcsv `:tmp_dbw_badcols.csv,1,1,readcsv rejects a csv with
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_3col.csv,1,1,readcsv rejects a csv missing a required column
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_5col.csv,1,1,readcsv rejects a csv with an extra column
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_nohdr.csv,1,1,readcsv rejects a csv with no header row
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_short.csv,1,1,readcsv rejects a data row with too few fields
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_long.csv,1,1,readcsv rejects a data row with too many fields
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_badbool.csv,1,1,readcsv rejects a non-boolean sort value
 
 comment,,,,,,,sort - happy path via both a hand-built table and a csv
 run,0,0,q,dbwrite.sort[cfg;`trade;()],1,1,sort accepts a hand-built config table
@@ -55,6 +61,8 @@ fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`sy
 fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`z; column:enlist`sym; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects unknown attribute values
 fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`sym; sort:enlist 1);`trade;enlist`:/],1,1,sort rejects a non-boolean sort column
 fail,0,0,q,dbwrite.sort[dbwrite.readcsv `:tmp_dbw_badatt.csv;`trade;enlist`:/],1,1,sort rejects csv config with invalid attributes
+fail,0,0,q,dbwrite.sort[([] tabname:enlist`; att:enlist`p; column:enlist`sym; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects a null tabname in config
+fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects a null column in config
 
 comment,,,,,,,sort - edge cases (empty config / null att / empty dirs / atom dir / type error)
 true,0,0,q,()~dbwrite.sort[([] tabname:`symbol$();att:`symbol$();column:`symbol$();sort:`boolean$());`trade;enlist`:/],1,1,empty config yields no work and returns ()
@@ -133,21 +141,8 @@ run,0,0,q,dbwrite.sort[(::);`trade;sdpath],1,1,sort the partition explicitly aft
 true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,1,sorted by time after the explicit sort
 fail,0,0,q,dbwrite.appenddown[`:dbw_hdb;2099.01.01;`nope;sdtbl],1,1,appenddown into a non-existent partition errors
 
-comment,,,,,,,manipulate - registered / passthrough / error fallback
-run,0,0,q,mymanip:enlist[`trade]!enlist {[x] update done:1b from x},1,1,define a save-down manipulation
-run,0,0,q,dbwrite.init[enlist[`savedownmanipulation]!enlist mymanip; enlist[`log]!enlist mylog],1,1,init wiring savedownmanipulation from config
-true,0,0,q,`done in cols dbwrite.manipulate[`trade; ([] sym:enlist`a; px:enlist 1.)],1,1,registered manipulation is applied
-true,0,0,q,not `done in cols dbwrite.manipulate[`quote; ([] sym:enlist`a)],1,1,unregistered table passes through unchanged
-run,0,0,q,"badmanip:enlist[`bad]!enlist {[x] '""boom""}",1,1,define a throwing manipulation
-run,0,0,q,dbwrite.init[enlist[`savedownmanipulation]!enlist badmanip; enlist[`log]!enlist mylog],1,1,init wiring the throwing manipulation
-true,0,0,q,([] a:enlist 1)~dbwrite.manipulate[`bad; ([] a:enlist 1)],1,1,manipulation error falls back to the unmodified table
-run,0,0,q,dbwrite.init[(::); enlist[`log]!enlist mylog],1,1,re-init with no manipulations
-
-comment,,,,,,,postreplay - exported override hook (dbwrite never calls it internally); confirm it is a callable no-op
-true,0,0,q,(::)~dbwrite.postreplay[`:hdb;2024.01.01],1,1,postreplay is callable and returns generic null
-
 comment,,,,,,,log call verification - switch to capturing logger
-run,0,0,q,dbwrite.init[(::); enlist[`log]!enlist logcap],1,1,re-init with capturing logger
+run,0,0,q,dbwrite.init[enlist[`log]!enlist logcap],1,1,re-init with capturing logger
 
 comment,,,,,,,log call verification - readcsv info logging on successful read
 run,0,0,q,caplog:0#caplog,1,1,reset log capture
@@ -198,6 +193,9 @@ after,0,0,q,hdel`:tmp_dbw_nodfl.csv,1,1,remove no-default csv
 after,0,0,q,hdel`:tmp_dbw_5col.csv,1,1,remove 5-col csv
 after,0,0,q,hdel`:tmp_dbw_reorder.csv,1,1,remove reordered csv
 after,0,0,q,hdel`:tmp_dbw_nohdr.csv,1,1,remove no-header csv
+after,0,0,q,hdel`:tmp_dbw_short.csv,1,1,remove short-row csv
+after,0,0,q,hdel`:tmp_dbw_long.csv,1,1,remove long-row csv
+after,0,0,q,hdel`:tmp_dbw_badbool.csv,1,1,remove bad-bool csv
 after,0,0,q,rmrf`:dbw_sort_tp,1,1,cleanup sort partition
 after,0,0,q,rmrf`:dbw_def_tp,1,1,cleanup default partition
 after,0,0,q,rmrf`:dbw_md1,1,1,cleanup md1 partition

From c631d34cabb25608e132bf0b91fbbcf40a70325a Mon Sep 17 00:00:00 2001
From: ascottDI <ascott@homer.aquaq.co.uk>
Date: Mon, 22 Jun 2026 15:27:38 +0100
Subject: [PATCH 10/15] expanding on tests to remove stubbed tests, fixed a bug
 with applying p attribute before sorting resulting in a ufail

---
 di/dbwrite/dbwrite.md |  2 +-
 di/dbwrite/dbwrite.q  |  6 +++---
 di/dbwrite/test.csv   | 14 ++++++++++++++
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index e58a40f0..974862ec 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -118,7 +118,7 @@ dbwrite.sort[config; `trade; `:/hdb/2024.01.02/trade`:/hdb/2024.01.03/trade]
 
 ### `savedown[config;dir;part;tabname;data]`
 
-Write an in-memory table to a date-partitioned HDB partition, then sort it per `config`. Enumerates symbol columns against the HDB sym file and applies `p#` to `sym` (if present) before writing.
+Write an in-memory table to a date-partitioned HDB partition, then sort it per `config`. Enumerates symbol columns against the HDB sym file before writing. Sorting **and** attributes are driven entirely by `config` and applied by `sort` *after* the write — so an attribute like `p#` is only applied once its column is grouped (e.g. a config that sorts by `sym` and parts `sym`), never to unsorted data.
 
 | Parameter | Type | Description |
 |---|---|---|
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index c0c2b3fe..5621965e 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -166,11 +166,11 @@ savedown:{[config;dir;part;tabname;data]
   / write an in-memory table to a date-partitioned hdb partition, then sort it per config
   / config: a sort-config table, or (::) for the built-in default; dir: hdb root (hsym)
   / part: partition value (date/month/int); tabname: symbol; data: in-memory table
-  / enumerates syms against the hdb sym file and applies p# to sym before writing
+  / enumerates syms against the hdb sym file; sorting and attributes are driven by config
+  / (attributes such as p# are applied post-sort by sort, where the data is correctly grouped)
   .z.m.log[`info][`dbwrite;"saving ",string[tabname]," partition ",string[part]," to ",string dir];
   path:` sv (.Q.par[dir;part;tabname];`);
-  data:.Q.en[dir;data];
-  path set $[`sym in cols data; @[data;`sym;{`p#x}]; data];
+  path set .Q.en[dir;data];
   sort[config;tabname;path];
   .z.m.log[`info][`dbwrite;"finished saving ",string tabname];
   };
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index 641f866c..438e119f 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -141,6 +141,19 @@ run,0,0,q,dbwrite.sort[(::);`trade;sdpath],1,1,sort the partition explicitly aft
 true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,1,sorted by time after the explicit sort
 fail,0,0,q,dbwrite.appenddown[`:dbw_hdb;2099.01.01;`nope;sdtbl],1,1,appenddown into a non-existent partition errors
 
+comment,,,,,,,integration - savedown realistic (repeated/unsorted sym) data builds a valid queryable partition
+run,0,0,q,symcfg:([] tabname:enlist`trade; att:enlist`p; column:enlist`sym; sort:enlist 1b),1,1,config that sorts by sym and applies p#
+run,0,0,q,"rd:([] time:2024.01.01D10 2024.01.01D09 2024.01.01D11; sym:`IBM`AAPL`IBM; px:1 2 3f)",1,1,realistic unsorted data with a repeated sym
+run,0,0,q,dbwrite.savedown[symcfg;`:dbw_ihdb;2024.01.01;`trade;rd],1,1,savedown repeated-sym data (must not crash applying p#)
+run,0,0,q,"ip:get ` sv (.Q.par[`:dbw_ihdb;2024.01.01;`trade];`)",1,1,read the written partition back
+true,0,0,q,`AAPL`IBM`IBM~value ip`sym,1,1,partition is sorted (grouped) by sym on disk (sym is enumerated - resolve with value)
+true,0,0,q,`p=attr ip`sym,1,1,p# applied to sym post-sort (valid because the data is grouped)
+true,0,0,q,2=count select from ip where sym=`IBM,1,1,p# lookup is correct - two IBM rows
+true,0,0,q,1=count select from ip where sym=`AAPL,1,1,p# lookup is correct - one AAPL row
+run,0,0,q,dbwrite.savedown[(::);`:dbw_ihdb;2024.01.02;`trade;rd],1,1,savedown repeated-sym data with the default (time) config must not crash
+true,0,0,q,(asc x)~x:exec time from get ` sv (.Q.par[`:dbw_ihdb;2024.01.02;`trade];`),1,1,default-config partition sorted by time
+true,0,0,q,`=attr (get ` sv (.Q.par[`:dbw_ihdb;2024.01.02;`trade];`))`sym,1,1,default-config leaves sym unattributed (not falsely parted)
+
 comment,,,,,,,log call verification - switch to capturing logger
 run,0,0,q,dbwrite.init[enlist[`log]!enlist logcap],1,1,re-init with capturing logger
 
@@ -203,3 +216,4 @@ after,0,0,q,rmrf`:dbw_md2,1,1,cleanup md2 partition
 after,0,0,q,rmrf`:dbw_good,1,1,cleanup good partition
 after,0,0,q,rmrf`:dbw_attr_tp,1,1,cleanup attr partition
 after,0,0,q,rmrf`:dbw_hdb,1,1,cleanup savedown hdb tree
+after,0,0,q,rmrf`:dbw_ihdb,1,1,cleanup integration hdb tree

From 919378266f843c0b4b1c6f5a7e30d03f67ea3ee0 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Tue, 23 Jun 2026 10:20:34 +0100
Subject: [PATCH 11/15] store sort config in module state; add setconfig; gc
 after savedown

- readcsv now stores parsed config in .z.m.sortconfig instead of returning it
- add setconfig to allow consumers to provide an in-memory config table directly
- sort and savedown drop their config parameter; both read .z.m.sortconfig, falling back to defaultparams
- gc is called at the end of savedown; removed from exports as it is not dbwrite-specific
- init initialises .z.m.sortconfig to (::)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.q | 33 ++++++++++++++++++++-------------
 di/dbwrite/init.q    |  2 +-
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 5621965e..5e207cff 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -19,17 +19,25 @@ init:{[deps]
   if[not all (`info`warn`error) in key deps`log;
     '"di.dbwrite: log dict must have `info`warn`error keys; got: ",(", " sv string key deps`log)];
   .z.m.log:deps`log;
+  .z.m.sortconfig:(::);
   };
 
 readcsv:{[file]
-  / read a sort-config csv and return it as a table (does not store it); pass the result to sort
+  / read a sort-config csv and store it in .z.m.sortconfig; used by sort and savedown
   / the csv must have the columns tabname,att,column,sort (in any order)
   if[not -11h=type file;
     '"di.dbwrite: readcsv file must be a symbol, got type ",string type file];
   file:hsym file;
   t:parsecsv @[readfile; file; readerr[file]];
   .z.m.log[`info][`dbwrite;"read ",(string count t)," sort config row(s) from ",string file];
-  :t;
+  .z.m.sortconfig:t;
+  };
+
+setconfig:{[t]
+  / set .z.m.sortconfig from an in-memory table; alternative to readcsv when config is built in-session
+  / t must be a table with columns tabname,att,column,sort
+  checkconfig t;
+  .z.m.sortconfig:t;
   };
 
 / internal - protected file read; only the i/o so a genuine read failure gets the readerr message
@@ -88,11 +96,11 @@ checkconfig:{[t]
     '"di.dbwrite: unrecognised attribute(s) in att column: ",", " sv string badatts];
   };
 
-sort:{[config;tabname;dirs]
-  / sort and apply attributes to the on-disk partition dirs for one table using config
-  / config: a sort-config table (build it directly or via readcsv); (::) uses the default config
+sort:{[tabname;dirs]
+  / sort and apply attributes to the on-disk partition dirs for one table using .z.m.sortconfig
+  / falls back to defaultparams if readcsv has not been called
   / tabname: symbol table name; dirs: hsym or list of hsyms (partition directories)
-  config:$[(::)~config; defaultparams; config];
+  config:$[(::)~.z.m.sortconfig; defaultparams; .z.m.sortconfig];
   checkconfig config;
   if[not -11h=type tabname;
     '"di.dbwrite: tabname must be a symbol, got type ",string type tabname];
@@ -162,17 +170,16 @@ applyattr:{[dloc;colname;att]
     attrerr[dloc;colname;att]];
   };
 
-savedown:{[config;dir;part;tabname;data]
-  / write an in-memory table to a date-partitioned hdb partition, then sort it per config
-  / config: a sort-config table, or (::) for the built-in default; dir: hdb root (hsym)
-  / part: partition value (date/month/int); tabname: symbol; data: in-memory table
-  / enumerates syms against the hdb sym file; sorting and attributes are driven by config
-  / (attributes such as p# are applied post-sort by sort, where the data is correctly grouped)
+savedown:{[dir;part;tabname;data]
+  / write an in-memory table to a date-partitioned hdb partition, then sort it per .z.m.sortconfig
+  / dir: hdb root (hsym); part: partition value (date/month/int); tabname: symbol; data: in-memory table
+  / enumerates syms against the hdb sym file; sorting and attributes are driven by .z.m.sortconfig
   .z.m.log[`info][`dbwrite;"saving ",string[tabname]," partition ",string[part]," to ",string dir];
   path:` sv (.Q.par[dir;part;tabname];`);
   path set .Q.en[dir;data];
-  sort[config;tabname;path];
+  sort[tabname;path];
   .z.m.log[`info][`dbwrite;"finished saving ",string tabname];
+  gc[];
   };
 
 appenddown:{[dir;part;tabname;data]
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index 53f5c44e..d9347d4c 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -3,4 +3,4 @@
 
 \l ::dbwrite.q
 
-export:([init;readcsv;sort;applyattr;savedown;appenddown;gc])
\ No newline at end of file
+export:([init;readcsv;setconfig;sort;applyattr;savedown;appenddown])
\ No newline at end of file

From fc019edc2b4ad2a7b42f14dea784629d0ee387d6 Mon Sep 17 00:00:00 2001
From: ascottDI <ascott@homer.aquaq.co.uk>
Date: Tue, 23 Jun 2026 10:31:20 +0100
Subject: [PATCH 12/15] Changing the logging from di logging to kx logging

---
 di/dbwrite/dbwrite.md | 21 +++++++--------------
 di/dbwrite/dbwrite.q  | 37 +++++++++++++++++++------------------
 di/dbwrite/init.q     |  2 +-
 di/dbwrite/test.csv   | 21 +++++++++++----------
 4 files changed, 38 insertions(+), 43 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 974862ec..04d76f9d 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -14,7 +14,6 @@ The sorting/attribute engine is `di.sort` baked in directly: a **config table**
 - Append rows to an existing partition with `appenddown` — enumerates syms and appends; sort separately when the partition is complete
 - Sort on-disk table partitions by configured columns using `xasc`, then apply kdb+ attributes (`p`,`s`,`g`,`u`)
 - Config supplied as an in-memory table or read from a CSV (`readcsv`); a `default` row or `(::)` provides a fallback
-- Run `.Q.gc[]` with before/after memory logging
 - Sort and attribute errors are caught-and-logged (a single partition failure does not halt the run); config and write errors are raised to the caller with a `di.dbwrite:` prefix
 
 ---
@@ -23,13 +22,14 @@ The sorting/attribute engine is `di.sort` baked in directly: a **config table**
 
 | Dependency | Key | Required | Description |
 |---|---|---|---|
-| logger | `` `log `` | yes | Functions `info`,`warn`,`error` — each `{[ctx;msg] ...}` |
+| logger | `` `log `` | yes | Functions `info`,`warn`,`error` — each monadic `{[msg] ...}` |
 
-The `log` dependency must be passed to `init`. The module throws if it is absent, `(::)`, or missing any of the three keys. `di.log` satisfies the contract:
+The `log` dependency must be passed to `init`. The module throws if it is absent, `(::)`, or missing any of the three keys. The functions are monadic message loggers (`{[msg] ...}`); a [`kx.log`](https://github.com/KxSystems/logging) instance satisfies the contract. The module folds its own context into each message as a `dbwrite:` prefix, so no per-call context argument is needed:
 
 ```q
-log:use`di.log
-logdep:`info`warn`error!(log.info;log.warn;log.error)
+logger:use`kx.log
+loginst:logger.createLog[]
+logdep:`info`warn`error!(loginst`info;loginst`warn;loginst`error)
 dbwrite:use`di.dbwrite
 dbwrite.init[enlist[`log]!enlist logdep]
 ```
@@ -66,7 +66,6 @@ default,,time,1
 | `savedown[config;dir;part;tabname;data]` | Write an in-memory table to an HDB partition, then sort it per config |
 | `appenddown[dir;part;tabname;data]` | Append rows to an existing partition (no sort) |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
-| `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
 
 ---
 
@@ -168,12 +167,6 @@ dbwrite.applyattr[`:/hdb/2024.01.02/trade; `sym; `p]
 
 ---
 
-### `gc[]`
-
-Run `.Q.gc[]`, logging memory stats before and after.
-
----
-
 ## Running tests
 
 ```q
@@ -181,12 +174,12 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-The suite injects mock loggers: a no-op logger, and a capturing logger that records `(level;ctx;msg)` so log behaviour can be asserted. On-disk behaviour (sort, attributes, `savedown`/`appenddown`) is exercised against real splayed partitions and cleaned up afterwards. It covers: dependency-injection validation; `readcsv` returns / column-order independence / header-validation failures; `sort` validation / edge cases / resolution / on-disk results / multi-dir / non-fatal partition failure; `savedown` write+sort (default and explicit config, and a table without `sym`); `appenddown` append-without-sort then explicit sort, and the non-existent-partition error; `applyattr`; `gc`; and the logging contract.
+The suite injects monadic mock loggers (`{[msg] ...}`): a no-op logger, and a capturing logger that records `(level;msg)` so log behaviour can be asserted. It also wires a real `kx.log` instance through `init` to confirm the module works end-to-end against the system logger. On-disk behaviour (sort, attributes, `savedown`/`appenddown`) is exercised against real splayed partitions and cleaned up afterwards. It covers: dependency-injection validation; `readcsv` returns / column-order independence / header-validation failures; `sort` validation / edge cases / resolution / on-disk results / multi-dir / non-fatal partition failure; `savedown` write+sort (default and explicit config, and a table without `sym`); `appenddown` append-without-sort then explicit sort, and the non-existent-partition error; `applyattr`; the `dbwrite:`-prefixed logging contract; and the real `kx.log` integration.
 
 ---
 
 ## Exported symbols
 
 ```q
-export:([init;readcsv;sort;applyattr;savedown;appenddown;gc])
+export:([init;readcsv;sort;applyattr;savedown;appenddown])
 ```
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 5621965e..0e0e6738 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -10,8 +10,9 @@ defaultparams:([] tabname:enlist`default; att:enlist`; column:enlist`time; sort:
 init:{[deps]
   / wire the injected log dependency
   / deps: `log!(logdict) where logdict is `info`warn`error!(infofn;warnfn;errfn) - required
+  / the functions are monadic {[msg]} loggers; a kx.log instance satisfies this (use`kx.log;createLog[])
   if[99h<>type deps;
-    '"di.dbwrite: deps must be a dict with a `log key; see di.log for a default logger"];
+    '"di.dbwrite: deps must be a dict with a `log key; see kx.log for a logger"];
   if[not `log in key deps;
     '"di.dbwrite: log dependency is required; pass `info`warn`error functions keyed on `log"];
   if[99h<>type deps`log;
@@ -28,14 +29,14 @@ readcsv:{[file]
     '"di.dbwrite: readcsv file must be a symbol, got type ",string type file];
   file:hsym file;
   t:parsecsv @[readfile; file; readerr[file]];
-  .z.m.log[`info][`dbwrite;"read ",(string count t)," sort config row(s) from ",string file];
+  .z.m.log[`info]["dbwrite: read ",(string count t)," sort config row(s) from ",string file];
   :t;
   };
 
 / internal - protected file read; only the i/o so a genuine read failure gets the readerr message
 readfile:{[file]
   / returns the raw csv lines; header validation and parsing happen in parsecsv
-  .z.m.log[`info][`dbwrite;"reading sort config from ",string file];
+  .z.m.log[`info]["dbwrite: reading sort config from ",string file];
   :read0 file;
   };
 
@@ -43,7 +44,7 @@ readfile:{[file]
 readerr:{[file;e]
   / build the message once, surface it under the dbwrite context, then rethrow it to the caller
   m:"failed to read ",string[file],": ",e;
-  .z.m.log[`error][`dbwrite;m];
+  .z.m.log[`error]["dbwrite: ",m];
   'm;
   };
 
@@ -97,17 +98,17 @@ sort:{[config;tabname;dirs]
   if[not -11h=type tabname;
     '"di.dbwrite: tabname must be a symbol, got type ",string type tabname];
   st:string tabname;
-  .z.m.log[`info][`dbwrite;"sorting the ",st," table"];
+  .z.m.log[`info]["dbwrite: sorting the ",st," table"];
   sp:getsortparams[config;tabname;st];
   if[not count sp; :()];
   sortdir[sp] each distinct (),dirs;
-  .z.m.log[`info][`dbwrite;"finished sorting the ",st," table"];
+  .z.m.log[`info]["dbwrite: finished sorting the ",st," table"];
   };
 
 / internal - log a sort message then return the resolved rows
 logreturn:{[lvl;msg;rows]
   / keeps each branch body in getsortparams to a single statement
-  .z.m.log[lvl][`dbwrite;msg];
+  .z.m.log[lvl]["dbwrite: ",msg];
   :rows;
   };
 
@@ -125,14 +126,14 @@ getsortparams:{[config;tab;st]
 / internal - log a sort failure without rethrowing so remaining partitions still run
 sorterr:{[sc;dl;e]
   / a single partition failure should not halt the whole run
-  .z.m.log[`error][`dbwrite;"failed to sort ",string[dl]," by ",(", " sv string sc),": ",e];
+  .z.m.log[`error]["dbwrite: failed to sort ",string[dl]," by ",(", " sv string sc),": ",e];
   :();
   };
 
 / internal - sort one partition directory by the given columns
 sortcolumns:{[dloc;sortcols]
   / split out of sortdir so the conditional body there stays a single statement
-  .z.m.log[`info][`dbwrite;"sorting ",string[dloc]," by: ",", " sv string sortcols];
+  .z.m.log[`info]["dbwrite: sorting ",string[dloc]," by: ",", " sv string sortcols];
   .[xasc;(sortcols;dloc);
     sorterr[sortcols;dloc]];
   };
@@ -148,7 +149,7 @@ sortdir:{[sp;dloc]
 / internal - log an attribute application failure without rethrowing
 attrerr:{[dl;cn;at;e]
   / logs failure and continues so other columns and partitions still get processed
-  .z.m.log[`error][`dbwrite;"unable to apply ",string[at]," attr to ",string[cn]," in ",string[dl],": ",e];
+  .z.m.log[`error]["dbwrite: unable to apply ",string[at]," attr to ",string[cn]," in ",string[dl],": ",e];
   :();
   };
 
@@ -156,7 +157,7 @@ applyattr:{[dloc;colname;att]
   / apply a single kdb+ attribute to an on-disk column; logs and swallows errors so a run continues
   / skip anything that is not a real attribute - covers the empty none-sentinel and any bad value
   if[not att in `p`s`g`u; :()];
-  .z.m.log[`info][`dbwrite;"applying ",string[att]," attr to ",string[colname]," in ",string dloc];
+  .z.m.log[`info]["dbwrite: applying ",string[att]," attr to ",string[colname]," in ",string dloc];
   .[{@[x;y;z#]};
     (dloc;colname;att);
     attrerr[dloc;colname;att]];
@@ -168,23 +169,23 @@ savedown:{[config;dir;part;tabname;data]
   / part: partition value (date/month/int); tabname: symbol; data: in-memory table
   / enumerates syms against the hdb sym file; sorting and attributes are driven by config
   / (attributes such as p# are applied post-sort by sort, where the data is correctly grouped)
-  .z.m.log[`info][`dbwrite;"saving ",string[tabname]," partition ",string[part]," to ",string dir];
+  .z.m.log[`info]["dbwrite: saving ",string[tabname]," partition ",string[part]," to ",string dir];
   path:` sv (.Q.par[dir;part;tabname];`);
   path set .Q.en[dir;data];
   sort[config;tabname;path];
-  .z.m.log[`info][`dbwrite;"finished saving ",string tabname];
+  .z.m.log[`info]["dbwrite: finished saving ",string tabname];
   };
 
 appenddown:{[dir;part;tabname;data]
   / append rows to an existing on-disk partition (enumerates syms); does not sort
   / call sort separately once the partition is complete, to avoid re-sorting on every append
   / dir: hdb root (hsym); part: partition value; tabname: symbol; data: in-memory table
-  .z.m.log[`info][`dbwrite;"appending ",string[tabname]," partition ",string[part]," in ",string dir];
+  .z.m.log[`info]["dbwrite: appending ",string[tabname]," partition ",string[part]," in ",string dir];
   path:` sv (.Q.par[dir;part;tabname];`);
   if[not count @[key;path;{`$()}];
     '"di.dbwrite: appenddown partition does not exist at ",string path];
   .[path;();,;.Q.en[dir;data]];
-  .z.m.log[`info][`dbwrite;"finished appending ",string tabname];
+  .z.m.log[`info]["dbwrite: finished appending ",string tabname];
   };
 
 / internal - render a memory-usage dict as a "key=val MB; ..." string
@@ -201,7 +202,7 @@ memstats:{[]
 
 gc:{[]
   / run .Q.gc[] and log before/after memory stats
-  .z.m.log[`info][`dbwrite;"starting garbage collect. ",memstats[]];
+  .z.m.log[`info]["dbwrite: starting garbage collect. ",memstats[]];
   r:.Q.gc[];
-  .z.m.log[`info][`dbwrite;"garbage collection returned ",(string `long$r%1048576),"MB. ",memstats[]];
-  };
+  .z.m.log[`info]["dbwrite: garbage collection returned ",(string `long$r%1048576),"MB. ",memstats[]];
+  };
\ No newline at end of file
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index 53f5c44e..5e72482a 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -3,4 +3,4 @@
 
 \l ::dbwrite.q
 
-export:([init;readcsv;sort;applyattr;savedown;appenddown;gc])
\ No newline at end of file
+export:([init;readcsv;sort;applyattr;savedown;appenddown])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index 438e119f..93df2e3d 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -1,8 +1,8 @@
 action,ms,bytes,lang,code,repeat,minver,comment
 before,0,0,q,dbwrite:use`di.dbwrite,1,1,load module
-before,0,0,q,mylog:`info`warn`error!({[c;m]};{[c;m]};{[c;m]}),1,1,define no-op mock logger
-before,0,0,q,"caplog:([] fn:`symbol$();ctx:`symbol$();msg:())",1,1,initialise log capture table
-before,0,0,q,logcap:`info`warn`error!({[c;m] `caplog upsert(`info;c;m)};{[c;m] `caplog upsert(`warn;c;m)};{[c;m] `caplog upsert(`error;c;m)}),1,1,define capturing mock logger
+before,0,0,q,mylog:`info`warn`error!({[m]};{[m]};{[m]}),1,1,define no-op monadic mock logger (kx.log-style)
+before,0,0,q,"caplog:([] fn:`symbol$();msg:())",1,1,initialise log capture table
+before,0,0,q,logcap:`info`warn`error!({[m] `caplog upsert(`info;m)};{[m] `caplog upsert(`warn;m)};{[m] `caplog upsert(`error;m)}),1,1,define capturing monadic mock logger
 before,0,0,q,"rmrf:{[d] if[11h=type k:key d; rmrf each ` sv/:d,/:k]; @[hdel;d;{}]}",1,1,recursive delete helper for on-disk cleanup
 before,0,0,q,"`:tmp_dbw_valid.csv 0: (""tabname,att,column,sort"";""trade,p,sym,1"";""trade,,time,0"";""default,p,sym,1"")",1,1,write valid sort csv
 before,0,0,q,"`:tmp_dbw_badcols.csv 0: (""tabname,attr,col,sort"";""trade,p,sym,1"")",1,1,write csv with bad column names
@@ -163,7 +163,7 @@ run,0,0,q,dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,call readcsv with capturing lo
 true,0,0,q,2=count select from caplog where fn=`info,1,1,readcsv emits exactly two info entries
 true,0,0,q,"any caplog[`msg] like ""*reading sort config*""",1,1,readcsv logs read start message
 true,0,0,q,"any caplog[`msg] like ""*read 3 sort config*""",1,1,readcsv logs the row count read
-true,0,0,q,all (select from caplog where fn=`info)[`ctx]=`dbwrite,1,1,readcsv logs under the dbwrite context symbol
+true,0,0,q,"all (select from caplog where fn=`info)[`msg] like ""dbwrite: *""",1,1,readcsv logs are tagged with the dbwrite context prefix
 true,0,0,q,not any caplog[`fn]=`error,1,1,readcsv logs no errors on success
 
 comment,,,,,,,log call verification - readcsv error logging on file read failure
@@ -180,7 +180,7 @@ true,0,0,q,"any caplog[`msg] like ""*sort params found for: trade*""",1,1,sort l
 true,0,0,q,"any caplog[`msg] like ""*finished sorting the trade table*""",1,1,sort logs completion
 true,0,0,q,"any caplog[`msg] like ""*failed to sort*""",1,1,sort logs the partition sort failure
 true,0,0,q,"any caplog[`msg] like ""*unable to apply*""",1,1,sort logs the attribute application failure
-true,0,0,q,all (select from caplog where fn in `info`warn`error)[`ctx]=`dbwrite,1,1,all log entries use the dbwrite context symbol
+true,0,0,q,"all (select from caplog where fn in `info`warn`error)[`msg] like ""dbwrite: *""",1,1,all log entries carry the dbwrite context prefix
 
 comment,,,,,,,log call verification - default fallback and skip warning
 run,0,0,q,caplog:0#caplog,1,1,reset log capture
@@ -191,11 +191,12 @@ run,0,0,q,dbwrite.sort[nodflcfg;`other;enlist`:/],1,1,sort with no matching para
 true,0,0,q,any caplog[`fn]=`warn,1,1,sort emits a warn when no config applies
 true,0,0,q,"any caplog[`msg] like ""*skipping sort*""",1,1,warn message indicates the table is skipped
 
-comment,,,,,,,gc - runs and logs before/after
-run,0,0,q,caplog:0#caplog,1,1,reset log capture
-run,0,0,q,dbwrite.gc[],1,1,run garbage collect
-true,0,0,q,2=count select from caplog where fn=`info,1,1,gc logs two info entries (start and end)
-true,0,0,q,"any caplog[`msg] like ""*garbage collect*""",1,1,gc logs a garbage-collect message
+comment,,,,,,,kx.log integration - wire a real kx.log instance (system module) and actually format+emit
+run,0,0,q,kxlogger:use`kx.log,1,1,load the system kx.log module
+run,0,0,q,kxinst:kxlogger.createLog[],1,1,create a kx.log instance (default info level so messages truly format+emit)
+run,0,0,q,dbwrite.init[enlist[`log]!enlist `info`warn`error!(kxinst`info;kxinst`warn;kxinst`error)],1,1,init with a real kx.log-backed logger
+true,0,0,q,3=count dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv works end-to-end and emits real info logs through kx.log
+run,0,0,q,@[dbwrite.readcsv;`:nonexistent_file.csv;{x}],1,1,error path formats+emits a real error log through kx.log without crashing
 
 after,0,0,q,hdel`:tmp_dbw_valid.csv,1,1,remove valid csv
 after,0,0,q,hdel`:tmp_dbw_badcols.csv,1,1,remove bad cols csv

From 197336b76ed0b99325b786e1730b24c56ee6f432 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Tue, 23 Jun 2026 10:34:05 +0100
Subject: [PATCH 13/15] add getconfig; validate att in readcsv; update tests
 for new API

- add getconfig export to allow consumers to inspect .z.m.sortconfig
- readcsv now calls checkconfig after parsecsv so invalid att values are caught at load time
- tests updated throughout: readcsv/sort/savedown call sites use new signatures; config validation tests moved to setconfig section; gc section removed; setconfig happy-path and validation tests added (159 tests, all passing)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.q |   6 ++
 di/dbwrite/init.q    |   2 +-
 di/dbwrite/test.csv  | 138 +++++++++++++++++++++++++------------------
 3 files changed, 88 insertions(+), 58 deletions(-)

diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index 5e207cff..c9d00123 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -29,6 +29,7 @@ readcsv:{[file]
     '"di.dbwrite: readcsv file must be a symbol, got type ",string type file];
   file:hsym file;
   t:parsecsv @[readfile; file; readerr[file]];
+  checkconfig t;
   .z.m.log[`info][`dbwrite;"read ",(string count t)," sort config row(s) from ",string file];
   .z.m.sortconfig:t;
   };
@@ -40,6 +41,11 @@ setconfig:{[t]
   .z.m.sortconfig:t;
   };
 
+getconfig:{[]
+  / return the current sort config stored in .z.m.sortconfig; (::) if not yet set
+  :.z.m.sortconfig;
+  };
+
 / internal - protected file read; only the i/o so a genuine read failure gets the readerr message
 readfile:{[file]
   / returns the raw csv lines; header validation and parsing happen in parsecsv
diff --git a/di/dbwrite/init.q b/di/dbwrite/init.q
index d9347d4c..ae5dd0d5 100644
--- a/di/dbwrite/init.q
+++ b/di/dbwrite/init.q
@@ -3,4 +3,4 @@
 
 \l ::dbwrite.q
 
-export:([init;readcsv;setconfig;sort;applyattr;savedown;appenddown])
\ No newline at end of file
+export:([init;readcsv;setconfig;getconfig;sort;applyattr;savedown;appenddown])
\ No newline at end of file
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index 438e119f..88f0805d 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -18,7 +18,7 @@ before,0,0,q,"`:tmp_dbw_long.csv 0: (""tabname,att,column,sort"";""trade,p,sym,1
 before,0,0,q,"`:tmp_dbw_badbool.csv 0: (""tabname,att,column,sort"";""trade,p,sym,x"")",1,1,write csv with a non-boolean sort value
 before,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,init with no-op logger
 before,0,0,q,cfg:([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,reusable valid config table (hand-built)
-before,0,0,q,nodflcfg:dbwrite.readcsv `:tmp_dbw_nodfl.csv,1,1,reusable config with a trade row but no default
+before,0,0,q,nodflcfg:([] tabname:enlist`trade; att:enlist`p; column:enlist`sym; sort:enlist 1b),1,1,reusable config with a trade row but no default
 
 comment,,,,,,,init - dependency injection validation (type errors / empty / null)
 fail,0,0,q,dbwrite.init[(::)],1,1,init rejects :: as deps
@@ -31,13 +31,16 @@ fail,0,0,q,dbwrite.init[enlist[`log]!enlist(`info`warn!(mylog`info;mylog`warn))]
 true,0,0,q,"""di.dbwrite:""~11#@[{dbwrite.init[(::)]};(::);{x}]",1,1,error message is prefixed di.dbwrite
 run,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,re-init with valid log dep
 
-comment,,,,,,,readcsv - returns a config table from a csv
-true,0,0,q,3=count dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv returns a 3-row config table
-true,0,0,q,`tabname`att`column`sort~cols dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv table has the correct column schema
-true,0,0,q,(dbwrite.readcsv `:tmp_dbw_valid.csv)~([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,readcsv round-trips the csv content
-true,0,0,q,0=count dbwrite.readcsv `:tmp_dbw_empty.csv,1,1,readcsv returns an empty table for a header-only csv
-true,0,0,q,(dbwrite.readcsv `:tmp_dbw_reorder.csv)~dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv is column-order independent
-true,0,0,q,`tabname`att`column`sort~cols dbwrite.readcsv `:tmp_dbw_reorder.csv,1,1,readcsv normalises reordered columns to canonical order
+comment,,,,,,,readcsv - stores a config from a csv into module state
+run,0,0,q,dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,readcsv stores config from a valid csv without error
+true,0,0,q,3=count dbwrite.getconfig[],1,1,stored config has 3 rows
+true,0,0,q,`tabname`att`column`sort~cols dbwrite.getconfig[],1,1,stored config has the correct column schema
+true,0,0,q,(dbwrite.getconfig[])~([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,stored config matches csv content
+run,0,0,q,dbwrite.readcsv `:tmp_dbw_empty.csv,1,1,readcsv stores an empty config for a header-only csv
+true,0,0,q,0=count dbwrite.getconfig[],1,1,empty csv produces empty stored config
+run,0,0,q,dbwrite.readcsv `:tmp_dbw_reorder.csv,1,1,readcsv with reordered columns stores correct config
+true,0,0,q,(dbwrite.getconfig[])~([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,readcsv is column-order independent
+true,0,0,q,`tabname`att`column`sort~cols dbwrite.getconfig[],1,1,readcsv normalises reordered columns to canonical order
 
 comment,,,,,,,readcsv - input and header validation failures
 fail,0,0,q,dbwrite.readcsv `:nonexistent_file.csv,1,1,readcsv errors on a missing file
@@ -49,64 +52,83 @@ fail,0,0,q,dbwrite.readcsv `:tmp_dbw_nohdr.csv,1,1,readcsv rejects a csv with no
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_short.csv,1,1,readcsv rejects a data row with too few fields
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_long.csv,1,1,readcsv rejects a data row with too many fields
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_badbool.csv,1,1,readcsv rejects a non-boolean sort value
-
-comment,,,,,,,sort - happy path via both a hand-built table and a csv
-run,0,0,q,dbwrite.sort[cfg;`trade;()],1,1,sort accepts a hand-built config table
-run,0,0,q,dbwrite.sort[dbwrite.readcsv `:tmp_dbw_valid.csv;`trade;()],1,1,sort accepts a config table read from a csv
-
-comment,,,,,,,sort - config validation (type errors / bad content)
-fail,0,0,q,dbwrite.sort[42;`trade;enlist`:/],1,1,sort rejects a non-table config
-fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; bad:enlist`p; column:enlist`sym; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects unrecognised config columns
-fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`sym);`trade;enlist`:/],1,1,sort rejects a missing required config column
-fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`z; column:enlist`sym; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects unknown attribute values
-fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`sym; sort:enlist 1);`trade;enlist`:/],1,1,sort rejects a non-boolean sort column
-fail,0,0,q,dbwrite.sort[dbwrite.readcsv `:tmp_dbw_badatt.csv;`trade;enlist`:/],1,1,sort rejects csv config with invalid attributes
-fail,0,0,q,dbwrite.sort[([] tabname:enlist`; att:enlist`p; column:enlist`sym; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects a null tabname in config
-fail,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`p; column:enlist`; sort:enlist 1b);`trade;enlist`:/],1,1,sort rejects a null column in config
-
-comment,,,,,,,sort - edge cases (empty config / null att / empty dirs / atom dir / type error)
-true,0,0,q,()~dbwrite.sort[([] tabname:`symbol$();att:`symbol$();column:`symbol$();sort:`boolean$());`trade;enlist`:/],1,1,empty config yields no work and returns ()
-run,0,0,q,dbwrite.sort[([] tabname:enlist`trade; att:enlist`; column:enlist`sym; sort:enlist 0b);`trade;enlist`:/],1,1,sort accepts a null (no) attribute row
-run,0,0,q,dbwrite.sort[cfg;`trade;()],1,1,sort handles an empty partition list without error
-run,0,0,q,dbwrite.sort[cfg;`trade;`:/],1,1,sort accepts a single atom hsym dir not just a list
-fail,0,0,q,dbwrite.sort[cfg;42;enlist`:/],1,1,sort errors on a non-symbol table name
-true,0,0,q,"""di.dbwrite:""~11#@[{dbwrite.sort[cfg;42;enlist`:/]};(::);{x}]",1,1,non-symbol table name gives a di.dbwrite:-prefixed error
-
-comment,,,,,,,sort - params resolution (table-specific / default / none / (::) fallback)
-run,0,0,q,dbwrite.sort[cfg;`trade;enlist`:/],1,1,sort uses the table-specific params row
-run,0,0,q,dbwrite.sort[cfg;`other;enlist`:/],1,1,sort falls back to the default params row
-true,0,0,q,()~dbwrite.sort[nodflcfg;`other;enlist`:/],1,1,sort returns () when no params found and no default row
-run,0,0,q,dbwrite.sort[(::);`anytable;()],1,1,sort with (::) config uses the built-in default config
+fail,0,0,q,dbwrite.readcsv `:tmp_dbw_badatt.csv,1,1,readcsv rejects a csv with invalid attribute values
+
+comment,,,,,,,setconfig - stores a hand-built table into module state
+run,0,0,q,dbwrite.setconfig[cfg],1,1,setconfig accepts a valid hand-built table
+true,0,0,q,(dbwrite.getconfig[])~cfg,1,1,getconfig returns the table stored by setconfig
+run,0,0,q,dbwrite.setconfig[0#cfg],1,1,setconfig accepts an empty (but correctly typed) table
+true,0,0,q,0=count dbwrite.getconfig[],1,1,empty table stored correctly
+
+comment,,,,,,,setconfig - config validation (type errors / bad content)
+fail,0,0,q,dbwrite.setconfig[42],1,1,setconfig rejects a non-table value
+fail,0,0,q,dbwrite.setconfig[([] tabname:enlist`trade; bad:enlist`p; column:enlist`sym; sort:enlist 1b)],1,1,setconfig rejects unrecognised config columns
+fail,0,0,q,dbwrite.setconfig[([] tabname:enlist`trade; att:enlist`p; column:enlist`sym)],1,1,setconfig rejects a missing required config column
+fail,0,0,q,dbwrite.setconfig[([] tabname:enlist`trade; att:enlist`z; column:enlist`sym; sort:enlist 1b)],1,1,setconfig rejects unknown attribute values
+fail,0,0,q,dbwrite.setconfig[([] tabname:enlist`trade; att:enlist`p; column:enlist`sym; sort:enlist 1)],1,1,setconfig rejects a non-boolean sort column
+fail,0,0,q,dbwrite.setconfig[([] tabname:enlist`; att:enlist`p; column:enlist`sym; sort:enlist 1b)],1,1,setconfig rejects a null tabname in config
+fail,0,0,q,dbwrite.setconfig[([] tabname:enlist`trade; att:enlist`p; column:enlist`; sort:enlist 1b)],1,1,setconfig rejects a null column in config
+
+comment,,,,,,,sort - happy path using stored config (set via setconfig or readcsv)
+run,0,0,q,dbwrite.setconfig[cfg],1,1,set config to hand-built cfg
+run,0,0,q,dbwrite.sort[`trade;()],1,1,sort with a hand-built stored config and no partitions
+run,0,0,q,dbwrite.readcsv `:tmp_dbw_valid.csv,1,1,set config via readcsv
+run,0,0,q,dbwrite.sort[`trade;()],1,1,sort with a csv-loaded stored config and no partitions
+
+comment,,,,,,,sort - tabname type validation
+fail,0,0,q,dbwrite.sort[42;enlist`:/],1,1,sort errors on a non-symbol table name
+true,0,0,q,"""di.dbwrite:""~11#@[{dbwrite.sort[42;enlist`:/]};(::);{x}]",1,1,non-symbol table name gives a di.dbwrite:-prefixed error
+
+comment,,,,,,,sort - edge cases (empty config / null att / empty dirs / atom dir)
+run,0,0,q,dbwrite.setconfig[0#cfg],1,1,set empty config
+true,0,0,q,()~dbwrite.sort[`trade;enlist`:/],1,1,empty config yields no work and returns ()
+run,0,0,q,dbwrite.setconfig[([] tabname:enlist`trade; att:enlist`; column:enlist`sym; sort:enlist 0b)],1,1,set config with a null (no) attribute row
+run,0,0,q,dbwrite.sort[`trade;enlist`:/],1,1,sort accepts a null (no) attribute row
+run,0,0,q,dbwrite.setconfig[cfg],1,1,set config to cfg
+run,0,0,q,dbwrite.sort[`trade;()],1,1,sort handles an empty partition list without error
+run,0,0,q,dbwrite.sort[`trade;`:/],1,1,sort accepts a single atom hsym dir not just a list
+
+comment,,,,,,,sort - params resolution (table-specific / default / none / default fallback)
+run,0,0,q,dbwrite.setconfig[cfg],1,1,set config to cfg
+run,0,0,q,dbwrite.sort[`trade;enlist`:/],1,1,sort uses the table-specific params row
+run,0,0,q,dbwrite.sort[`other;enlist`:/],1,1,sort falls back to the default params row
+run,0,0,q,dbwrite.setconfig[nodflcfg],1,1,set config to nodflcfg (no default row)
+true,0,0,q,()~dbwrite.sort[`other;enlist`:/],1,1,sort returns () when no params found and no default row
+run,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,reset module state (clears sortconfig to (::))
+run,0,0,q,dbwrite.sort[`anytable;()],1,1,sort with unset sortconfig falls back to the built-in default config
 
 comment,,,,,,,sort - on-disk end-to-end (explicit config sorts by sym and applies p#)
 run,0,0,q,`:dbw_sort_tp/.d set `sym`price,1,1,write column order file
 run,0,0,q,`:dbw_sort_tp/sym set `IBM`AAPL`MSFT,1,1,write unsorted sym column
 run,0,0,q,`:dbw_sort_tp/price set 200 100 300f,1,1,write price column
-run,0,0,q,dbwrite.sort[cfg;`trade;`:dbw_sort_tp/],1,1,sort the trade partition by its config
+run,0,0,q,dbwrite.setconfig[cfg],1,1,set config to cfg before sort
+run,0,0,q,dbwrite.sort[`trade;`:dbw_sort_tp/],1,1,sort the trade partition by its config
 true,0,0,q,`AAPL`IBM`MSFT~exec sym from get `:dbw_sort_tp/,1,1,sym sorted ascending on disk
 true,0,0,q,`p=attr get `:dbw_sort_tp/sym,1,1,p attribute applied to sym on disk
 
-comment,,,,,,,sort - on-disk end-to-end ((::) default config sorts by time)
+comment,,,,,,,sort - on-disk end-to-end (default config sorts by time)
 run,0,0,q,`:dbw_def_tp/.d set `time`sym,1,1,write column order file
 run,0,0,q,`:dbw_def_tp/time set 2024.01.01D09:00 2024.01.01D08:00,1,1,write unsorted time column
 run,0,0,q,`:dbw_def_tp/sym set `IBM`AAPL,1,1,write sym column
-run,0,0,q,dbwrite.sort[(::);`anytable;`:dbw_def_tp/],1,1,sort using the built-in default config
+run,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,reset sortconfig to (::) for default-fallback test
+run,0,0,q,dbwrite.sort[`anytable;`:dbw_def_tp/],1,1,sort using the built-in default config
 true,0,0,q,(asc exec time from get `:dbw_def_tp/)~exec time from get `:dbw_def_tp/,1,1,time sorted ascending by the default config
 
-comment,,,,,,,sort - multiple partition dirs in one call, and a non-fatal partition failure
+comment,,,,,,,sort - multiple partition dirs in one call and a non-fatal partition failure
 run,0,0,q,`:dbw_md1/.d set `sym`px,1,1,write md1 column order
 run,0,0,q,`:dbw_md1/sym set `c`a`b,1,1,write md1 unsorted sym
 run,0,0,q,`:dbw_md1/px set 1 2 3f,1,1,write md1 price
 run,0,0,q,`:dbw_md2/.d set `sym`px,1,1,write md2 column order
 run,0,0,q,`:dbw_md2/sym set `b`c`a,1,1,write md2 unsorted sym
 run,0,0,q,`:dbw_md2/px set 1 2 3f,1,1,write md2 price
-run,0,0,q,dbwrite.sort[cfg;`trade;(`:dbw_md1/;`:dbw_md2/)],1,1,sort two partition dirs in one call
+run,0,0,q,dbwrite.setconfig[cfg],1,1,set config to cfg
+run,0,0,q,dbwrite.sort[`trade;(`:dbw_md1/;`:dbw_md2/)],1,1,sort two partition dirs in one call
 true,0,0,q,`a`b`c~exec sym from get `:dbw_md1/,1,1,first partition sorted
 true,0,0,q,`a`b`c~exec sym from get `:dbw_md2/,1,1,second partition sorted
 run,0,0,q,`:dbw_good/.d set `sym`px,1,1,write good-partition column order
 run,0,0,q,`:dbw_good/sym set `c`a`b,1,1,write good-partition unsorted sym
 run,0,0,q,`:dbw_good/px set 1 2 3f,1,1,write good-partition price
-run,0,0,q,dbwrite.sort[cfg;`trade;(`:dbw_good/;`:/)],1,1,sort a good dir alongside a bad one
+run,0,0,q,dbwrite.sort[`trade;(`:dbw_good/;`:/)],1,1,sort a good dir alongside a bad one
 true,0,0,q,`a`b`c~exec sym from get `:dbw_good/,1,1,good partition still sorted despite the bad one failing
 
 comment,,,,,,,applyattr - applies an attribute on disk and is a no-op for non-attributes
@@ -120,16 +142,18 @@ true,0,0,q,`=attr get `:dbw_attr_tp/price,1,1,invalid attribute leaves the colum
 
 comment,,,,,,,savedown - write an in-memory table to an hdb partition then sort it
 run,0,0,q,"sdtbl:([]time:2024.01.01D09:00 2024.01.01D08:00;sym:`IBM`AAPL;price:100 200f)",1,1,unsorted test table
-run,0,0,q,dbwrite.savedown[(::);`:dbw_hdb;2024.01.01;`trade;sdtbl],1,1,savedown with the default config (sort by time)
+run,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,reset sortconfig to (::) for default-config savedown
+run,0,0,q,dbwrite.savedown[`:dbw_hdb;2024.01.01;`trade;sdtbl],1,1,savedown with the default config (sort by time)
 run,0,0,q,"sdpath:` sv (.Q.par[`:dbw_hdb;2024.01.01;`trade];`)",1,1,resolve the partition path
 true,0,0,q,2=count get sdpath,1,1,two rows written to the partition
 true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,1,partition sorted by time after savedown
 true,0,0,q,0<count key `:dbw_hdb/sym,1,1,sym enumeration file created in the hdb root
-run,0,0,q,"dbwrite.savedown[(::);`:dbw_hdb;2024.01.02;`nosym;([]time:enlist 2024.01.02D10:00;price:enlist 50f)]",1,1,savedown a table without a sym column
+run,0,0,q,"dbwrite.savedown[`:dbw_hdb;2024.01.02;`nosym;([]time:enlist 2024.01.02D10:00;price:enlist 50f)]",1,1,savedown a table without a sym column
 true,0,0,q,"1=count get ` sv (.Q.par[`:dbw_hdb;2024.01.02;`nosym];`)",1,1,row written for a table without sym
 run,0,0,q,"sdtbl2:([]time:2024.01.03D09:00 2024.01.03D08:00;sym:`IBM`AAPL;price:300 100f)",1,1,table with descending price for an explicit-config savedown
 run,0,0,q,"pcfg:([] tabname:enlist`trade; att:enlist`; column:enlist`price; sort:enlist 1b)",1,1,explicit config sorting trade by price
-run,0,0,q,"dbwrite.savedown[pcfg;`:dbw_hdb;2024.01.03;`trade;sdtbl2]",1,1,savedown with an explicit (non-default) config
+run,0,0,q,dbwrite.setconfig[pcfg],1,1,set config to pcfg for price-sorted savedown
+run,0,0,q,"dbwrite.savedown[`:dbw_hdb;2024.01.03;`trade;sdtbl2]",1,1,savedown with an explicit (non-default) config
 run,0,0,q,"sd3:get ` sv (.Q.par[`:dbw_hdb;2024.01.03;`trade];`)",1,1,read the explicit-config partition
 true,0,0,q,(asc exec price from sd3)~exec price from sd3,1,1,partition sorted by price per the explicit config
 
@@ -137,20 +161,23 @@ comment,,,,,,,appenddown - append to a partition without sorting then sort expli
 run,0,0,q,"dbwrite.appenddown[`:dbw_hdb;2024.01.01;`trade;([]time:enlist 2024.01.01D07:00;sym:enlist`MSFT;price:enlist 300f)]",1,1,append an earlier-time row to an existing partition
 true,0,0,q,3=count get sdpath,1,1,three rows on disk after appenddown
 true,0,0,q,not (asc exec time from get sdpath)~exec time from get sdpath,1,1,rows not auto-sorted by appenddown
-run,0,0,q,dbwrite.sort[(::);`trade;sdpath],1,1,sort the partition explicitly after appending
+run,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,reset sortconfig to (::) for default-config sort
+run,0,0,q,dbwrite.sort[`trade;sdpath],1,1,sort the partition explicitly after appending
 true,0,0,q,(asc exec time from get sdpath)~exec time from get sdpath,1,1,sorted by time after the explicit sort
 fail,0,0,q,dbwrite.appenddown[`:dbw_hdb;2099.01.01;`nope;sdtbl],1,1,appenddown into a non-existent partition errors
 
 comment,,,,,,,integration - savedown realistic (repeated/unsorted sym) data builds a valid queryable partition
 run,0,0,q,symcfg:([] tabname:enlist`trade; att:enlist`p; column:enlist`sym; sort:enlist 1b),1,1,config that sorts by sym and applies p#
 run,0,0,q,"rd:([] time:2024.01.01D10 2024.01.01D09 2024.01.01D11; sym:`IBM`AAPL`IBM; px:1 2 3f)",1,1,realistic unsorted data with a repeated sym
-run,0,0,q,dbwrite.savedown[symcfg;`:dbw_ihdb;2024.01.01;`trade;rd],1,1,savedown repeated-sym data (must not crash applying p#)
+run,0,0,q,dbwrite.setconfig[symcfg],1,1,set config to symcfg
+run,0,0,q,dbwrite.savedown[`:dbw_ihdb;2024.01.01;`trade;rd],1,1,savedown repeated-sym data (must not crash applying p#)
 run,0,0,q,"ip:get ` sv (.Q.par[`:dbw_ihdb;2024.01.01;`trade];`)",1,1,read the written partition back
 true,0,0,q,`AAPL`IBM`IBM~value ip`sym,1,1,partition is sorted (grouped) by sym on disk (sym is enumerated - resolve with value)
 true,0,0,q,`p=attr ip`sym,1,1,p# applied to sym post-sort (valid because the data is grouped)
 true,0,0,q,2=count select from ip where sym=`IBM,1,1,p# lookup is correct - two IBM rows
 true,0,0,q,1=count select from ip where sym=`AAPL,1,1,p# lookup is correct - one AAPL row
-run,0,0,q,dbwrite.savedown[(::);`:dbw_ihdb;2024.01.02;`trade;rd],1,1,savedown repeated-sym data with the default (time) config must not crash
+run,0,0,q,dbwrite.init[enlist[`log]!enlist mylog],1,1,reset sortconfig to (::) for default-config savedown
+run,0,0,q,dbwrite.savedown[`:dbw_ihdb;2024.01.02;`trade;rd],1,1,savedown repeated-sym data with the default (time) config must not crash
 true,0,0,q,(asc x)~x:exec time from get ` sv (.Q.par[`:dbw_ihdb;2024.01.02;`trade];`),1,1,default-config partition sorted by time
 true,0,0,q,`=attr (get ` sv (.Q.par[`:dbw_ihdb;2024.01.02;`trade];`))`sym,1,1,default-config leaves sym unattributed (not falsely parted)
 
@@ -174,7 +201,8 @@ true,0,0,q,"any caplog[`msg] like ""*failed to read*""",1,1,error log message me
 
 comment,,,,,,,log call verification - sort info and partition-failure logging
 run,0,0,q,caplog:0#caplog,1,1,reset log capture
-run,0,0,q,dbwrite.sort[cfg;`trade;enlist`:/],1,1,sort with table-specific params against a bad dir
+run,0,0,q,dbwrite.setconfig[cfg],1,1,set config to cfg
+run,0,0,q,dbwrite.sort[`trade;enlist`:/],1,1,sort with table-specific params against a bad dir
 true,0,0,q,"any caplog[`msg] like ""*sorting the trade table*""",1,1,sort logs sort start
 true,0,0,q,"any caplog[`msg] like ""*sort params found for: trade*""",1,1,sort logs the params lookup
 true,0,0,q,"any caplog[`msg] like ""*finished sorting the trade table*""",1,1,sort logs completion
@@ -184,19 +212,15 @@ true,0,0,q,all (select from caplog where fn in `info`warn`error)[`ctx]=`dbwrite,
 
 comment,,,,,,,log call verification - default fallback and skip warning
 run,0,0,q,caplog:0#caplog,1,1,reset log capture
-run,0,0,q,dbwrite.sort[cfg;`other;enlist`:/],1,1,sort falls back to the default params row
+run,0,0,q,dbwrite.setconfig[cfg],1,1,set config to cfg
+run,0,0,q,dbwrite.sort[`other;enlist`:/],1,1,sort falls back to the default params row
 true,0,0,q,"any caplog[`msg] like ""*using defaults*""",1,1,sort logs the default params fallback
 run,0,0,q,caplog:0#caplog,1,1,reset log capture
-run,0,0,q,dbwrite.sort[nodflcfg;`other;enlist`:/],1,1,sort with no matching params and no default
+run,0,0,q,dbwrite.setconfig[nodflcfg],1,1,set config to nodflcfg (no default row)
+run,0,0,q,dbwrite.sort[`other;enlist`:/],1,1,sort with no matching params and no default
 true,0,0,q,any caplog[`fn]=`warn,1,1,sort emits a warn when no config applies
 true,0,0,q,"any caplog[`msg] like ""*skipping sort*""",1,1,warn message indicates the table is skipped
 
-comment,,,,,,,gc - runs and logs before/after
-run,0,0,q,caplog:0#caplog,1,1,reset log capture
-run,0,0,q,dbwrite.gc[],1,1,run garbage collect
-true,0,0,q,2=count select from caplog where fn=`info,1,1,gc logs two info entries (start and end)
-true,0,0,q,"any caplog[`msg] like ""*garbage collect*""",1,1,gc logs a garbage-collect message
-
 after,0,0,q,hdel`:tmp_dbw_valid.csv,1,1,remove valid csv
 after,0,0,q,hdel`:tmp_dbw_badcols.csv,1,1,remove bad cols csv
 after,0,0,q,hdel`:tmp_dbw_badatt.csv,1,1,remove bad att csv

From 23b9b7f52b2554da9876ea0aae52dddfd25acd3a Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Tue, 23 Jun 2026 10:35:51 +0100
Subject: [PATCH 14/15] update dbwrite docs to reflect new config state API

- remove stateless config-per-call design description; module now stores config in .z.m.sortconfig
- document readcsv as a store operation (not a return); add setconfig and getconfig sections
- update sort and savedown signatures (config arg removed from both)
- note gc is called automatically by savedown; remove gc from exported symbols list
- update test suite description to match new setconfig/getconfig coverage

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 di/dbwrite/dbwrite.md | 88 +++++++++++++++++++++++++------------------
 1 file changed, 51 insertions(+), 37 deletions(-)

diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 974862ec..82035955 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -2,19 +2,16 @@
 
 Write, sort, and attribute utilities for kdb+ processes that persist data to disk (rdb, wdb, tickerlogreplay).
 
-The sorting/attribute engine is `di.sort` baked in directly: a **config table** (`tabname`,`att`,`column`,`sort`) drives which columns are sorted and which attributes are applied per table. You pass that config straight to `sort`/`savedown`; if it lives in a CSV, `readcsv` reads it into the right shape. A table with no explicit entry falls back to a `default` row, and `(::)` selects a built-in default (sort every table by `time` ascending).
-
-`di.dbwrite` holds no sort state of its own — each call takes the config it should use, so you can build that config by hand, from a query, or from a CSV and reuse or vary it freely.
+A **config table** (`tabname`,`att`,`column`,`sort`) drives which columns are sorted and which attributes are applied per table. Load it once with `readcsv` (from a CSV) or `setconfig` (from an in-memory table); `sort` and `savedown` read from module state automatically. A table with no explicit entry falls back to a `default` row; if no config has been loaded, the built-in default (sort every table by `time` ascending) applies.
 
 ---
 
 ## Features
 
-- Write an in-memory table to a date-partitioned HDB with `savedown` — enumerates syms, applies `p#` to `sym`, writes, then sorts per config
+- Write an in-memory table to a date-partitioned HDB with `savedown` — enumerates syms, writes, sorts per stored config, then runs `.Q.gc[]`
 - Append rows to an existing partition with `appenddown` — enumerates syms and appends; sort separately when the partition is complete
 - Sort on-disk table partitions by configured columns using `xasc`, then apply kdb+ attributes (`p`,`s`,`g`,`u`)
-- Config supplied as an in-memory table or read from a CSV (`readcsv`); a `default` row or `(::)` provides a fallback
-- Run `.Q.gc[]` with before/after memory logging
+- Config loaded once via `readcsv` (from a CSV) or `setconfig` (from a table); inspectable at any time with `getconfig`
 - Sort and attribute errors are caught-and-logged (a single partition failure does not halt the run); config and write errors are raised to the caller with a `di.dbwrite:` prefix
 
 ---
@@ -45,7 +42,7 @@ dbwrite.init[enlist[`log]!enlist logdep]
 | `column` | symbol | Column to sort and/or attribute |
 | `sort` | boolean | `1b` — include in the `xasc` sort key; `0b` — attribute only |
 
-Build it directly, or read it from a CSV with `readcsv`. A CSV must have exactly the four columns `tabname,att,column,sort`, in **any** order (the result is normalised to canonical order); a missing, extra, or misnamed column raises a clear `di.dbwrite:` error rather than silently mis-parsing.
+Build it directly and load with `setconfig`, or read it from a CSV with `readcsv`. A CSV must have exactly the four columns `tabname,att,column,sort` in **any** order (the result is normalised to canonical order); a missing, extra, or misnamed column raises a clear `di.dbwrite:` error rather than silently mis-parsing.
 
 ```
 tabname,att,column,sort
@@ -61,22 +58,23 @@ default,,time,1
 | Function | Description |
 |---|---|
 | `init[deps]` | Wire injected dependencies; must be called first |
-| `readcsv[file]` | Read a config CSV and return it as a table |
-| `sort[config;tabname;dirs]` | Sort on-disk partition(s) for a table and apply attributes per config |
-| `savedown[config;dir;part;tabname;data]` | Write an in-memory table to an HDB partition, then sort it per config |
+| `readcsv[file]` | Read a config CSV and store it in module state |
+| `setconfig[t]` | Store a hand-built config table in module state |
+| `getconfig[]` | Return the currently stored config (`::` if not yet set) |
+| `sort[tabname;dirs]` | Sort on-disk partition(s) for a table and apply attributes per stored config |
+| `savedown[dir;part;tabname;data]` | Write an in-memory table to an HDB partition, sort it, then run gc |
 | `appenddown[dir;part;tabname;data]` | Append rows to an existing partition (no sort) |
 | `applyattr[dloc;colname;att]` | Apply a single kdb+ attribute to an on-disk column |
-| `gc[]` | Run `.Q.gc[]` and log before/after memory stats |
 
 ---
 
 ### `init[deps]`
 
-Wire injected dependencies. Must be called before any other function.
+Wire injected dependencies. Must be called before any other function. Also resets the stored sort config to `(::)`.
 
 | Arg | Type | Description |
 |---|---|---|
-| `deps` | dict | Must contain `` `log `` → `` `info`warn`error!(infofn;warnfn;errfn) ``. |
+| `deps` | dict | Must contain `` `log `` → `` `info`warn`error!(infofn;warnfn;errfn) `` |
 
 Throws (prefixed `di.dbwrite:`) if `deps` is not a dict, `log` is missing, or the log dict lacks any required key.
 
@@ -84,54 +82,76 @@ Throws (prefixed `di.dbwrite:`) if `deps` is not a dict, `log` is missing, or th
 
 ### `readcsv[file]`
 
-Read a config CSV and **return** it as a table (does not store it) — pass the result to `sort` or `savedown`.
+Read a config CSV and store it in module state (equivalent to calling `setconfig` with the parsed result). The stored config is used by subsequent calls to `sort` and `savedown`.
 
 | Parameter | Type | Description |
 |---|---|---|
 | `file` | symbol/hsym | Path to the CSV; coerced with `hsym`. Errors (`di.dbwrite:`) if not a symbol. |
 
-The CSV is parsed field-by-field and validated as it is read — it does **not** silently pad, truncate, or coerce malformed rows. A clear `di.dbwrite:` error is raised if: the header is not exactly `tabname,att,column,sort` (any order); any data row does not have exactly four fields; or any `sort` value is not `0` or `1`. Column order is normalised to canonical. Attribute-value validation happens later in `sort`.
+The CSV is parsed field-by-field and fully validated before storing — it does **not** silently pad, truncate, or coerce malformed rows. A clear `di.dbwrite:` error is raised if: the header is not exactly `tabname,att,column,sort` (any order); any data row does not have exactly four fields; any `sort` value is not `0` or `1`; or any `att` value is not in `` ` `p`s`g`u ``. Column order is normalised to canonical.
+
+```q
+dbwrite.readcsv `:config/sort.csv
+```
+
+---
+
+### `setconfig[t]`
+
+Store a hand-built config table in module state. Alternative to `readcsv` when the config is constructed in-session rather than read from a file. Validates the table before storing — throws (`di.dbwrite:`) on any schema or content error.
+
+| Parameter | Type | Description |
+|---|---|---|
+| `t` | table | Config table with columns `tabname`,`att`,`column`,`sort` |
+
+```q
+dbwrite.setconfig ([] tabname:`trade`default; att:`p`; column:`sym`time; sort:11b)
+```
+
+---
+
+### `getconfig[]`
+
+Return the currently stored sort config. Returns `(::)` if `init` has been called but neither `readcsv` nor `setconfig` has been called yet.
 
 ```q
-config:dbwrite.readcsv `:config/sort.csv
+dbwrite.getconfig[]
 ```
 
 ---
 
-### `sort[config;tabname;dirs]`
+### `sort[tabname;dirs]`
 
-Sort and apply attributes to the on-disk partition(s) for one table.
+Sort and apply attributes to the on-disk partition(s) for one table, using the config stored by `readcsv` or `setconfig`. Falls back to the built-in default (sort by `time` ascending) if no config has been loaded.
 
 | Parameter | Type | Description |
 |---|---|---|
-| `config` | table, or `::` | Config table (built directly or via `readcsv`). `::` uses the built-in default config. |
 | `tabname` | symbol | Table name. Errors (`di.dbwrite:`) if not a symbol. |
-| `dirs` | hsym, or list of hsyms | Partition directory (or directories). |
+| `dirs` | hsym, or list of hsyms | Partition directory or directories. |
 
-`config` is validated first (errors `di.dbwrite:` if not a table, has unknown/missing columns, a non-boolean `sort`, or an `att` outside `` ` `p`s`g`u ``). Row lookup: the table's own rows → the `default` row → otherwise a warn is logged and `()` returned. Each partition is processed independently; a failure on one is logged and does not halt the rest.
+Row lookup: the table's own rows → the `default` row → otherwise a warn is logged and `()` returned. Each partition is processed independently; a failure on one is logged and does not halt the rest.
 
 ```q
-dbwrite.sort[config; `trade; `:/hdb/2024.01.02/trade`:/hdb/2024.01.03/trade]
+dbwrite.sort[`trade; (`:hdb/2024.01.02/trade; `:hdb/2024.01.03/trade)]
 ```
 
 ---
 
-### `savedown[config;dir;part;tabname;data]`
+### `savedown[dir;part;tabname;data]`
 
-Write an in-memory table to a date-partitioned HDB partition, then sort it per `config`. Enumerates symbol columns against the HDB sym file before writing. Sorting **and** attributes are driven entirely by `config` and applied by `sort` *after* the write — so an attribute like `p#` is only applied once its column is grouped (e.g. a config that sorts by `sym` and parts `sym`), never to unsorted data.
+Write an in-memory table to a date-partitioned HDB partition, sort it per the stored config, then run `.Q.gc[]`. Enumerates symbol columns against the HDB sym file before writing. Sorting and attributes are applied by `sort` *after* the write — so an attribute like `p#` is only applied once its column is correctly grouped.
 
 | Parameter | Type | Description |
 |---|---|---|
-| `config` | table, or `::` | Config passed through to `sort` (`::` for the default). |
 | `dir` | hsym | HDB root directory (e.g. `` `:hdb ``). |
 | `part` | date/month/int | Partition value. |
 | `tabname` | symbol | Table name — determines the partition subdirectory. |
 | `data` | table | In-memory table to write. |
 
-Throws on write failure. If the table has no `sym` column, enumeration and `p#` are skipped.
+Throws on write failure.
 
 ```q
-dbwrite.savedown[config; `:hdb; 2024.01.02; `trade; data]
+dbwrite.savedown[`:hdb; 2024.01.02; `trade; data]
 ```
 
 ---
@@ -153,7 +173,7 @@ Throws (prefixed `di.dbwrite:`) if the partition does not exist, or on write fai
 / intraday: append each batch as it arrives
 dbwrite.appenddown[`:hdb; 2024.01.02; `trade; batch]
 / end-of-day: sort once when done
-dbwrite.sort[config; `trade; .Q.par[`:hdb;2024.01.02;`trade]]
+dbwrite.sort[`trade; .Q.par[`:hdb; 2024.01.02; `trade]]
 ```
 
 ---
@@ -163,17 +183,11 @@ dbwrite.sort[config; `trade; .Q.par[`:hdb;2024.01.02;`trade]]
 Apply a single kdb+ attribute to one on-disk column (best-effort: logs and swallows errors so a run continues). A non-attribute `att` (the empty sentinel or any value outside `` `p`s`g`u ``) is a silent no-op.
 
 ```q
-dbwrite.applyattr[`:/hdb/2024.01.02/trade; `sym; `p]
+dbwrite.applyattr[`:hdb/2024.01.02/trade; `sym; `p]
 ```
 
 ---
 
-### `gc[]`
-
-Run `.Q.gc[]`, logging memory stats before and after.
-
----
-
 ## Running tests
 
 ```q
@@ -181,12 +195,12 @@ k4unit:use`di.k4unit
 k4unit.moduletest`di.dbwrite
 ```
 
-The suite injects mock loggers: a no-op logger, and a capturing logger that records `(level;ctx;msg)` so log behaviour can be asserted. On-disk behaviour (sort, attributes, `savedown`/`appenddown`) is exercised against real splayed partitions and cleaned up afterwards. It covers: dependency-injection validation; `readcsv` returns / column-order independence / header-validation failures; `sort` validation / edge cases / resolution / on-disk results / multi-dir / non-fatal partition failure; `savedown` write+sort (default and explicit config, and a table without `sym`); `appenddown` append-without-sort then explicit sort, and the non-existent-partition error; `applyattr`; `gc`; and the logging contract.
+The suite injects mock loggers: a no-op logger, and a capturing logger that records `(level;ctx;msg)` so log behaviour can be asserted. On-disk behaviour (sort, attributes, `savedown`/`appenddown`) is exercised against real splayed partitions and cleaned up afterwards. It covers: dependency-injection validation; `readcsv` stores-and-verifies / column-order independence / header-validation failures; `setconfig` happy path and validation failures; `sort` edge cases / resolution / on-disk results / multi-dir / non-fatal partition failure; `savedown` write+sort (default and explicit config, and a table without `sym`); `appenddown` append-without-sort then explicit sort, and the non-existent-partition error; `applyattr`; and the logging contract.
 
 ---
 
 ## Exported symbols
 
 ```q
-export:([init;readcsv;sort;applyattr;savedown;appenddown;gc])
+export:([init;readcsv;setconfig;getconfig;sort;applyattr;savedown;appenddown])
 ```

From 2a54956897cb0ed600e3a9f745528b1cf9b1ca59 Mon Sep 17 00:00:00 2001
From: Ruairi-wq2 <ruairi.conlon@dataintellect.com>
Date: Wed, 24 Jun 2026 10:40:18 +0100
Subject: [PATCH 15/15] accept string paths in readcsv; fix test csv quoting;
 clarify sort/applyattr docs

- readcsv now accepts string paths (10h) by converting via hsym `$, avoiding
  the manual hsym `$"..." dance; bare symbols still coerced as before
- fix malformed test csv: string path test line had unescaped double quotes
  causing '/ parse error when k4unit loaded the suite
- add three tests covering string path happy path
- add type comment to applyattr (dloc/colname/att were undocumented)
- clarify sort comment: explicit that att assignments from config are applied

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .gitignore            |  1 +
 di/dbwrite/dbwrite.md |  3 ++-
 di/dbwrite/dbwrite.q  | 12 ++++++++----
 di/dbwrite/test.csv   |  5 ++++-
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 1377554e..54399fca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 *.swp
+di/dbwrite/sortconfig.csv
diff --git a/di/dbwrite/dbwrite.md b/di/dbwrite/dbwrite.md
index 5052a14c..c4ab8fc6 100644
--- a/di/dbwrite/dbwrite.md
+++ b/di/dbwrite/dbwrite.md
@@ -87,12 +87,13 @@ Read a config CSV and store it in module state (equivalent to calling `setconfig
 
 | Parameter | Type | Description |
 |---|---|---|
-| `file` | symbol/hsym | Path to the CSV; coerced with `hsym`. Errors (`di.dbwrite:`) if not a symbol. |
+| `file` | symbol/hsym/string | Path to the CSV. Symbols are coerced with `hsym`; strings are converted via `hsym `$`. Errors (`di.dbwrite:`) if not a symbol or string. |
 
 The CSV is parsed field-by-field and fully validated before storing — it does **not** silently pad, truncate, or coerce malformed rows. A clear `di.dbwrite:` error is raised if: the header is not exactly `tabname,att,column,sort` (any order); any data row does not have exactly four fields; any `sort` value is not `0` or `1`; or any `att` value is not in `` ` `p`s`g`u ``. Column order is normalised to canonical.
 
 ```q
 dbwrite.readcsv `:config/sort.csv
+dbwrite.readcsv "config/sort.csv"
 ```
 
 ---
diff --git a/di/dbwrite/dbwrite.q b/di/dbwrite/dbwrite.q
index bcb21108..c948236d 100644
--- a/di/dbwrite/dbwrite.q
+++ b/di/dbwrite/dbwrite.q
@@ -26,9 +26,11 @@ init:{[deps]
 readcsv:{[file]
   / read a sort-config csv and store it in .z.m.sortconfig; used by sort and savedown
   / the csv must have the columns tabname,att,column,sort (in any order)
+  / file: hsym, bare symbol, or string path
+  if[10h=type file; file:hsym `$file];
+  if[-11h=type file; if[not ":" = first string file; file:hsym file]];
   if[not -11h=type file;
-    '"di.dbwrite: readcsv file must be a symbol, got type ",string type file];
-  file:hsym file;
+    '"di.dbwrite: readcsv file must be a symbol or string path, got type ",string type file];
   t:parsecsv @[readfile; file; readerr[file]];
   checkconfig t;
   .z.m.log[`info]["dbwrite: read ",(string count t)," sort config row(s) from ",string file];
@@ -105,8 +107,9 @@ checkconfig:{[t]
 
 sort:{[tabname;dirs]
   / sort and apply attributes to the on-disk partition dirs for one table using .z.m.sortconfig
-  / falls back to defaultparams if readcsv has not been called
-  / tabname: symbol table name; dirs: hsym or list of hsyms (partition directories)
+  / both the sort column order and the att assignments (p/s/g/u) are driven by the config
+  / falls back to defaultparams if config has not been set via readcsv or setconfig
+  / tabname: symbol; dirs: hsym or list of hsyms (partition directories e.g. from .Q.par)
   config:$[(::)~.z.m.sortconfig; defaultparams; .z.m.sortconfig];
   checkconfig config;
   if[not -11h=type tabname;
@@ -169,6 +172,7 @@ attrerr:{[dl;cn;at;e]
 
 applyattr:{[dloc;colname;att]
   / apply a single kdb+ attribute to an on-disk column; logs and swallows errors so a run continues
+  / dloc: hsym (partition directory e.g. `:hdb/2024.01.01/trade); colname: symbol; att: symbol (p|s|g|u or empty)
   / skip anything that is not a real attribute - covers the empty none-sentinel and any bad value
   if[not att in `p`s`g`u; :()];
   .z.m.log[`info]["dbwrite: applying ",string[att]," attr to ",string[colname]," in ",string dloc];
diff --git a/di/dbwrite/test.csv b/di/dbwrite/test.csv
index a5d0032d..90bc79a7 100644
--- a/di/dbwrite/test.csv
+++ b/di/dbwrite/test.csv
@@ -41,10 +41,13 @@ true,0,0,q,0=count dbwrite.getconfig[],1,1,empty csv produces empty stored confi
 run,0,0,q,dbwrite.readcsv `:tmp_dbw_reorder.csv,1,1,readcsv with reordered columns stores correct config
 true,0,0,q,(dbwrite.getconfig[])~([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,readcsv is column-order independent
 true,0,0,q,`tabname`att`column`sort~cols dbwrite.getconfig[],1,1,readcsv normalises reordered columns to canonical order
+run,0,0,q,"dbwrite.readcsv ""tmp_dbw_valid.csv""",1,1,readcsv accepts a string file path
+true,0,0,q,3=count dbwrite.getconfig[],1,1,string path readcsv stores the correct number of rows
+true,0,0,q,(dbwrite.getconfig[])~([] tabname:`trade`trade`default; att:`p``p; column:`sym`time`sym; sort:101b),1,1,string path readcsv stores the correct config
 
 comment,,,,,,,readcsv - input and header validation failures
 fail,0,0,q,dbwrite.readcsv `:nonexistent_file.csv,1,1,readcsv errors on a missing file
-fail,0,0,q,dbwrite.readcsv 42,1,1,readcsv errors on a non-symbol file argument
+fail,0,0,q,dbwrite.readcsv 42,1,1,readcsv errors on a non-symbol non-string file argument
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_badcols.csv,1,1,readcsv rejects a csv with wrong column names
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_3col.csv,1,1,readcsv rejects a csv missing a required column
 fail,0,0,q,dbwrite.readcsv `:tmp_dbw_5col.csv,1,1,readcsv rejects a csv with an extra column