diff --git a/di/serverselect/init.q b/di/serverselect/init.q new file mode 100644 index 00000000..135b1e20 --- /dev/null +++ b/di/serverselect/init.q @@ -0,0 +1,10 @@ +\l ::serverselect.q + +/ logging is an injected dependency: the start-up script that wires the modules together - +/ or the user at run time - must call init with a required `log dependency before using the +/ module. kx.log is intentionally NOT loaded here. +/ note: the injected log dict is stored in .z.m.log and called as .z.m.log[`info][`ctx;"msg"] + +export:([init; + addserverfull;addserverattr;addserver;setserveractive;getserverstable;addserversfromtable; + getservers;selector;getserverbytype;gethandlebytype;gethpbytype;getserverids]) diff --git a/di/serverselect/integration.q b/di/serverselect/integration.q new file mode 100644 index 00000000..1975277a --- /dev/null +++ b/di/serverselect/integration.q @@ -0,0 +1,271 @@ +/ =========================================================================== +/ di.serverselect integration test (real processes) +/ Spawns child q listeners, opens REAL IPC handles to them, and drives every +/ exported function through a realistic gateway lifecycle - register -> +/ query -> select -> route a live query to the chosen handle -> bulk-register +/ -> disconnect -> error handling -> init. Unlike a simulated test, the +/ selection functions hand back handles that are actually queried, so the +/ assertions prove queries reach the expected backend process. +/ . +/ Self-contained: no helper files. The child backends are bare `q` listeners +/ whose identity (`whoami`) and `ping` api are injected over IPC at setup; the +/ test launches and tears them down itself. +/ . +/ Requirements: +/ - run THIS script with KDB-X q (needs `use`) +/ - a q on PATH (override with $QBIN) to launch the child listeners +/ - some free TCP ports: the test scans for free ports from a pid-derived +/ base (so concurrent runs don't collide); set $SSPORT to pin the base +/ Run (QPATH must include this repo and the kx module dir): +/ QPATH=/path/to/kx/mod:/path/to/kdbx-modules /q integration.q +/ Exits with a non-zero code equal to the number of failed assertions. +/ =========================================================================== + +srvsel:use`di.serverselect; + +/ ---- tiny assertion harness ---- +/ note: avoid `desc`/`exp`/`log` as names - they are reserved words in KDB-X +PASS:0; FAIL:0; +lbl:{[ok] $[ok;" ok | ";" FAIL | "]}; +chk:{[nm;got;expd] + ok:got~expd; + $[ok;PASS+:1;FAIL+:1]; + m:lbl[ok],nm; + if[not ok; m:m," || got=",(.Q.s1 got)," exp=",.Q.s1 expd]; + -1 m; + }; +errtok:`$"__ERRORED__"; +chkerr:{[nm;f] + r:@[f;();{[e](errtok;e)}]; + ok:(0count free; + '"could not find ",string[n]," free ports scanning up from ",string start]; + :n#free; + }; +ports:findfree[baseport;count names]; + +launch:{[port] + / start a bare q listener detached and capture its shell background pid ($!) - so every + / child can be reaped by exact pid later, whether or not we ever connect to it, and with + / no dependence on the qbin path (identity/api are injected over IPC once connected) + :"I"$first system qbin," -p ",string[port]," -q /dev/null 2>&1 & echo $!"; + }; + +connect:{[port] + / open a handle, retrying for up to ~10s while the child binds and accepts + hp:`$":localhost:",string port; + step:{[hp;h] if[not null h; :h]; system"sleep 0.2"; @[hopen;(hp;500);{0Ni}]}[hp]; + :50 step/ 0Ni; + }; + +teardown:{[] + / always-run cleanup. kill every launched child by its captured pid (exact). the pkill + / fallback is a last resort for the rare case a pid was never captured - keyed on our own + / unique port and -q flag (NOT the qbin path, which is configurable), so a custom qbin + / cannot break it. the [-]p bracket keeps the pattern from matching this very kill command. + live:pids where not null pids; + if[count live; @[system;"kill ",(" " sv string live)," 2>/dev/null; true";{}]]; + {@[system;"pkill -f \"[-]p ",string[x]," -q\" 2>/dev/null; true";{}]} each ports; + @[hclose;;{}] each handles where not null handles; + }; + +/ launch the fleet (capturing each child's shell pid for exact-pid teardown), then connect +-1 "selected free ports: ",.Q.s1 ports; +pids:launch each ports; +handles:connect each ports; +/ guarantee the fleet is reaped on ANY exit (normal, error-abort, or exit code) +.z.exit:{teardown[]}; +if[any null handles; + -2 "failed to connect to child processes on ports: ",(" " sv string ports where null handles); + exit 3]; +/ inject each backend's identity + ping api over IPC +{[h;nm] h "whoami:`",string nm; h "ping:{whoami}";}'[handles;names]; +-1 "spawned ",(string count handles)," backend processes; real handles: ",.Q.s1 handles; + +/ route: select a server of type t via strategy sel, then SEND ping[] over the chosen handle +route:{[t;sel] h:srvsel.gethandlebytype[t;sel]; $[()~h; `NONE; h "ping[]"]}; +/ map serverids -> their handles (to route attribute-path selections) +sidhandles:{[sids] exec handle from srvsel.getserverstable[] where serverid in sids}; +/ the gateway's host/port for the hpup column - its real (dynamically chosen) address +gwhp:`$":localhost:",string ports 4; + +/ test steps run at top level (a single enclosing function would exceed q's +/ per-function constant limit). chk/chkerr trap their own assertion failures; +/ .z.exit guarantees teardown regardless of how the script ends. + +/ init must be called before use (log is a required injected dependency, no default) +srvsel.init[enlist[`log]!enlist `info`warn`error!({[c;m]};{[c;m]};{[c;m]})]; + +/ ========================================================================= +hdr"STEP 1 initial empty state"; +chk["fresh module: server table empty";0=count srvsel.getserverstable[];1b]; +chk["schema columns correct";`serverid`handle`procname`servertype`hpup`active`lastp`hits`attributes;cols srvsel.getserverstable[]]; + +/ ========================================================================= +hdr"STEP 2 registration of live handles (addserver / addserverattr / addserverfull)"; +srvsel.addserver[handles 0;`rdb]; +srvsel.addserver[handles 1;`rdb]; +srvsel.addserverattr[handles 2;`hdb;`date`sym!((d1;d2);`A`B)]; +srvsel.addserverattr[handles 3;`hdb;`date`sym!((d2;d3);`B`C)]; +srvsel.addserverfull[handles 4;`gw1;`gateway;gwhp;(enlist`region)!enlist`EU]; +chk["five servers registered";5=count srvsel.getserverstable[];1b]; +chk["serverids autoincrement 1..5";1 2 3 4 5i;exec serverid from srvsel.getserverstable[]]; +chk["handles stored are the real open handles";handles til 5;exec handle from srvsel.getserverstable[]]; +chk["servertypes as registered";`rdb`rdb`hdb`hdb`gateway;exec servertype from srvsel.getserverstable[]]; +chk["addserverfull populated procname";`gw1;first exec procname from srvsel.getserverstable[] where servertype=`gateway]; +chk["addserverfull populated hpup";gwhp;first exec hpup from srvsel.getserverstable[] where servertype=`gateway]; +chk["addserver -> empty attributes";(()!());first exec attributes from srvsel.getserverstable[] where handle=handles 0]; +chk["all active by default";1b;all exec active from srvsel.getserverstable[]]; + +/ ========================================================================= +hdr"STEP 3 getservers (lookup + attribute match scoring)"; +chk["lookups=` returns all active";5;count srvsel.getservers[`;`;()!()]]; +chk["servertype=hdb returns 2";2;count srvsel.getservers[`servertype;`hdb;()!()]]; +chk["servertype list rdb,hdb returns 4";4;count srvsel.getservers[`servertype;`rdb`hdb;()!()]]; +chk["procname=gw1 returns the gateway handle";enlist handles 4;exec handle from srvsel.getservers[`procname;`gw1;()!()]]; +chk["unknown servertype returns empty";0;count srvsel.getservers[`servertype;`nope;()!()]]; +chk["attribmatch column present";1b;`attribmatch in cols srvsel.getservers[`;`;()!()]]; +chk["complete date match flagged for some hdb on d1";1b; + any {x[`date]0} each (srvsel.getservers[`servertype;`hdb;(enlist`date)!enlist enlist d1])`attribmatch]; + +/ ========================================================================= +hdr"STEP 4 selector (strategies on a known table)"; +seltab:([]serverid:101 102 103i;handle:201 202 203i;lastp:(2024.01.03D0;2024.01.01D0;2024.01.02D0)); +chk["roundrobin picks oldest lastp";202i;(srvsel.selector[seltab;`roundrobin])`handle]; +chk["last picks newest lastp";201i;(srvsel.selector[seltab;`last])`handle]; +chk["any picks a row from the table";1b;((srvsel.selector[seltab;`any])`handle) in 201 202 203i]; +empt:([]serverid:`int$();handle:`int$();lastp:`timestamp$()); +chk["empty table -> null handle";0Ni;(srvsel.selector[empt;`roundrobin])`handle]; + +/ ========================================================================= +hdr"STEP 5 live routing (gethandlebytype / gethpbytype / getserverbytype)"; +/ round-robin across the two live rdbs: both start with null lastp, so LRU +/ visits rdb1, rdb2, rdb1, rdb2 - and each query is answered by that process +resp:{[i] route[`rdb;`roundrobin]} each til 4; +chk["round-robin routes to live rdbs in LRU order";`rdb1`rdb2`rdb1`rdb2;resp]; +chk["only the two live rdbs ever answer";`rdb1`rdb2;asc distinct resp]; +chk["gethpbytype gateway returns its hpup";gwhp;srvsel.gethpbytype[`gateway;`roundrobin]]; +chk["getserverbytype returns requested column";`gw1;srvsel.getserverbytype[`gateway;`procname;`roundrobin]]; +chk["gethandlebytype unknown type -> ()";();srvsel.gethandlebytype[`nope;`roundrobin]]; +chk["a single hdb query is answered by an hdb";1b;(route[`hdb;`roundrobin]) in `hdb1`hdb2]; +hitsbefore:exec sum hits from srvsel.getserverstable[] where servertype=`gateway; +srvsel.gethandlebytype[`gateway;`roundrobin]; +chk["selection increments hits";1b;(hitsbefore+1)=exec sum hits from srvsel.getserverstable[] where servertype=`gateway]; + +/ ========================================================================= +hdr"STEP 6 getserverids - servertype (symbol) path"; +chk["rdb -> the two rdb serverids";1 2i;asc raze srvsel.getserverids[`rdb]]; +chk["rdb,hdb -> all four serverids";1 2 3 4i;asc raze srvsel.getserverids[`rdb`hdb]]; +srvsel.setserveractive[handles 0;0b]; srvsel.setserveractive[handles 1;0b]; +chkerr["all requested servertype inactive -> error";{srvsel.getserverids[`rdb]}]; +srvsel.setserveractive[handles 0;1b]; srvsel.setserveractive[handles 1;1b]; + +/ ========================================================================= +hdr"STEP 7 getserverids - attribute path + routing to the matched backend"; +/ hdb1 covers (d1,d2)/(A,B); hdb2 covers (d2,d3)/(B,C) +chk["date=d1 -> hdb1 only";enlist`hdb1; + distinct {x"ping[]"} each sidhandles raze srvsel.getserverids[enlist[`date]!enlist enlist d1]]; +chk["cross (d1,d2)x(A,B) -> hdb1 alone covers it";enlist`hdb1; + distinct {x"ping[]"} each sidhandles raze srvsel.getserverids[`date`sym!((d1;d2);`A`B)]]; +chk["independent over d1,d2,d3 + A,B,C -> both hdbs contribute";`hdb1`hdb2; + asc distinct {x"ping[]"} each sidhandles raze srvsel.getserverids[`date`sym`attributetype!((d1;d2;d3);`A`B`C;`independent)]]; +chk["servertype key scopes attribute filter to hdbs";3 4i;asc raze srvsel.getserverids[`servertype`date!(`hdb;enlist d2)]]; +chk["empty requirement dict -> all active serverids";1 2 3 4 5i;asc raze srvsel.getserverids[()!()]]; +chkerr["besteffort=0b impossible -> error";{srvsel.getserverids[`date`besteffort!(enlist 2099.01.01;0b)]}]; +chkerr["no server has requested attribute value -> error";{srvsel.getserverids[enlist[`date]!enlist enlist 2099.01.01]}]; + +/ ========================================================================= +hdr"STEP 8 addserversfromtable (bulk registration of live handles)"; +conntab:([]w:handles 5 6;proctype:`rdb`hdb;attributes:(()!();`date`sym!((d1;d3);`A`C))); +srvsel.addserversfromtable[`rdb`hdb;conntab]; +chk["bulk-registered handles 5,6 now present";2=count select from srvsel.getserverstable[] where handle in handles 5 6;1b]; +chk["bulk-registered rdb3 is reachable via its handle";`rdb3;(handles 5)"ping[]"]; +chk["bulk-registered rdb3 now appears in rdb serverids";1b;6i in raze srvsel.getserverids[`rdb]]; +/ skip-already-active: re-offering a live handle must not duplicate it +n0:count srvsel.getserverstable[]; +srvsel.addserversfromtable[`rdb;([]w:enlist handles 0;proctype:enlist`rdb;attributes:enlist()!())]; +chk["already-active handle skipped (no new row)";n0;count srvsel.getserverstable[]]; +/ (proctype filter, `ALL, and optional procname/hpup columns are covered in test.csv) + +/ ========================================================================= +hdr"STEP 9 disconnect: kill a backend, deactivate it, prove routing adapts"; +/ a real gateway's .z.pc handler would call setserveractive[h;0b] on the dropped handle +/ guard: the kill is only a genuine test if rdb1's pid was captured. if pids 0 were null, +/ string 0Ni -> "0Ni" and the kill silently no-ops, degrading STEP 9 into a deactivation-only +/ test. fail loudly here rather than pass on a half-exercised path. +chk["rdb1 pid captured (so the kill below is real)";1b;not null pids 0]; +@[system;"kill ",string[pids 0]," 2>/dev/null; true";{}]; system"sleep 0.3"; +srvsel.setserveractive[handles 0;0b]; +/ active rdbs are now rdb2 (live) and rdb3 (live); the dead rdb1 must never be picked +postresp:{[i] route[`rdb;`roundrobin]} each til 6; +chk["killed/deactivated rdb1 is never selected";1b;not `rdb1 in postresp]; +chk["no routed query hits a dead handle (all answered)";1b;not `NONE in postresp]; +chk["the two remaining live rdbs both serve";`rdb2`rdb3;asc distinct postresp]; + +/ ========================================================================= +hdr"STEP 10 error handling (every guarded path signals + logs)"; +chkerr["addserverfull rejects non-int handle";{srvsel.addserverfull[`bad;`;`rdb;`;()!()]}]; +chkerr["addserverfull rejects non-symbol servertype";{srvsel.addserverfull[71i;`;"rdb";`;()!()]}]; +chkerr["addserver rejects bad handle";{srvsel.addserver[`bad;`rdb]}]; +chkerr["addserverattr rejects bad servertype";{srvsel.addserverattr[73i;"hdb";()!()]}]; +chkerr["setserveractive rejects non-int handle";{srvsel.setserveractive["bad";0b]}]; +chkerr["setserveractive rejects non-boolean flag";{srvsel.setserveractive[1i;`yes]}]; +chkerr["getservers rejects bad nameortype";{srvsel.getservers[`badname;`x;()!()]}]; +chkerr["selector rejects unknown strategy";{srvsel.selector[seltab;`bogus]}]; +chkerr["getserverids rejects null servertype";{srvsel.getserverids[`]}]; +chkerr["getserverids rejects unregistered servertype";{srvsel.getserverids[`nope]}]; +chkerr["getserverids rejects non-symbol non-dict arg";{srvsel.getserverids[42]}]; +chkerr["getserverids rejects nested/malformed attribute value";{srvsel.getserverids[(enlist`date)!enlist enlist d1,d2]}]; +chkerr["addserversfromtable rejects missing columns";{srvsel.addserversfromtable[`rdb;([]x:enlist 1i)]}]; + +/ ========================================================================= +hdr"STEP 11 init (required log dependency - bespoke + kx.log)"; +caplog:([]lvl:`symbol$();msg:()); +caplogger:`info`warn`error!({[c;m]`caplog upsert(`info;m)};{[c;m]`caplog upsert(`warn;m)};{[c;m]`caplog upsert(`error;m)}); +srvsel.init[enlist[`log]!enlist caplogger]; +caplog:0#caplog; +srvsel.addserver[40i;`rdb]; +chk["injected logger captures an info message";1b;0type deps; + '"di.serverselect: deps must be a dict with `log key"]; + if[not `log in key deps; + '"di.serverselect: log dependency is required; pass `info`warn`error functions keyed on `log"]; + if[99h<>type deps`log; + '"di.serverselect: log value must be a dict; pass `info`warn`error functions"]; + if[not all (`info`warn`error) in key deps`log; + '"di.serverselect: log dict must have `info`warn`error keys; got: ",(", " sv string key deps`log)]; + .z.m.log:normlog deps`log; + }; + +normlog:{[logdict] + / internal - normalise the injected logger to a binary `info`warn`error!{[c;m]} dict + / detect kx.log instance by presence of kx.log-specific keys (getlvl, sinks, fmts) + / kx.log functions are monadic - wrap each into binary {[c;m]} and embed context in the message + / plain {[c;m]} log dicts (info`warn`error only) pass through unchanged + $[any `getlvl`sinks`fmts in key logdict; + `info`warn`error!( + {[fn;c;m] fn[string[c],": ",m]}[logdict`info;]; + {[fn;c;m] fn[string[c],": ",m]}[logdict`warn;]; + {[fn;c;m] fn[string[c],": ",m]}[logdict`error;]); + logdict] + }; + +raiseerror:{[ctx;msg] + / internal - log an error under ctx then signal it, so failures are observable as well as thrown + .z.m.log[`error][ctx;msg]; + '"di.serverselect: ",string[ctx],": ",msg; + }; + +nextserverid:{ + / internal - increment and return the next unique server ID + .z.m.serverid:serverid+1i; + :.z.m.serverid; + }; + +updatestats:{[sid] + / internal - update last-access timestamp and hit count for a single server + / keyed on serverid (unique) rather than handle, which may be shared by multiple servers + .z.m.servers:update lastp:.z.p,hits:hits+1i from servers where serverid=sid; + }; + +addserverfull:{[h;pname;st;hp;att] + / register a server with full details: handle, procname, servertype, hpup and attribute dictionary + if[not -6h=type h;raiseerror[`addserverfull;"handle must be an int; got type ",string type h]]; + if[not -11h=type st;raiseerror[`addserverfull;"servertype must be a symbol; got type ",string type st]]; + .z.m.log[`info][`addserverfull;"registering server: handle=",(string h),", procname=",string[pname],", type=",string st]; + .z.m.servers:servers upsert (nextserverid[];h;pname;st;hp;1b;0Np;0i;att); + }; + +addserverattr:{[h;st;att] + / register a server with servertype and attributes; procname and hpup default to null + addserverfull[h;`;st;`;att]; + }; + +addserver:{[h;st] + / register a server with no attributes; procname and hpup default to null + addserverfull[h;`;st;`;()!()]; + }; + +setserveractive:{[h;a] + / mark a registered server active (1b) or inactive (0b); called on connect and disconnect + if[not -6h=type h;raiseerror[`setserveractive;"handle must be an int; got type ",string type h]]; + if[not -1h=type a;raiseerror[`setserveractive;"active flag must be a boolean; got type ",string type a]]; + .z.m.log[`info][`setserveractive;"marking handle=",(string h)," active=",string a]; + .z.m.servers:update active:a from servers where handle=h; + }; + +getserverstable:{[] + / return the current registered server table + :servers; + }; + +addserversfromtable:{[proctypes;conntable] + / register active servers from a connection table filtered by proctype + / conntable must have columns: w (int handle), proctype (symbol), attributes (dict per row) + / optional columns: procname (symbol), hpup (symbol) - populated from conntable if present + / pass proctypes:`ALL to register all process types + if[not all `w`proctype`attributes in cols conntable; + raiseerror[`addserversfromtable;"conntable must have columns w, proctype and attributes; got: ",", " sv string cols conntable]; + ]; + activehandles:(0i;0Ni),exec handle from servers where active; + rows:select from conntable where + ((proctype in proctypes) or proctypes~`ALL), + not w in activehandles; + .z.m.log[`info][`addserversfromtable;"registering ",(string count rows)," servers from connection table"]; + pnames:$[`procname in cols rows; rows`procname; count[rows]#`]; + hpups:$[`hpup in cols rows; rows`hpup; count[rows]#`]; + addserverfull'[rows`w;pnames;rows`proctype;hpups;rows`attributes]; + }; + +attributematch:{[req;avail] + / compute match result for each key in req against what avail advertises + / returns dict of attrname!(complete_match_bool;matched_values) for each required attribute key + / keys present in req but absent in avail return (0b;()) + vals:key[req] inter key avail; + notpresent:noval!(count noval:key[req] except key avail)#enlist(0b;()); + :notpresent,vals!{($[0>type y;x~y;all x in y];(x,()) inter y,())}'[req vals;avail vals]; + }; + +getservers:{[nameortype;lookups;req] + / look up active servers by servertype or procname with per-attribute match scoring + / nameortype: `servertype or `procname; pass ` as lookups to return all active servers + / req: attribute requirements dict - use ()!() for no attribute filtering + / returns table with attribmatch column showing (complete_bool;matched_values) per attribute key + r:$[`~lookups; + select serverid,procname,servertype,hpup,handle,lastp,attributes from servers where active; + nameortype~`servertype; + select serverid,procname,servertype,hpup,handle,lastp,attributes from servers where active,servertype in lookups; + nameortype~`procname; + select serverid,procname,servertype,hpup,handle,lastp,attributes from servers where active,procname in lookups; + raiseerror[`getservers;"nameortype must be `servertype or `procname; got: ",string nameortype]]; + if[0=count r;:update attribmatch:attributes from r]; + am:attributematch[req] each r`attributes; + :update attribmatch:am from r; + }; + +selector:{[servertable;selection] + / pick one row from servertable using the given strategy + / selection: `roundrobin (least recently used), `any (random), `last (most recently used) + :$[selection=`roundrobin; first `lastp xasc servertable; + selection=`any; rand servertable; + selection=`last; last `lastp xasc servertable; + raiseerror[`selector;"unknown selection strategy: ",string selection]]; + }; + +getserverbytype:{[ptype;serverval;selection] + / return a single server attribute value for a servertype using the given selection strategy + / ptype: servertype symbol; serverval: column to return e.g. `handle or `hpup; selection: `roundrobin`any`last + r:getservers[`servertype;ptype;()!()]; + if[not count r;:()]; + r:selector[r;selection]; + updatestats[r`serverid]; + :r serverval; + }; + +gethandlebytype:getserverbytype[;`handle;]; +gethpbytype:getserverbytype[;`hpup;]; + +getserverids:{[att] + / return server IDs matching a servertype list or attribute requirement dictionary + / att: symbol list of servertypes, or dict of attribute requirements (optionally keyed on `servertype) + / dispatch the symbol-list path to getserveridsbytype and the dict path to getserveridstype + if[99h<>type att; :getserveridsbytype att]; + / validate attribute-requirement values are atoms or simple vectors (control keys excepted); + / a nested/general-list value (type 0h) would otherwise crash the cross matcher with a raw 'type + reqkeys:key[att] except `servertype`attributetype`besteffort; + if[count badkeys:reqkeys where 0h=type each att reqkeys; + raiseerror[`getserverids;"attribute requirement values must be atoms or simple vectors; nested/mixed for: ",", " sv string badkeys]]; + serverids:$[`servertype in key att; + raze getserveridstype[delete servertype from att] each (),att`servertype; + getserveridstype[att;`all]]; + if[all 0=count each serverids; + raiseerror[`getserverids;"no servers match requested attributes"]; + ]; + :serverids; + }; + +getserveridsbytype:{[att] + / internal - resolve server IDs for a servertype symbol or symbol list + / validates each requested type is registered and currently active + if[not 11h=abs type att; + raiseerror[`getserveridsbytype;"servertype must be a symbol list (11h) or attribute dict (99h)"]; + ]; + servertype:distinct att,(); + activeservers:exec distinct servertype from servers where active; + allservers:exec distinct servertype from servers; + activeserversmsg:". available servers: ",", " sv string activeservers; + if[any null att; + raiseerror[`getserveridsbytype;"null servertype passed as argument",activeserversmsg]; + ]; + if[count servertype except activeservers; + raiseerror[`getserveridsbytype; + $[max not servertype in allservers; + "not valid servers: ",", " sv string servertype except allservers; + "requested servers currently inactive: ",", " sv string servertype except activeservers + ],activeserversmsg]; + ]; + :(exec serverid by servertype from servers where active)[servertype]; + }; + +getserveridstype:{[att;typ] + / internal - filter active servers by servertype and attribute requirements + / resolves besteffort and attributetype control keys then dispatches to cross or independent matching + besteffort:1b; + attype:`cross; + svrs:$[typ=`all; + exec serverid!attributes from servers where active; + exec serverid!attributes from servers where active,servertype=typ]; + if[`besteffort in key att; + if[-1h=type att`besteffort; besteffort:att`besteffort]; + att:delete besteffort from att; + ]; + if[`attributetype in key att; + if[-11h=type att`attributetype; attype:att`attributetype]; + att:delete attributetype from att; + ]; + res:$[attype=`independent; + getserversindependent[att;svrs;besteffort]; + getserverscross[att;svrs;besteffort]]; + serverids:first value flip $[99h=type res; key res; res]; + if[all 0=count each serverids; + raiseerror[`getserveridstype;"no servers match ",string[typ]," requested attributes"]; + ]; + :serverids; + }; + +/ internal - build a cross product table from a nested dictionary +buildcross:{(cross/){flip (enlist y)#x}[x] each key x}; + +getserversinitial:{[req;att] + / internal - initial filter shared by cross and independent matching + / drops servers missing any required attribute key, ranks survivors by coverage + if[0=count req; :([]serverid:enlist key att)]; + att:(where all each (key req) in/: key each att)#att; + if[not count att;raiseerror[`getserversinitial;"no servers report all requested attributes"]]; + s:update serverid:key att from value req in'/: (key req)#/:att; + s:s idesc value min each sum each' `serverid xkey s; + s:`serverid xkey 0!(key req) xgroup s; + :s; + }; + +getserverscross:{[req;att;besteffort] + / internal - find servers satisfying the cross product of all attribute requirements + / each attribute combination must be coverable by a single server + if[0=count req; :([]serverid:enlist key att)]; + s:getserversinitial[req;att]; + reqcross:buildcross[req]; + / scan through each server group accumulating which cross-product rows have been covered + util:flip `remaining`found!flip ( + {[x;y;z] (y[0] except found; y[0] inter found:$[0=count y[0];y[0];buildcross x@'where each z])}[req]\ + )[(reqcross;());value s]; + if[(count last util`remaining) and not besteffort; + raiseerror[`getserverscross;"cannot satisfy query - cross product of all attributes cannot be matched"]; + ]; + s:1!(0!s) w:where not 0=count each util`found; + :(key s)!distinct each' flip each util[w]`found; + }; + +getserversindependent:{[req;att;besteffort] + / internal - find servers satisfying attribute requirements independently + / each individual requirement only needs to be matched by one server + if[0=count req; :([]serverid:enlist key att)]; + s:getserversinitial[req;att]; + / mask out server groups whose contribution is already covered by earlier groups + filter:(value s)¬ -1 _ (0b&(value s) enlist 0),maxs value s; + alldone:1+first where all each all each' maxs value s; + if[(null alldone) and not besteffort; + raiseerror[`getserversindependent;"cannot satisfy query - not all attributes can be matched"]; + ]; + s:1!(0!s) w:where any each any each' filter; + :(key s)!{(key x)!(value x)@'where each y key x}[req] each value s&filter w; + }; diff --git a/di/serverselect/test.csv b/di/serverselect/test.csv new file mode 100644 index 00000000..1db409d5 --- /dev/null +++ b/di/serverselect/test.csv @@ -0,0 +1,194 @@ +action,ms,bytes,lang,code,repeat,minver,comment +before,0,0,q,srvsel:use`di.serverselect,1,1,load module +before,0,0,q,"noop:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,1,define a no-op binary logger {[c;m]} +before,0,0,q,srvsel.init[enlist[`log]!enlist noop],1,1,init the module with the required log dependency before use + +/ ---- getserverstable: initial state ---- +comment,,,,,,,getserverstable - initial state before any addserverattr +true,0,0,q,0=count srvsel.getserverstable[],1,1,servers empty on fresh module load +true,0,0,q,`serverid`handle`procname`servertype`hpup`active`lastp`hits`attributes~cols srvsel.getserverstable[],1,1,servers has correct schema + +/ ---- addserverattr / addserver / addserverfull: registration ---- +comment,,,,,,,addserverattr and addserver - server registration +run,0,0,q,srvsel.addserver[1i;`rdb],1,1,register rdb server with no attributes +run,0,0,q,srvsel.addserver[2i;`rdb],1,1,register second rdb server +run,0,0,q,"srvsel.addserverattr[3i;`hdb;`date`sym!(2024.01.01 2024.01.02;`AAPL`MSFT)]",1,1,register hdb server with date and sym attributes +true,0,0,q,3=count srvsel.getserverstable[],1,1,three servers registered +true,0,0,q,all exec active from srvsel.getserverstable[],1,1,all registered servers are active by default +true,0,0,q,`rdb`rdb`hdb~exec servertype from srvsel.getserverstable[],1,1,servertypes match registered order +true,0,0,q,(1i;2i;3i)~exec handle from srvsel.getserverstable[],1,1,handles match registered order +true,0,0,q,1 2 3i~exec serverid from srvsel.getserverstable[],1,1,server IDs autoincrement from 1 +true,0,0,q,all null exec procname from srvsel.getserverstable[],1,1,procname is null when registered via addserverattr +true,0,0,q,all null exec hpup from srvsel.getserverstable[],1,1,hpup is null when registered via addserverattr +run,0,0,q,"srvsel.addserverfull[4i;`myproc;`hdb;`:localhost:5555;(enlist`date)!enlist 2024.12.01]",1,1,register server with full details via addserverfull +true,0,0,q,4=count srvsel.getserverstable[],1,1,fourth server registered via addserverfull +true,0,0,q,`myproc~first exec procname from srvsel.getserverstable[] where handle=4i,1,1,procname populated via addserverfull +true,0,0,q,`:localhost:5555~first exec hpup from srvsel.getserverstable[] where handle=4i,1,1,hpup populated via addserverfull + +/ ---- setserveractive: marking active/inactive ---- +comment,,,,,,,setserveractive - active flag management +run,0,0,q,srvsel.setserveractive[2i;0b],1,1,mark second rdb server inactive +true,0,0,q,not (exec active from srvsel.getserverstable[] where handle=2i)[0],1,1,handle 2 is now inactive +true,0,0,q,(exec active from srvsel.getserverstable[] where handle=1i)[0],1,1,handle 1 remains active +run,0,0,q,srvsel.setserveractive[2i;1b],1,1,mark second rdb server active again +true,0,0,q,(exec active from srvsel.getserverstable[] where handle=2i)[0],1,1,handle 2 is active again + + +/ ---- getservers ---- +comment,,,,,,,getservers - server lookup with attribute match scoring +true,0,0,q,`serverid`procname`servertype`hpup`handle`lastp`attributes`attribmatch~cols srvsel.getservers[`;`;()!()],1,1,getservers returns table with correct columns +true,0,0,q,count[select from srvsel.getserverstable[] where active]=count srvsel.getservers[`;`;()!()],1,1,getservers with ` lookups returns all active servers +true,0,0,q,all `hdb=exec servertype from srvsel.getservers[`servertype;`hdb;()!()],1,1,getservers filters by servertype correctly +true,0,0,q,0=count srvsel.getservers[`servertype;`nonexistent;()!()],1,1,getservers returns empty table for unknown servertype +true,0,0,q,0