From e2801719cd63f8f4ff691539d1088db0e1798be9 Mon Sep 17 00:00:00 2001 From: David Stewart Date: Thu, 22 Jul 2021 17:24:25 +0100 Subject: [PATCH 1/2] add front end async postback function --- src/datasource.ts | 7 ++++++- src/model/kdb-request.ts | 4 ++++ src/model/query-param.ts | 1 + src/partials/query.editor.html | 34 ++++++++++++++++++++++++++++++++++ src/query_ctrl.ts | 4 ++++ 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/datasource.ts b/src/datasource.ts index 4fc9bd9a..53a96202 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -217,7 +217,9 @@ export class KDBDatasource { queryParam.column = this.buildColumnParams(target); queryParam.temporal_field = target.useTemporalField ? this.buildTemporalField(target) : []; queryParam.temporal_range = this.buildTemporalRange(target.range); - queryParam.maxRowCount = target.rowCountLimit + queryParam.maxRowCount = target.rowCountLimit; + queryParam.postbackFunction = target.postbackFunction; + if (target.queryType == 'selectQuery') queryParam.where = this.buildWhereParams(target.where); //conflation if (target.useConflation) { @@ -249,6 +251,9 @@ export class KDBDatasource { kdbRequest.format = target.format; kdbRequest.queryId = target.queryId; kdbRequest.version = target.version; + kdbRequest.useAsyncFunction = target.useAsyncFunction; + kdbRequest.useCustomPostback = target.useCustomPostback; + kdbRequest.asyncProcTypes = target.asyncProcTypes; return [ ((target.format == 'time series') ? graphFunction : tabFunction), diff --git a/src/model/kdb-request.ts b/src/model/kdb-request.ts index 82cf152e..851d35c8 100644 --- a/src/model/kdb-request.ts +++ b/src/model/kdb-request.ts @@ -9,4 +9,8 @@ export class KdbRequest { queryParam: QueryParam; format: string; version: string; + useAsyncFunction: boolean; + useCustomPostback: boolean; + asyncProcTypes: string; + } \ No newline at end of file diff --git a/src/model/query-param.ts b/src/model/query-param.ts index 7c2a2e81..e5db86ad 100644 --- a/src/model/query-param.ts +++ b/src/model/query-param.ts @@ -19,4 +19,5 @@ export class QueryParam { conflation: any; query: QueryDictionary; maxRowCount: number | string; + postbackFunction: string; } \ No newline at end of file diff --git a/src/partials/query.editor.html b/src/partials/query.editor.html index 477c2b27..59a89e7f 100644 --- a/src/partials/query.editor.html +++ b/src/partials/query.editor.html @@ -211,6 +211,40 @@ + +
+
+ +
+
+ Proc Types: + +
+
+ +
+
+
+
+
+
+
+ +
+
+
+
+
+
diff --git a/src/query_ctrl.ts b/src/query_ctrl.ts index b6f61baa..97bb99d6 100644 --- a/src/query_ctrl.ts +++ b/src/query_ctrl.ts @@ -772,4 +772,8 @@ export class KDBQueryCtrl extends QueryCtrl { this.panelCtrl.refresh(); } + asyncFieldChanged() { + this.panelCtrl.refresh(); + } + } From 57dd4711bbc9af3d07a4b485850aa26bf7ec2294 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 10 Oct 2021 23:07:28 +0100 Subject: [PATCH 2/2] DO NOT MERGE - proof of concept of join across gateway/.gw.asyncexecjpt working --- src/datasource.ts | 47 +++++++++++++++++++++----- src/model/kdb-request-config.ts | 58 ++++++++++++++++++++------------- src/model/kdb-request.ts | 5 +-- src/query_ctrl.ts | 1 + 4 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/datasource.ts b/src/datasource.ts index 53a96202..81a58000 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -8,9 +8,9 @@ import { KdbRequest } from "./model/kdb-request"; import { QueryParam } from "./model/query-param"; import { QueryDictionary } from "./model/queryDictionary"; import { ConflationParams } from "./model/conflationParams"; -import { graphFunction } from './model/kdb-request-config'; +import { graphFunction,graphFormatFunc, tableFormatFunc } from './model/kdb-request-config'; import { conflationDurationDefault, conflationUnitDefault } from './query_ctrl'; -import { tabFunction,defaultTimeout,kdbEpoch,durationMap } from './model/kdb-request-config'; +import { tabFunction,defaultTimeout,kdbEpoch,durationMap,defaultPostback } from './model/kdb-request-config'; export class KDBDatasource { //This is declaring the types of each member id: any; @@ -69,6 +69,10 @@ export class KDBDatasource { private variablesReplace(target:any, search: string, replace:any) { //Format Options as array or scalar // console.log('VARIABLESREPLACE TARGET: ', target) + // if target doesn't have queryType as a key, it is not initialised yet, so skip injection + if (!target.queryType) { + return + } if (Array.isArray(replace)) { target.kdbFunction = target.kdbFunction.replace(search, replace.join(',')) } else { @@ -249,11 +253,14 @@ export class KDBDatasource { kdbRequest.query = ''//query; kdbRequest.queryParam = Object.assign({}, queryParam); kdbRequest.format = target.format; + kdbRequest.formatFunc = target.format == 'time series' ? graphFormatFunc : tableFormatFunc; kdbRequest.queryId = target.queryId; kdbRequest.version = target.version; - kdbRequest.useAsyncFunction = target.useAsyncFunction; - kdbRequest.useCustomPostback = target.useCustomPostback; - kdbRequest.asyncProcTypes = target.asyncProcTypes; + kdbRequest.useAsyncFunction = (typeof target.useAsyncFunction === 'undefined') ? false : target.useAsyncFunction; + kdbRequest.useCustomPostback = (typeof target.useCustomPostback === 'undefined') ? false : target.useCustomPostback; + kdbRequest.asyncProcTypes = (typeof target.asyncProcTypes === 'undefined') ? '' : target.asyncProcTypes.split("`").splice(1).map(proc => "`"+proc); + //kdbRequest.asyncPostbackFunction = (typeof target.useCustomPostback === 'undefined') ? '' : target.useCustomPostback ? defaultPostback : target.postbackFunction; + kdbRequest.asyncPostbackFunction = kdbRequest.useCustomPostback ? target.postbackFunction : defaultPostback; return [ ((target.format == 'time series') ? graphFunction : tabFunction), @@ -558,6 +565,8 @@ export class KDBDatasource { //Response parser called here********************** private getQueryResult = (request: any): Promise => { let curRequest = request; + console.log('getQueryResult REQUEST:') + console.log(request) let timeoutError = "Query sent at " + new Date() + " timed out."; let malformedResError = "Malformed response. Check KDB+ WebSocket handler is correctly configured." let response = new Promise(resolve => { @@ -577,6 +586,7 @@ export class KDBDatasource { } connectWS() { + console.log('connectWS'); return new Promise (connected => { this.ws = new WebSocket(this.wsUrl); this.ws.binaryType = 'arraybuffer'; @@ -585,6 +595,7 @@ export class KDBDatasource { }; this.ws.onopen = () => { + console.log('open WS') connected(true); } @@ -608,9 +619,15 @@ export class KDBDatasource { executeAsyncQuery(request: any) { var requestResolve; let _c = this.c; + console.log('executeAsyncQuery: REQUEST:') + console.log(request); var requestPromise = new Promise(resolve => { let refIDn = Math.round(10000000 * Math.random()); - var wrappedRequest = {i:request, ID:refIDn}; + let formatter: string = typeof(request) == 'string' ? '{[x;y] :x}' : request[1].formatFunc; + let asyncbool:boolean = typeof(request) == 'string' ? false : request[1].useAsyncFunction; + var wrappedRequest = {async: asyncbool, formatFunc:formatter, i:request, ID:refIDn}; + console.log('WRAPPED REQUEST: ') + console.log(wrappedRequest) this.ws.send(_c.serialize(wrappedRequest)); this.requestSentIDList.push(refIDn); requestResolve = resolve; @@ -624,11 +641,20 @@ export class KDBDatasource { executeAsyncReceive(responseObj) { let _c = this.c; + console.log('executeAsyncReceive: RESPONSE:') + console.log(responseObj); let deserializedResult = _c.deserialize(responseObj.data); + if (Array.isArray(deserializedResult)) { + if (deserializedResult.length !== 3) { + return console.log('received malformed data (array)') + } + //deserializedResult = { ...deserializedResult[0], ...deserializedResult[2] } + deserializedResult = { ID: deserializedResult[0], o:deserializedResult[2].o} + }; if (!deserializedResult.ID) { - // return console.log('received malformed data') + return console.log('received malformed data') } else if (this.requestSentIDList.indexOf(deserializedResult.ID) === -1) { - // return console.log('received unrequested data'); + return console.log('received unrequested data'); } else { var requestNum = this.requestSentIDList.indexOf(deserializedResult.ID); this.requestSentList[requestNum].resolve(deserializedResult.o); @@ -703,6 +729,7 @@ export class KDBDatasource { //This is the function called by Grafana when it is testing a connection on the configuration page testDatasource() { + console.log('testDatasource') return this.connect() .then((result) => { return result; @@ -710,6 +737,7 @@ export class KDBDatasource { }; connect(): Promise { + console.log('connect') return new Promise((resolve, reject) => { if ("WebSocket" in window) { this.$q.when(this.setupWebSocket()).then(setTimeout(() => { @@ -726,6 +754,7 @@ export class KDBDatasource { //This checks the kdb+ connection state for the 'test connection' funciton checkConnectionState(): Promise { + console.log('checkconnectionstate') return new Promise(resolve => { return this.connectWS().then(connectStatus => { if (connectStatus === false) { @@ -738,7 +767,9 @@ export class KDBDatasource { }, this.timeoutLength) }); let response = new Promise(resolve => { + console.log('running .z.ws query') this.executeAsyncQuery('.z.ws').then(res => { + console.log('res: ',res) if (typeof res !== 'string') { resolve(this.buildResponse('fail', 'Malformed response. Check KDB+ WebSocket handler is correctly configured.', 'Fail')); } else if (res.replace(' ', '').includes('ds:-9!x;')) { diff --git a/src/model/kdb-request-config.ts b/src/model/kdb-request-config.ts index 703e30bb..0e5161a1 100644 --- a/src/model/kdb-request-config.ts +++ b/src/model/kdb-request-config.ts @@ -11,10 +11,43 @@ export const durationMap = { h: 3600 * Math.pow(10,9) } +export const defaultPostback : string = '{[query;formatter;id] .dg.dev.r:.gw.asyncexecjpt[.dg.dev.query:@[query;0;value];.dg.dev.proctypes:query[1;`asyncProcTypes],();{[res;id;fmt] -8!`o`ID!(fmt[raze res;""];id)}[;id;formatter];.dg.dev.pb:();.dg.dev.timeout:0Wn]}' + +export const callbackHandler: string = '.z.ws:{[x] \n ' + +' ds:-9!x; \n '+ +' fmtter:@[value;ds[`formatFunc];{`$"\'",x}]; \n ' + +' r:$[ds`async; \n ' + +' @[value;ds[`i;1;`asyncPostbackFunction];{`$"\'",x}][ds[`i];fmtter;ds`ID]; \n ' + +' `o`ID!(fmtter[;ds`queryId] @[value;ds[`i];{`$"\'",x}];ds[`ID])]; \n ' + +' if[not ds`async;neg[.z.w] -8! r] \n '+ +' }'; + +// .z.ws:{[x] ds:-9!x;fmtter:@[value;ds[`formatFunc];{`$"'",x}];r:$[ds`async;@[value;ds[`i;1;`asyncPostbackFunction];{`$"'",x}][ds[`i];fmtter;ds`ID];`o`ID!(fmtter[;ds`queryId] @[value;ds[`i];{`$"'",x}];ds[`ID])];if[not ds`async;neg[.z.w] -8! r]} + +export const graphFormatFunc: string = '{ \n ' + +` .[x;(y;z);{'\"Function:format - Error:\",x}]}[{[x;gfid] \n ` + +' \n ' + +' if[not .Q.qt x;:x];\n ' + +' t:$[99h=type x; \n ' + +' key[x]!([]data:flip each value x); \n ' + +' ([id:1#`x]data:`#enlist x)]; \n ' + +' :`payload`id`error`success!(t;gfid;$[(99h=type t)and(0