Merge pull request #276 from doitintl/feature/lior/partitonTable

Add partition and macros support at raw sql mode

Author: spark2ignite

README.md

@@ -144,6 +144,8 @@ The query builder provides a simple yet a user-friendly interface to help you qu
   WHERE `createDate` < TIMESTAMP_MILLIS (1592828659681)  AND _PARTITIONTIME >= '2020-06-15 15:24:19' AND _PARTITIONTIME < '2020-06-22 15:24:19'
 ```
 
+You can now use timeFilter macro in raw sql mode
+
 4. GROUP BY option - You can use a pre-defined macro or use one of the fields from your query
     a. time ($__interval,none)
 5. ORDER BY option

dist/README.md

@@ -144,6 +144,8 @@ The query builder provides a simple yet a user-friendly interface to help you qu
   WHERE `createDate` < TIMESTAMP_MILLIS (1592828659681)  AND _PARTITIONTIME >= '2020-06-15 15:24:19' AND _PARTITIONTIME < '2020-06-22 15:24:19'
 ```
 
+You can now use pre-defined macros in raw sql mode
+
 4. GROUP BY option - You can use a pre-defined macro or use one of the fields from your query
     a. time ($__interval,none)
 5. ORDER BY option

dist/module.js

@@ -93,7 +93,7 @@ export default class BigQueryQuery {
   public static replaceTimeShift(q) {
     return q.replace(/(\$__timeShifting\().*?(?=\))./g, "");
   }
-  private static convertToUtc(d) {
+  static convertToUtc(d) {
     return new Date(d.getTime() + d.getTimezoneOffset() * 60000);
   }
 
@@ -454,8 +454,16 @@ export default class BigQueryQuery {
           return tag.params.join(" ");
       }
     });
-    if (conditions.length > 0) {
-      query = "\nWHERE\n  " + conditions.join(" AND\n  ");
+    if (this.target.partitioned) {
+      const partitionedField = this.target.partitionedField ? this.target.partitionedField : "_PARTITIONTIME";
+      if (this.templateSrv.timeRange && this.templateSrv.timeRange.from) {
+        const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(this.templateSrv.timeRange.from._d, "-", true)}'`;
+        conditions.push(from);
+      }
+      if (this.templateSrv.timeRange && this.templateSrv.timeRange.to) {
+        const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(this.templateSrv.timeRange.to._d, "-", true)}'`;
+        conditions.push(to);
+      }
     }
     if (this.target.sharded) {
       const from = BigQueryQuery.formatDateToString(
@@ -464,23 +472,11 @@ export default class BigQueryQuery {
       const to = BigQueryQuery.formatDateToString(
         this.templateSrv.timeRange.to._d
       );
-      query += " AND  _TABLE_SUFFIX BETWEEN '" + from + "' AND '" + to + "' ";
-    }
-    if (this.target.partitioned && this.target.partitionedField === "") {
-      query +=
-        " AND _PARTITIONTIME >= '" +
-        BigQueryQuery.formatDateToString(
-          this.templateSrv.timeRange.from._d,
-          "-",
-          true
-        ) +
-        "' AND _PARTITIONTIME < '" +
-        BigQueryQuery.formatDateToString(
-          this.templateSrv.timeRange.to._d,
-          "-",
-          true
-        ) +
-        "'";
+      const sharded = "_TABLE_SUFFIX BETWEEN '" + from + "' AND '" + to + "' ";
+      conditions.push(sharded);
+    }
+    if (conditions.length > 0) {
+      query = "\nWHERE\n  " + conditions.join(" AND\n  ");
     }
     return query;
   }
@@ -657,7 +653,7 @@ export default class BigQueryQuery {
       BigQueryQuery.quoteFiledName(this.target.timeColumn) + " > " + from + " ";
     const toRange =
       BigQueryQuery.quoteFiledName(this.target.timeColumn) + " < " + to + " ";
-    q = q.replace(/\$__timeFilter\(([\w_.]+)\)/g, range);
+    q = q.replace(/\$__timeFilter\((.*?)\)/g, range);
     q = q.replace(/\$__timeFrom\(([\w_.]+)\)/g, fromRange);
     q = q.replace(/\$__timeTo\(([\w_.]+)\)/g, toRange);
     q = q.replace(/\$__millisTimeTo\(([\w_.]+)\)/g, to);

dist/module.js.map

@@ -228,9 +228,7 @@ export class BigQueryDatasource {
     this.authenticationType =
       instanceSettings.jsonData.authenticationType || "jwt";
     (async () => {
-      this.projectName =
-        instanceSettings.jsonData.defaultProject ||
-        (await this.getDefaultProject());
+      this.projectName = instanceSettings.jsonData.defaultProject || (await this.getDefaultProject());
     })();
     this.mixpanel = require("mixpanel-browser");
     if (this.jsonData.sendUsageData !== false) {
@@ -525,6 +523,7 @@ export class BigQueryDatasource {
   }
   private setUpQ(modOptions, options, query) {
     let q = this.queryModel.expend_macros(modOptions);
+    q = this.setUpPartition(q, query.partitioned, query.partitionedField, modOptions);
     q = BigQueryDatasource._updatePartition(q, modOptions);
     q = BigQueryDatasource._updateTableSuffix(q, modOptions);
     if (query.refId.search(Shifted) > -1) {
@@ -543,6 +542,31 @@ export class BigQueryDatasource {
     }
     return q;
   }
+  /**
+   * Add partition to query unless it has one
+   * @param query 
+   * @param isPartitioned 
+   * @param partitionedField 
+   * @param options 
+   */
+  private setUpPartition(query, isPartitioned, partitionedField, options){
+    partitionedField = partitionedField ? partitionedField : '_PARTITIONTIME';
+    if(isPartitioned && !query.match(partitionedField)) {
+      let fromD = BigQueryQuery.convertToUtc(options.range.from._d);
+      let toD = BigQueryQuery.convertToUtc(options.range.to._d);
+      const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(fromD, "-", true)}'`;
+      const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(toD, "-", true)}'`;
+      const partition = `where ${from} AND ${to} AND `;
+      if(query.match(/where/i))
+        query = query.replace(/where/i, partition);
+      else {
+        const reg = /from ('|`|"|){1}(.*?)('|`|"|){1} as ('|`|"|)(\S*)('|`|"|){1}|from ('|`|"|){1}(\S*)('|`|"|){1}/i;
+        const fromMatch = query.match(reg);
+        query = query.replace(reg, `${fromMatch} ${fromMatch}`);
+      }
+    }
+    return query;
+  }
   private async doRequest(url, requestId = "requestId", maxRetries = 3) {
     return this.backendSrv
       .datasourceRequest({

src/bigquery_query.ts

@@ -195,6 +195,12 @@ describe("BigQueryQuery", () => {
     expect(query.buildWhereClause()).toBe(
       "\nWHERE\n  $__timeFilter(t) AND\n  v = 1"
     );
+    query.target.where = [];
+    query.target.partitioned = true;
+    const time = { from: { _d: "1987-06-30" }, to: { _d: "1987-06-30" } };
+    query.templateSrv.timeRange = time;
+    const whereClause = query.buildWhereClause();
+    expect(whereClause).toBe("\nWHERE\n  _PARTITIONTIME >= '1987-06-30 03:00:00' AND\n  _PARTITIONTIME < '1987-06-30 03:00:00'");
   });
 
   describe("When generating GROUP BY clause", () => {