Merge branch 'main' into calumgrant/qlql

Author: calumgrant

javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml

@@ -1,6 +1,6 @@
 name: codeql/javascript-experimental-atm-lib
 description: CodeQL libraries for the experimental ML-powered queries
-version: 0.4.8
+version: 0.4.9
 extractor: javascript
 library: true
 groups:

javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml

@@ -1,7 +1,7 @@
 name: codeql/javascript-experimental-atm-queries
 description: Experimental ML-powered queries for JavaScript
 language: javascript
-version: 0.4.8
+version: 0.4.9
 suites: codeql-suites
 defaultSuiteFile: codeql-suites/javascript-atm-code-scanning.qls
 groups:

ql/ql/src/codeql_ql/StructuredLogs.qll

@@ -271,6 +271,10 @@ module KindPredicatesLog {
 
     float getResultSize() { result = this.getFloat("resultSize") }
 
+    Array getRA(string ordering) { result = this.getObject("ra").getArray(ordering) }
+
+    string getAnOrdering() { exists(this.getRA(result)) }
+
     override string toString() {
       if exists(this.getPredicateName())
       then result = this.getPredicateName()
@@ -335,8 +339,12 @@ module KindPredicatesLog {
       raExpr.getLine() = i
     }
 
+    Array getCounts() { result = this.getArray("counts") }
+
     float getCount(int i) { result = getRanked(this.getArray("counts"), i) }
 
+    Array getDuplicationPercentage() { result = this.getArray("duplicationPercentages") }
+
     float getDuplicationPercentage(int i) {
       result = getRanked(this.getArray("duplicationPercentages"), i)
     }
@@ -374,6 +382,123 @@ module KindPredicatesLog {
     string getPosition() { result = this.getString("position") }
 
     Predicate getPredicate() { result = getPredicateFromPosition(this.getPosition()) }
+
+    /**
+     * Gets the RA for this event. Unlike recursive predicates, a COMPUTE_SIMPLE
+     * event only has one pipeline ordering (and it's named "pipeline").
+     */
+    PipeLine getPipeLine() { result = this.getObject("ra").getArray("pipeline") }
+  }
+
+  /** Gets the `index`'th event that's evaluated by `recursive`. */
+  private InLayer layerEventRank(ComputeRecursive recursive, int index) {
+    result =
+      rank[index + 1](InLayer cand, int startline, string filepath |
+        cand.getComputeRecursiveEvent() = recursive and
+        cand.hasLocationInfo(filepath, startline, _, _, _)
+      |
+        cand order by filepath, startline
+      )
+  }
+
+  /**
+   * Gets the first predicate that's evaluated in an iteration
+   * of the SCC computation rooted at `recursive`.
+   */
+  private InLayer firstPredicate(ComputeRecursive recursive) {
+    result = layerEventRank(recursive, 0)
+  }
+
+  /**
+   * Gets the last predicate that's evaluated in an iteration
+   * of the SCC computation rooted at `recursive`.
+   */
+  private InLayer lastPredicate(ComputeRecursive recursive) {
+    exists(int n |
+      result = layerEventRank(recursive, n) and
+      not exists(layerEventRank(recursive, n + 1))
+    )
+  }
+
+  /**
+   * Holds if the predicate represented by `next` was evaluated after the
+   * predicate represented by `prev` in the SCC computation rooted at `recursive`.
+   */
+  predicate successor(ComputeRecursive recursive, InLayer prev, InLayer next) {
+    exists(int index |
+      layerEventRank(recursive, index) = prev and
+      layerEventRank(recursive, index + 1) = next
+    )
+  }
+
+  bindingset[this]
+  signature class ResultSig;
+
+  /**
+   * A signature for generically traversing a SCC computation.
+   */
+  signature module Fold<ResultSig R> {
+    /**
+     * Gets the base case for the fold. That is, the initial value that
+     * is produced from the first evaluation of the first IN_LAYER event
+     * in the recursive evaluation.
+     */
+    bindingset[run]
+    R base(PipeLineRun run);
+
+    /**
+     * Gets the recursive case for the fold. That is, `r` is the accumulation
+     * of the previous evaluations, and `run` is the pipeline of the next IN_LAYER
+     * event that is evaluated.
+     */
+    bindingset[run, r]
+    R fold(PipeLineRun run, R r);
+  }
+
+  module Iterate<ResultSig R, Fold<R> F> {
+    private R iterate(ComputeRecursive recursive, int iteration, InLayer pred) {
+      // Case: The first iteration
+      iteration = 0 and
+      (
+        // Subcase: The first predicate in the first iteration
+        pred = firstPredicate(recursive) and
+        result = F::base(pred.getPipelineRuns().getRun(0))
+        or
+        // Subcase: The predicate has a predecessor
+        exists(InLayer pred0, R r |
+          successor(recursive, pred0, pred) and
+          r = iterate(recursive, 0, pred0) and
+          result = F::fold(pred.getPipelineRuns().getRun(0), r)
+        )
+      )
+      or
+      // Case: Not the first iteration
+      iteration > 0 and
+      (
+        // Subcase: The first predicate in the iteration
+        pred = firstPredicate(recursive) and
+        exists(InLayer last, R r |
+          last = lastPredicate(recursive) and
+          r = iterate(recursive, iteration - 1, last) and
+          result = F::fold(pred.getPipelineRuns().getRun(iteration), r)
+        )
+        or
+        // Subcase: The predicate has a predecessor in the same iteration
+        exists(InLayer pred0, R r |
+          successor(recursive, pred0, pred) and
+          r = iterate(recursive, iteration, pred0) and
+          result = F::fold(pred.getPipelineRuns().getRun(iteration), r)
+        )
+      )
+    }
+
+    R iterate(ComputeRecursive recursive) {
+      exists(int iteration, InLayer pred |
+        pred = lastPredicate(recursive) and
+        result = iterate(recursive, iteration, pred) and
+        not exists(iterate(recursive, iteration + 1, pred))
+      )
+    }
   }
 
   class ComputeRecursive extends SummaryEvent {
@@ -382,6 +507,20 @@ module KindPredicatesLog {
 
   class InLayer extends SummaryEvent {
     InLayer() { evaluationStrategy = "IN_LAYER" }
+
+    string getMainHash() { result = this.getString("mainHash") }
+
+    ComputeRecursive getComputeRecursiveEvent() { result.getRAHash() = this.getMainHash() }
+
+    Array getPredicateIterationMillis() { result = this.getArray("predicateIterationMillis") }
+
+    float getPredicateIterationMillis(int i) {
+      result = getRanked(this.getArray("predicateIterationMillis"), i)
+    }
+
+    PipeLineRuns getPipelineRuns() { result = this.getArray("pipelineRuns") }
+
+    float getDeltaSize(int i) { result = getRanked(this.getArray("deltaSizes"), i) }
   }
 
   class ComputedExtensional extends SummaryEvent {

ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll

@@ -1866,6 +1866,12 @@ module JSON {
     /** Gets the location of this element. */
     final L::Location getLocation() { json_ast_node_info(this, _, _, result) }
 
+    predicate hasLocationInfo(
+      string filepath, int startline, int startcolumn, int endline, int endcolumn
+    ) {
+      this.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+    }
+
     /** Gets the parent of this element. */
     final AstNode getParent() { json_ast_node_info(this, result, _, _) }
 

ql/ql/src/queries/performance/LargeJoinOrder.ql

@@ -16,14 +16,37 @@ import KindPredicatesLog
 float getBadness(ComputeSimple simple) {
   exists(float maxTupleCount, float resultSize, float largestDependency, float denom |
     resultSize = simple.getResultSize() and
-    maxTupleCount = max(simple.getPipelineRun().getCount(_)) and
+    extractInformation(simple, maxTupleCount, _, _, _) and
     largestDependency = max(simple.getDependencies().getADependency().getResultSize()) and
     denom = resultSize.maximum(largestDependency) and
     denom > 0 and // avoid division by zero (which would create a NaN result).
     result = maxTupleCount / denom
   )
 }
 
-from ComputeSimple evt, float f
-where f = getBadness(evt) and f > 1.5
-select evt, f order by f desc
+pragma[nomagic]
+predicate hasPipelineRun(ComputeSimple simple, PipeLineRun run) { run = simple.getPipelineRun() }
+
+predicate extractInformation(
+  ComputeSimple simple, float maxTupleCount, Array tuples, Array duplicationPercentages, Array ra
+) {
+  exists(PipeLineRun run |
+    hasPipelineRun(simple, run) and
+    maxTupleCount = max(run.getCount(_)) and
+    tuples = run.getCounts() and
+    duplicationPercentages = run.getDuplicationPercentage() and
+    ra = simple.getRA()
+  )
+}
+
+from
+  ComputeSimple evt, float badness, float maxTupleCount, Array tuples, Array duplicationPercentages,
+  Array ra, int index
+where
+  badness = getBadness(evt) and
+  badness > 1.5 and
+  extractInformation(evt, maxTupleCount, tuples, duplicationPercentages, ra)
+select evt.getPredicateName() as predicate_name, badness, index,
+  tuples.getFloat(index) as tuple_count,
+  duplicationPercentages.getFloat(index) as duplication_percentage, ra.getString(index) order by
+    badness desc

ql/ql/src/queries/performance/LargeTupleSum.ql

@@ -0,0 +1,28 @@
+/**
+ * Shows a list of the "slow" predicates by tuple sum.
+ */
+
+import ql
+import codeql_ql.StructuredLogs
+import KindPredicatesLog
+
+module SumCounts implements Fold<int> {
+  int base(PipeLineRun run) { result = sum(int i | | run.getCount(i)) }
+
+  bindingset[s]
+  int fold(PipeLineRun run, int s) { result = sum(int i | | run.getCount(i)) + s }
+}
+
+int sumTuples(SummaryEvent event) {
+  result = strictsum(int i | | event.(ComputeSimple).getPipelineRun().getCount(i))
+  or
+  result = Iterate<int, SumCounts>::iterate(event)
+}
+
+int predicateRank(SummaryEvent evt) {
+  evt = rank[result](SummaryEvent y, int s | s = sumTuples(y) | y order by s desc)
+}
+
+from SummaryEvent evt, int s
+where predicateRank(evt) < 50 and s = sumTuples(evt)
+select evt, s order by s desc