Histogram EventData group-by, spike-outline cleanup, and timeline-aware single-log sort

src/EventLogExpert.Eventing/Common/Events/EventColumnStore.cs

@@ -75,6 +75,9 @@ public int GetHashCode(int[] obj)
 /// </summary>
 public sealed class EventColumnStore
 {
+    // Per-scan EventData field-index memo sentinels: a schema not yet looked up, and a schema that lacks the field.
+    private const int SchemaFieldAbsent = -1;
+    private const int SchemaFieldUnresolved = -2;
     // Target sealed-chunk size. The pending tail columnarizes into one new chunk each time it reaches this many rows,
     // giving amortized O(1) append with no partially filled columnar chunks.
     private const int TargetChunkSize = 4096;
@@ -227,6 +230,54 @@ public bool TryGetTimeRange(out long minTicks, out long maxTicks)
         return Count > 0;
     }
 
+    internal void BucketTimeTicksByEventData(
+        ReadOnlySpan<int> rankByPhysical,
+        long minTicks,
+        long bucketSpanTicks,
+        int bucketCount,
+        string fieldName,
+        long[] targetCodes,
+        int slotCount,
+        int[] slotCounts,
+        CancellationToken cancellationToken)
+    {
+        int otherSlot = targetCodes.Length;
+        int[] fieldIndexBySchema = NewSchemaFieldMemo();
+        int offset = 0;
+
+        foreach (EventColumnChunk chunk in _sealedChunks)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            ReadOnlySpan<long> timeColumn = chunk.TimeTicksColumn;
+
+            for (int row = 0; row < timeColumn.Length; row++)
+            {
+                if (rankByPhysical[offset + row] < 0) { continue; }
+
+                int slot = TryGetSealedEventDataCode(chunk, row, fieldName, fieldIndexBySchema, out long code)
+                    ? SlotForCode(code, targetCodes, otherSlot)
+                    : otherSlot;
+                int bucket = ToBucket(timeColumn[row], minTicks, bucketSpanTicks, bucketCount);
+                slotCounts[(bucket * slotCount) + slot]++;
+            }
+
+            offset += chunk.RowCount;
+        }
+
+        for (int index = _sealedCount; index < Count; index++)
+        {
+            if (rankByPhysical[index] < 0) { continue; }
+
+            ResolvedEvent pending = Pending(index);
+            int slot = TryGetPendingEventDataCode(pending, fieldName, out long code)
+                ? SlotForCode(code, targetCodes, otherSlot)
+                : otherSlot;
+            int bucket = ToBucket(pending.TimeCreated.Ticks, minTicks, bucketSpanTicks, bucketCount);
+            slotCounts[(bucket * slotCount) + slot]++;
+        }
+    }
+
     internal void BucketTimeTicksByEventId(
         ReadOnlySpan<int> rankByPhysical,
         long minTicks,
@@ -546,6 +597,41 @@ internal void CopyTimeTicks(long[] values, bool[] hasValue)
         }
     }
 
+    internal void CountEventDataValues(ReadOnlySpan<int> rankByPhysical, string fieldName, IDictionary<long, int> counts, CancellationToken cancellationToken)
+    {
+        int[] fieldIndexBySchema = NewSchemaFieldMemo();
+        int offset = 0;
+
+        foreach (EventColumnChunk chunk in _sealedChunks)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            ReadOnlySpan<long> timeColumn = chunk.TimeTicksColumn;
+
+            for (int row = 0; row < timeColumn.Length; row++)
+            {
+                if (rankByPhysical[offset + row] < 0) { continue; }
+
+                if (TryGetSealedEventDataCode(chunk, row, fieldName, fieldIndexBySchema, out long code))
+                {
+                    counts[code] = counts.TryGetValue(code, out int existing) ? existing + 1 : 1;
+                }
+            }
+
+            offset += chunk.RowCount;
+        }
+
+        for (int index = _sealedCount; index < Count; index++)
+        {
+            if (rankByPhysical[index] < 0) { continue; }
+
+            if (TryGetPendingEventDataCode(Pending(index), fieldName, out long code))
+            {
+                counts[code] = counts.TryGetValue(code, out int existing) ? existing + 1 : 1;
+            }
+        }
+    }
+
     internal void CountEventIds(ReadOnlySpan<int> rankByPhysical, IDictionary<int, int> counts, CancellationToken cancellationToken)
     {
         int offset = 0;
@@ -1052,6 +1138,16 @@ private static int FindChunk(int[] prefix, int index)
         _ => throw new ArgumentOutOfRangeException(nameof(field), field, "Field is not a supported group-by dimension.")
     };
 
+    private static int SlotForCode(long value, long[] targets, int otherSlot)
+    {
+        for (int slot = 0; slot < targets.Length; slot++)
+        {
+            if (value == targets[slot]) { return slot; }
+        }
+
+        return otherSlot;
+    }
+
     // Shared by the pooled-field and event-id group-by scans: a pure integer match, so a negative event id still matches its own target; callers resolve absent pooled targets to int.MinValue (not -1) so a null-field row can't collide.
     private static int SlotForIndex(int value, int[] targets, int otherSlot)
     {
@@ -1087,6 +1183,15 @@ private static int ToBucket(long ticks, long minTicks, long bucketSpanTicks, int
         return bucket < 0 ? 0 : bucket >= bucketCount ? bucketCount - 1 : (int)bucket;
     }
 
+    private static bool TryGetPendingEventDataCode(ResolvedEvent pending, string fieldName, out long code)
+    {
+        if (pending.EventData.TryGetValue(fieldName, out EventFieldValue value)) { return value.TryGetWholeNumber(out code); }
+
+        code = 0;
+
+        return false;
+    }
+
     private (EventColumnChunk Chunk, int Row) Locate(int index)
     {
         int[] prefix = SealedPrefix();
@@ -1100,6 +1205,16 @@ private static int ToBucket(long ticks, long minTicks, long bucketSpanTicks, int
             "The columnar representation exists only for sealed rows; check IsPending first.") :
         Locate(index);
 
+    // A fresh per-scan memo (schemaId -> field index, or Unresolved until first hit), so each distinct schema's field
+    // position is resolved exactly once per scan rather than once per row.
+    private int[] NewSchemaFieldMemo()
+    {
+        int[] memo = new int[_schemas.Length];
+        Array.Fill(memo, SchemaFieldUnresolved);
+
+        return memo;
+    }
+
     private ResolvedEvent Pending(int index) => _pendingTail[index - _sealedCount];
 
     private ImmutableArray<EventProperty> ReconstructEventDataValues(int index)
@@ -1208,6 +1323,20 @@ private ImmutableArray<UserDataField> ReconstructUserData(int index)
         return fields.MoveToImmutable();
     }
 
+    private int ResolveSchemaFieldIndex(int schemaId, string fieldName)
+    {
+        int[] nameIndices = _schemas[schemaId];
+
+        for (int i = 0; i < nameIndices.Length; i++)
+        {
+            string? name = PoolGet(nameIndices[i]);
+
+            if (!string.IsNullOrEmpty(name) && string.Equals(name, fieldName, StringComparison.Ordinal)) { return i; }
+        }
+
+        return SchemaFieldAbsent;
+    }
+
     private int[] SealedPrefix()
     {
         int[]? prefix = Volatile.Read(ref _sealedPrefix);
@@ -1222,4 +1351,61 @@ private int[] SealedPrefix()
 
         return prefix;
     }
+
+    private bool TryGetSealedEventDataCode(EventColumnChunk chunk, int row, string fieldName, int[] fieldIndexBySchema, out long code)
+    {
+        int schemaId = chunk.RowEventDataSchemaId(row);
+
+        if ((uint)schemaId >= (uint)fieldIndexBySchema.Length) { code = 0; return false; }
+
+        int fieldIndex = fieldIndexBySchema[schemaId];
+
+        if (fieldIndex == SchemaFieldUnresolved)
+        {
+            fieldIndex = ResolveSchemaFieldIndex(schemaId, fieldName);
+            fieldIndexBySchema[schemaId] = fieldIndex;
+        }
+
+        if (fieldIndex < 0 || fieldIndex >= chunk.RowEventDataCount(row))
+        {
+            code = 0;
+
+            return false;
+        }
+
+        return TryGetWholeNumberFromRawField(chunk.RowEventDataField(row, fieldIndex), out code);
+    }
+
+    private bool TryGetWholeNumberFromRawField(in RawEventDataField field, out long code)
+    {
+        switch (field.Kind)
+        {
+            case StoredFieldKind.SByte:
+            case StoredFieldKind.Int16:
+            case StoredFieldKind.Int32:
+            case StoredFieldKind.Int64:
+                code = field.Bits;
+
+                return code >= 0;
+            case StoredFieldKind.Byte:
+            case StoredFieldKind.UInt16:
+            case StoredFieldKind.UInt32:
+            case StoredFieldKind.UInt64:
+            case StoredFieldKind.SizeT:
+                ulong unsigned = unchecked((ulong)field.Bits);
+
+                if (unsigned <= long.MaxValue) { code = (long)unsigned; return true; }
+
+                code = 0;
+
+                return false;
+            case StoredFieldKind.String:
+            case StoredFieldKind.StringForm:
+                return EventFieldValue.TryParseWholeNumber(PoolGet(field.RefIndex), out code);
+            default:
+                code = 0;
+
+                return false;
+        }
+    }
 }

src/EventLogExpert.Eventing/Common/Events/EventColumnStoreReader.cs

@@ -36,6 +36,29 @@ internal EventColumnStoreReader(EventLogId logId, EventColumnStore store)
 
     public IReadOnlyList<string?> Pool => new PoolView(_store, PendingPool());
 
+    public void BucketTimeTicksByEventData(
+        ReadOnlySpan<int> rankByPhysical,
+        long minTicks,
+        long bucketSpanTicks,
+        int bucketCount,
+        string fieldName,
+        long[] targetCodes,
+        int[] slotCounts,
+        CancellationToken cancellationToken)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(fieldName);
+        ArgumentNullException.ThrowIfNull(targetCodes);
+        ArgumentNullException.ThrowIfNull(slotCounts);
+        ArgumentOutOfRangeException.ThrowIfNotEqual(rankByPhysical.Length, Count);
+        ArgumentOutOfRangeException.ThrowIfLessThan(bucketSpanTicks, 1);
+        ArgumentOutOfRangeException.ThrowIfLessThan(bucketCount, 1);
+
+        int slotCount = targetCodes.Length + 1;
+        ArgumentOutOfRangeException.ThrowIfLessThan(slotCounts.Length, bucketCount * slotCount);
+
+        _store.BucketTimeTicksByEventData(rankByPhysical, minTicks, bucketSpanTicks, bucketCount, fieldName, targetCodes, slotCount, slotCounts, cancellationToken);
+    }
+
     public void BucketTimeTicksByEventId(
         ReadOnlySpan<int> rankByPhysical,
         long minTicks,
@@ -159,6 +182,15 @@ public void CopyPoolIndexColumn(EventFieldId field, int[] poolIndices)
         }
     }
 
+    public void CountEventDataValues(ReadOnlySpan<int> rankByPhysical, string fieldName, IDictionary<long, int> counts, CancellationToken cancellationToken)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(fieldName);
+        ArgumentNullException.ThrowIfNull(counts);
+        ArgumentOutOfRangeException.ThrowIfNotEqual(rankByPhysical.Length, Count);
+
+        _store.CountEventDataValues(rankByPhysical, fieldName, counts, cancellationToken);
+    }
+
     public void CountEventIds(ReadOnlySpan<int> rankByPhysical, IDictionary<int, int> counts, CancellationToken cancellationToken)
     {
         ArgumentNullException.ThrowIfNull(counts);

src/EventLogExpert.Eventing/Common/Events/EventFieldValue.cs

@@ -63,6 +63,54 @@ public bool TryGetUInt64(out ulong value)
         return false;
     }
 
+    /// <summary>
+    ///     Reads this value as a non-negative whole number when it is one: a typed integral kind, or a
+    ///     <see cref="EventFieldValueKind.String" /> holding a plain decimal (e.g. "23") or a <c>0x</c>-prefixed hex form
+    ///     (e.g. "0x17"). Decimal and hex spellings of the same code canonicalize to the same value. Allocation-free.
+    /// </summary>
+    public bool TryGetWholeNumber(out long value)
+    {
+        switch (_kind)
+        {
+            case EventFieldValueKind.Int64 when _bits >= 0:
+            case EventFieldValueKind.UInt64 when unchecked((ulong)_bits) <= long.MaxValue:
+                value = _bits;
+
+                return true;
+            case EventFieldValueKind.String:
+                return TryParseWholeNumber(_reference as string, out value);
+            default:
+                value = 0;
+
+                return false;
+        }
+    }
+
+    internal static bool TryParseWholeNumber(string? text, out long value)
+    {
+        value = 0;
+
+        if (string.IsNullOrEmpty(text)) { return false; }
+
+        ReadOnlySpan<char> span = text;
+
+        if (!span.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
+        {
+            return long.TryParse(span, NumberStyles.None, CultureInfo.InvariantCulture, out value);
+        }
+
+        // Parse hex as unsigned so a high-bit form (e.g. 0xFFFF...) can't wrap to a negative code, then keep only codes that fit a non-negative long.
+        if (!ulong.TryParse(span[2..], NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out ulong hex) ||
+            hex > long.MaxValue)
+        {
+            return false;
+        }
+
+        value = (long)hex;
+
+        return true;
+    }
+
     public bool TryGetDouble(out double value)
     {
         if (_kind == EventFieldValueKind.Double) { value = BitConverter.Int64BitsToDouble(_bits); return true; }

src/EventLogExpert.Eventing/Common/Events/IEventColumnReader.cs

@@ -22,6 +22,21 @@ public interface IEventColumnReader
     /// </summary>
     IReadOnlyList<string?> Pool { get; }
 
+    /// <summary>
+    ///     Bucket-by-EventData variant of <see cref="BucketTimeTicksByField" />: each survivor lands in its
+    ///     <paramref name="targetCodes" /> slot (matched on the field's whole-number code) else the trailing "other" slot,
+    ///     which also absorbs rows that lack the field; (<paramref name="targetCodes" /> length + 1) slots per bin.
+    /// </summary>
+    void BucketTimeTicksByEventData(
+        ReadOnlySpan<int> rankByPhysical,
+        long minTicks,
+        long bucketSpanTicks,
+        int bucketCount,
+        string fieldName,
+        long[] targetCodes,
+        int[] slotCounts,
+        CancellationToken cancellationToken);
+
     /// <summary>Group-by variant keyed on the numeric event id; (<paramref name="targetIds" /> length + 1) slots per bin.</summary>
     void BucketTimeTicksByEventId(
         ReadOnlySpan<int> rankByPhysical,
@@ -78,6 +93,13 @@ void BucketTimeTicksBySeverity(
     /// </summary>
     void CopyPoolIndexColumn(EventFieldId field, int[] poolIndices);
 
+    /// <summary>
+    ///     Group-by variant for a named EventData field of allowlisted numeric codes: tallies survivors by the field's
+    ///     whole-number code (decimal and <c>0x</c>-hex spellings canonicalize to one code), so the histogram can split, for
+    ///     example, by LogonType or TicketEncryptionType. Rows that lack the field are omitted.
+    /// </summary>
+    void CountEventDataValues(ReadOnlySpan<int> rankByPhysical, string fieldName, IDictionary<long, int> counts, CancellationToken cancellationToken);
+
     /// <summary>
     ///     Tallies each surviving row by its numeric event id into <paramref name="counts" /> (accumulating across a
     ///     combined view).