memory, more memory, even more memory

Author: Hirogen

src/ColumnizerLib/Column.cs

@@ -27,21 +27,10 @@ public class Column : IColumnMemory
 
     static Column ()
     {
-        if (Environment.Version >= Version.Parse("6.2"))
-        {
-            //Win8 or newer support full UTF8 chars with the preinstalled fonts.
-            //Replace null char with UTF8 Symbol U+2400 (␀)
-            _replacementsMemory.Add(input => ReplaceNullChar(input, '␀'));
-        }
-        else
-        {
-            //Everything below Win8 the installed fonts seems to not to support reliabel
-            //Replace null char with space
-            //.net 10 does no longer support windows lower then windows 10
-            //TODO: remove if with one of the next releases
-            //https://github.com/dotnet/core/blob/main/release-notes/10.0/supported-os.md
-            _replacementsMemory.Add(input => ReplaceNullChar(input, ' '));
-        }
+        //.net 10 only supports Windows10+ which has full UTF8-font support
+        // Replace null char with UTF-8 Symbol U+2400 (␀)
+        //https://github.com/dotnet/core/blob/main/release-notes/10.0/supported-os.md
+        _replacementsMemory.Add(input => ReplaceNullChar(input, ' '));
 
         EmptyColumn = new Column { FullValue = ReadOnlyMemory<char>.Empty };
     }

src/LogExpert.Core/Classes/Filter/Filter.cs

@@ -1,5 +1,4 @@
 using LogExpert.Core.Callback;
-using LogExpert.Core.Classes;
 
 using NLog;
 
@@ -69,7 +68,7 @@ private int DoFilter (FilterParams filterParams, int startLine, int maxCount, Li
                     return count;
                 }
 
-                var line = _callback.GetLogLine(lineNum);
+                var line = _callback.GetLogLineMemory(lineNum);
 
                 if (line == null)
                 {

src/LogExpert.Core/Classes/Filter/FilterParams.cs

@@ -1,4 +1,3 @@
-using System.Collections;
 using System.Collections.ObjectModel;
 using System.Drawing;
 using System.Text.RegularExpressions;
@@ -12,6 +11,10 @@
 
 namespace LogExpert.Core.Classes.Filter;
 
+// TODO: Convert LastLine to ReadOnlyMemory<char> as part of memory optimization effort
+// This will eliminate string allocations in TestFilterCondition and improve performance.
+// Will require updating all callers that currently expect string type.
+// Related to: ReadOnlyMemory migration in columnizers
 [Serializable]
 public class FilterParams : ICloneable
 {
@@ -75,7 +78,7 @@ public class FilterParams : ICloneable
 
     [JsonIgnore]
     [field: NonSerialized]
-    public Hashtable LastNonEmptyCols { get; set; } = [];
+    public Dictionary<int, ReadOnlyMemory<char>> LastNonEmptyCols { get; set; } = [];
 
     [JsonIgnore]
     [field: NonSerialized]

src/LogExpert.Core/Classes/Util.cs

@@ -53,29 +53,31 @@ public static string GetExtension (string fileName)
             : fileName[(i + 1)..];
     }
 
-
     public static string GetFileSizeAsText (long size)
     {
         return size < 1024
-            ? string.Empty + size + " bytes"
+            ? $"{size} bytes"
             : size < 1024 * 1024
-                ? string.Empty + (size / 1024) + " KB"
-                : string.Empty + $"{size / 1048576.0:0.00}" + " MB";
+                ? $"{size / 1024} KB"
+                : $"{size / 1048576.0:0.00} MB";
     }
 
-    //TOOD: check if the callers are checking for null before calling
-    public static bool TestFilterCondition (FilterParams filterParams, ILogLine line, ILogLineColumnizerCallback columnizerCallback)
+    public static bool TestFilterCondition (FilterParams filterParams, ILogLineMemory logLine, ILogLineMemoryColumnizerCallback columnizerCallback)
     {
         ArgumentNullException.ThrowIfNull(filterParams, nameof(filterParams));
-        ArgumentNullException.ThrowIfNull(line, nameof(line));
+        ArgumentNullException.ThrowIfNull(logLine, nameof(logLine));
 
-        if (filterParams.LastLine.Equals(line.FullLine, StringComparison.OrdinalIgnoreCase))
+        // TODO: Once FilterParams.LastLine is converted to ReadOnlyMemory<char>, this can be simplified to:
+        // if (MemoryExtensions.Equals(filterParams.LastLine.Span, logLine.FullLine.Span, StringComparison.OrdinalIgnoreCase))
+        if (MemoryExtensions.Equals(filterParams.LastLine.AsSpan(), logLine.FullLine.Span, StringComparison.OrdinalIgnoreCase))
         {
             return filterParams.LastResult;
         }
 
-        var match = TestFilterMatch(filterParams, line, columnizerCallback);
-        filterParams.LastLine = line.FullLine;
+        var match = TestFilterMatch(filterParams, logLine, columnizerCallback);
+
+        // TODO: This ToString() allocation will be eliminated when LastLine becomes ReadOnlyMemory<char>
+        filterParams.LastLine = logLine.FullLine.ToString();
 
         if (filterParams.IsRangeSearch)
         {
@@ -108,48 +110,68 @@ public static bool TestFilterCondition (FilterParams filterParams, ILogLine line
         return match;
     }
 
-    //TODO Add Null Checks (https://github.com/LogExperts/LogExpert/issues/403)
-    public static int DamerauLevenshteinDistance (string src, string dest)
+    public static int DamerauLevenshteinDistance (ReadOnlySpan<char> source, ReadOnlySpan<char> destination, bool ignoreCase = false)
     {
-        var d = new int[src.Length + 1, dest.Length + 1];
-        int i, j, cost;
-        var str1 = src.ToCharArray();
-        var str2 = dest.ToCharArray();
+        var d = new int[source.Length + 1, destination.Length + 1];
 
-        for (i = 0; i <= str1.Length; i++)
+        for (var i = 0; i <= source.Length; i++)
         {
             d[i, 0] = i;
         }
 
-        for (j = 0; j <= str2.Length; j++)
+        for (var j = 0; j <= destination.Length; j++)
         {
             d[0, j] = j;
         }
 
-        for (i = 1; i <= str1.Length; i++)
+        for (var i = 1; i <= source.Length; i++)
         {
-            for (j = 1; j <= str2.Length; j++)
+            for (var j = 1; j <= destination.Length; j++)
             {
-                cost = str1[i - 1] == str2[j - 1]
+                var char1 = ignoreCase
+                    ? char.ToUpperInvariant(source[i - 1])
+                    : source[i - 1];
+
+                var char2 = ignoreCase
+                    ? char.ToUpperInvariant(destination[j - 1])
+                    : destination[j - 1];
+
+                var cost = char1 == char2
                     ? 0
                     : 1;
 
-                d[i, j] =
-                    Math.Min(d[i - 1, j] + 1, // Deletion
-                        Math.Min(d[i, j - 1] + 1, // Insertion
-                            d[i - 1, j - 1] + cost)); // Substitution
-
-                if (i > 1 && j > 1 && str1[i - 1] == str2[j - 2] && str1[i - 2] == str2[j - 1])
+                d[i, j] = Math.Min
+                    (
+                        d[i - 1, j] + 1,        // Deletion
+                        Math.Min
+                        (
+                            d[i, j - 1] + 1,    // Insertion
+                            d[i - 1, j - 1] + cost // Substitution
+                        )
+                    );
+
+                // Transposition
+                if (i > 1 && j > 1)
                 {
-                    d[i, j] = Math.Min(d[i, j], d[i - 2, j - 2] + cost);
+                    var prevChar1 = ignoreCase
+                        ? char.ToUpperInvariant(source[i - 2])
+                        : source[i - 2];
+
+                    var prevChar2 = ignoreCase
+                        ? char.ToUpperInvariant(destination[j - 2])
+                        : destination[j - 2];
+
+                    if (char1 == prevChar2 && prevChar1 == char2)
+                    {
+                        d[i, j] = Math.Min(d[i, j], d[i - 2, j - 2] + cost);
+                    }
                 }
             }
         }
 
-        return d[str1.Length, str2.Length];
+        return d[source.Length, destination.Length];
     }
 
-    //TODO Add Null Checks (https://github.com/LogExperts/LogExpert/issues/403)
     public static unsafe int YetiLevenshtein (string s1, string s2)
     {
         fixed (char* p1 = s1)
@@ -159,13 +181,13 @@ public static unsafe int YetiLevenshtein (string s1, string s2)
         }
     }
 
-    public static unsafe int YetiLevenshtein (string s1, string s2, int substitionCost)
+    public static unsafe int YetiLevenshtein (string s1, string s2, int substitutionCost)
     {
-        var xc = substitionCost - 1;
+        var xc = substitutionCost - 1;
 
         if (xc is < 0 or > 1)
         {
-            throw new ArgumentException("", nameof(substitionCost));
+            throw new ArgumentException("", nameof(substitutionCost));
         }
 
         fixed (char* p1 = s1)
@@ -385,26 +407,6 @@ public static unsafe int YetiLevenshtein (char* s1, int l1, char* s2, int l2, in
         return i;
     }
 
-    /// <summary>
-    /// Returns true, if the given string is null or empty
-    /// </summary>
-    /// <param name="toTest"></param>
-    /// <returns></returns>
-    public static bool IsNull (string toTest)
-    {
-        return toTest == null || toTest.Length == 0;
-    }
-
-    /// <summary>
-    /// Returns true, if the given string is null or empty or contains only spaces
-    /// </summary>
-    /// <param name="toTest"></param>
-    /// <returns></returns>
-    public static bool IsNullOrSpaces (string toTest)
-    {
-        return toTest == null || toTest.Trim().Length == 0;
-    }
-
     [Conditional("DEBUG")]
     public static void AssertTrue (bool condition, string msg)
     {
@@ -418,7 +420,7 @@ public static void AssertTrue (bool condition, string msg)
 
     //TODO Add Null Check (https://github.com/LogExperts/LogExpert/issues/403)
     [SupportedOSPlatform("windows")]
-    public string? GetWordFromPos (int xPos, string text, Graphics g, Font font)
+    public static string? GetWordFromPos (int xPos, string text, Graphics g, Font font)
     {
         var words = text.Split([' ', '.', ':', ';']);
 
@@ -469,7 +471,7 @@ public static void AssertTrue (bool condition, string msg)
 
     #region Private Methods
 
-    private static bool TestFilterMatch (FilterParams filterParams, ILogLine line, ILogLineColumnizerCallback columnizerCallback)
+    private static bool TestFilterMatch (FilterParams filterParams, ILogLineMemory logLine, ILogLineMemoryColumnizerCallback columnizerCallback)
     {
         string normalizedSearchText;
         string searchText;
@@ -495,22 +497,19 @@ private static bool TestFilterMatch (FilterParams filterParams, ILogLine line, I
 
         if (filterParams.ColumnRestrict)
         {
-            var columns = filterParams.CurrentColumnizer.SplitLine(columnizerCallback, line);
+            var columns = filterParams.CurrentColumnizer.SplitLine(columnizerCallback, logLine);
             var found = false;
             foreach (var colIndex in filterParams.ColumnList)
             {
-                if (colIndex < columns.ColumnValues.Length
-                ) // just to be sure, maybe the columnizer has changed anyhow
+                if (colIndex < columns.ColumnValues.Length) // just to be sure, maybe the columnizer has changed anyhow
                 {
                     if (columns.ColumnValues[colIndex].FullValue.Trim().Length == 0)
                     {
                         if (filterParams.EmptyColumnUsePrev)
                         {
-                            var prevValue = (string)filterParams.LastNonEmptyCols[colIndex];
-                            if (prevValue != null)
+                            if (filterParams.LastNonEmptyCols.TryGetValue(colIndex, out var prevValue))
                             {
-                                if (TestMatchSub(filterParams, prevValue, normalizedSearchText, searchText, rex,
-                                    filterParams.ExactColumnMatch))
+                                if (TestMatchSub(filterParams, prevValue, normalizedSearchText.AsSpan(), searchText.AsSpan(), rex, filterParams.ExactColumnMatch))
                                 {
                                     found = true;
                                 }
@@ -524,9 +523,7 @@ private static bool TestFilterMatch (FilterParams filterParams, ILogLine line, I
                     else
                     {
                         filterParams.LastNonEmptyCols[colIndex] = columns.ColumnValues[colIndex].FullValue;
-                        if (TestMatchSub(filterParams, columns.ColumnValues[colIndex].FullValue, normalizedSearchText,
-                            searchText, rex,
-                            filterParams.ExactColumnMatch))
+                        if (TestMatchSub(filterParams, columns.ColumnValues[colIndex].FullValue, normalizedSearchText.AsSpan(), searchText.AsSpan(), rex, filterParams.ExactColumnMatch))
                         {
                             found = true;
                         }
@@ -538,11 +535,17 @@ private static bool TestFilterMatch (FilterParams filterParams, ILogLine line, I
         }
         else
         {
-            return TestMatchSub(filterParams, line.FullLine, normalizedSearchText, searchText, rex, false);
+            return TestMatchSub(filterParams, logLine.FullLine, normalizedSearchText.AsSpan(), searchText.AsSpan(), rex, false);
         }
     }
 
-    private static bool TestMatchSub (FilterParams filterParams, string line, string lowerSearchText, string searchText, Regex rex, bool exactMatch)
+    private static bool TestMatchSub (
+            FilterParams filterParams,
+            ReadOnlySpan<char> line,
+            ReadOnlySpan<char> normalizedSearchText,  // Pre-normalized (uppercase) // lowerSearchText
+            ReadOnlySpan<char> searchText,
+            Regex rex,
+            bool exactMatch)
     {
         if (filterParams.IsRegex)
         {
@@ -557,14 +560,15 @@ private static bool TestMatchSub (FilterParams filterParams, string line, string
             {
                 if (exactMatch)
                 {
-                    if (line.ToUpperInvariant().Trim().Equals(lowerSearchText, StringComparison.Ordinal))
+                    var trimmedLine = line.Trim();
+                    if (MemoryExtensions.Equals(trimmedLine, normalizedSearchText, StringComparison.OrdinalIgnoreCase))
                     {
                         return true;
                     }
                 }
                 else
                 {
-                    if (line.Contains(lowerSearchText, StringComparison.OrdinalIgnoreCase))
+                    if (line.Contains(normalizedSearchText, StringComparison.OrdinalIgnoreCase))
                     {
                         return true;
                     }
@@ -581,7 +585,7 @@ private static bool TestMatchSub (FilterParams filterParams, string line, string
                 }
                 else
                 {
-                    if (line.Contains(searchText, StringComparison.OrdinalIgnoreCase))
+                    if (line.Contains(searchText, StringComparison.Ordinal))
                     {
                         return true;
                     }
@@ -593,16 +597,11 @@ private static bool TestMatchSub (FilterParams filterParams, string line, string
                 var range = line.Length - searchText.Length;
                 if (range > 0)
                 {
-                    for (var i = 0; i < range; ++i)
+                    for (var i = 0; i <= range; ++i)
                     {
-                        var src = line.Substring(i, searchText.Length);
-
-                        if (!filterParams.IsCaseSensitive)
-                        {
-                            src = src.ToLowerInvariant();
-                        }
+                        var src = line.Slice(i, searchText.Length);
 
-                        var dist = DamerauLevenshteinDistance(src, searchText);
+                        var dist = DamerauLevenshteinDistance(src, searchText, !filterParams.IsCaseSensitive);
 
                         if ((searchText.Length + 1) / (float)(dist + 1) >= 11F / (float)(filterParams.FuzzyValue + 1F))
                         {
@@ -618,6 +617,22 @@ private static bool TestMatchSub (FilterParams filterParams, string line, string
         return false;
     }
 
+    private static bool TestMatchSub (
+            FilterParams filterParams,
+            ReadOnlyMemory<char> line,  // From ILogLineMemory.FullLine
+            ReadOnlySpan<char> normalizedSearchText,
+            ReadOnlySpan<char> searchText,
+            Regex rex,
+            bool exactMatch)
+    {
+        return TestMatchSub(filterParams, line.Span, normalizedSearchText, searchText, rex, exactMatch);
+    }
+
+    private static bool TestMatchSub (FilterParams filterParams, string line, string lowerSearchText, string searchText, Regex rex, bool exactMatch)
+    {
+        return TestMatchSub(filterParams, line.AsSpan(), lowerSearchText.AsSpan(), searchText.AsSpan(), rex, exactMatch);
+    }
+
     private static unsafe int MemchrRPLC (char* buffer, char c, int count)
     {
         var p = buffer;