cache log analysis & fetch it from the frontend

Pathoschild · Pathoschild · commit 6b16993eb74f · 2025-04-06T22:38:24.000-04:00
The backend now saves a pre-parsed representation of the log to storage, and returns a SAS URI which the frontend uses to fetch it. This should significantly reduce bandwidth usage and response times, by avoiding the repeated parsing and double data transfer.

This implementation is messy, but it's an intermediate step towards later refactoring.
diff --git a/docs/release-notes.md b/docs/release-notes.md
@@ -3,13 +3,17 @@
 # Release notes
 ## Upcoming release
 * For the web UI:
-  * Improved JSON parser code display:
+  * Improved JSON parser:
     * You can now hover/click braces to highlight matching pairs.
     * You can now hover and click 'Copy' on the top-right to copy the full code to the clipboard.
     * Updated to newer syntax highlighting library.
     * Fixed some JSON files breaking page layout.
+  * Improved log parser:
+    * Mods which failed to load are now shown in the mod list (with 'failed to load' in the error column).
+    * Reduced response times with a new analysis cache and client-side fetch.
+    * Removed support for very old SMAPI logs.
+    * You can now download a JSON representation of the parsed log (see the download link at the bottom of the log page).
   * Third-party libraries are now served from `smapi.io` instead of external CDNs.
-  * Removed support for very old SMAPI logs.
 
 ## 4.2.1
 Released 25 March 2025 for Stardew Valley 1.6.14 or later.
diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs
@@ -82,10 +82,10 @@ public async Task<ViewResult> Index(string? schemaName = null, string? id = null
         this.Response.Headers["X-Robots-Tag"] = "noindex";
 
         // fetch raw JSON
-        StoredFileInfo file = await this.Storage.GetAsync(id!, renew);
-        if (string.IsNullOrWhiteSpace(file.Content))
+        StoredFileInfo file = await this.Storage.GetAsync(id!, renew, forceDownloadContent: true);
+        if (string.IsNullOrWhiteSpace(file.FetchedData))
             return this.View("Index", result.SetUploadError("The JSON file seems to be empty."));
-        result.SetContent(file.Content, oldExpiry: file.OldExpiry, newExpiry: file.NewExpiry, uploadWarning: file.Warning);
+        result.SetContent(file.FetchedData, oldExpiry: file.OldExpiry, newExpiry: file.NewExpiry, uploadWarning: file.Warning);
 
         // skip parsing if we're going to the edit screen
         if (isEditView)
@@ -102,7 +102,7 @@ public async Task<ViewResult> Index(string? schemaName = null, string? id = null
             };
             try
             {
-                parsed = JToken.Parse(file.Content, settings);
+                parsed = JToken.Parse(file.FetchedData, settings);
             }
             catch (JsonReaderException ex)
             {
@@ -159,12 +159,13 @@ public async Task<ActionResult> PostAsync(JsonValidatorRequestModel? request)
             return this.View("Index", this.GetModel(null, schemaName, isEditView: true).SetUploadError("The JSON file seems to be empty."));
 
         // upload file
-        UploadResult result = await this.Storage.SaveAsync(input);
+        string id = Guid.NewGuid().ToString("N");
+        UploadResult result = await this.Storage.SaveAsync(id, input);
         if (!result.Succeeded)
-            return this.View("Index", this.GetModel(result.ID, schemaName, isEditView: true).SetContent(input, null, null).SetUploadError(result.UploadError));
+            return this.View("Index", this.GetModel(id, schemaName, isEditView: true).SetContent(input, null, null).SetUploadError(result.UploadError));
 
         // redirect to view
-        return this.Redirect(this.Url.PlainAction("Index", "JsonValidator", new { schemaName, id = result.ID })!);
+        return this.Redirect(this.Url.PlainAction("Index", "JsonValidator", new { schemaName, id })!);
     }
 
 
diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs
@@ -5,6 +5,7 @@
 using System.Threading.Tasks;
 using System.Web;
 using Microsoft.AspNetCore.Mvc;
+using Newtonsoft.Json;
 using StardewModdingAPI.Toolkit.Utilities;
 using StardewModdingAPI.Web.Framework;
 using StardewModdingAPI.Web.Framework.LogParsing;
@@ -56,25 +57,52 @@ public async Task<ActionResult> Index(string? id = null, LogViewFormat format =
         // don't index uploaded logs in search engines
         this.Response.Headers["X-Robots-Tag"] = "noindex";
 
-        // fetch log
-        StoredFileInfo file = await this.Storage.GetAsync(id, renew);
+        // fetch log info
+        StoredFileInfo file = await this.Storage.GetAsync($"parsed-{id}", renew);
+        string? fetchUri = file.FetchUri;
+        ParsedLog? rawData = null;
+        if (file.FetchedData != null)
+        {
+            fetchUri = null;
+            rawData = new LogParser().Parse(file.FetchedData);
+        }
+
+        // TEMPORARY: support logs created before 2025-04-07.
+        if (!file.Success)
+        {
+            StoredFileInfo fallbackFile = await this.Storage.GetAsync(id, renew, forceDownloadContent: true);
+            if (fallbackFile.Success)
+            {
+                file = fallbackFile;
+                fetchUri = null;
+                rawData = new LogParser().Parse(file.FetchedData);
+            }
+        }
+
+        // wrap error if no data available
+        if (fetchUri is null)
+            rawData ??= new ParsedLog { Error = file.Error ?? "An unknown error occurred." };
 
         // render view
         switch (format)
         {
             case LogViewFormat.Default:
             case LogViewFormat.RawView:
                 {
-                    ParsedLog log = file.Success
-                        ? new LogParser().Parse(file.Content)
-                        : new ParsedLog { IsValid = false, Error = file.Error };
-
-                    return this.View("Index", this.GetModel(id, uploadWarning: file.Warning, oldExpiry: file.OldExpiry, newExpiry: file.NewExpiry).SetResult(log, showRaw: format == LogViewFormat.RawView));
+                    LogParserModel model = this
+                        .GetModel(id, uploadWarning: file.Warning, oldExpiry: file.OldExpiry, newExpiry: file.NewExpiry)
+                        .SetResult(fetchUri, rawData, showRaw: format == LogViewFormat.RawView);
+                    return this.View("Index", model);
                 }
 
             case LogViewFormat.RawDownload:
+                // from fetch URL
+                if (file.FetchUri != null)
+                    return this.Redirect(file.FetchUri);
+
+                // from raw content
                 {
-                    string content = file.Error ?? file.Content ?? string.Empty;
+                    string content = file.FetchedData ?? file.Error ?? string.Empty;
                     return this.File(Encoding.UTF8.GetBytes(content), "plain/text", $"SMAPI log ({id}).txt");
                 }
 
@@ -102,13 +130,17 @@ public async Task<ActionResult> PostAsync()
                 return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty."));
         }
 
+        // parse log
+        ParsedLog log = new LogParser().Parse(input);
+
         // upload log
-        UploadResult uploadResult = await this.Storage.SaveAsync(input);
+        string id = Guid.NewGuid().ToString("N");
+        UploadResult uploadResult = await this.Storage.SaveAsync($"parsed-{id}", JsonConvert.SerializeObject(log));
         if (!uploadResult.Succeeded)
             return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError));
 
         // redirect to view
-        return this.Redirect(this.Url.PlainAction("Index", "LogParser", new { id = uploadResult.ID })!);
+        return this.Redirect(this.Url.PlainAction("Index", "LogParser", new { id })!);
     }
 
 
diff --git a/src/SMAPI.Web/Framework/LogParsing/Models/LogModInfo.cs b/src/SMAPI.Web/Framework/LogParsing/Models/LogModInfo.cs
@@ -24,7 +24,7 @@ public class LogModInfo
     public string Author { get; }
 
     /// <summary>The mod version.</summary>
-    public string Version { get; private set; }
+    public string? Version { get; private set; }
 
     /// <summary>The mod description.</summary>
     public string Description { get; }
diff --git a/src/SMAPI.Web/Framework/LogParsing/Models/ParsedLog.cs b/src/SMAPI.Web/Framework/LogParsing/Models/ParsedLog.cs
@@ -54,4 +54,33 @@ public class ParsedLog
 
     /// <summary>The log messages.</summary>
     public LogMessage[] Messages { get; set; } = [];
+
+
+    /*********
+    ** Public methods
+    *********/
+    /// <summary>Construct an empty instance.</summary>
+    public ParsedLog() { }
+
+    /// <summary>Construct an instance.</summary>
+    /// <param name="log">The other log instance to copy.</param>
+    public ParsedLog(ParsedLog log)
+    {
+        // metadata
+        this.IsValid = log.IsValid;
+        this.Error = log.Error;
+        this.RawText = log.RawText;
+        this.IsSplitScreen = log.IsSplitScreen;
+
+        // log data
+        this.ApiVersion = log.ApiVersion;
+        this.ApiVersionParsed = log.ApiVersionParsed;
+        this.GameVersion = log.GameVersion;
+        this.OperatingSystem = log.OperatingSystem;
+        this.GamePath = log.GamePath;
+        this.ModPath = log.ModPath;
+        this.Timestamp = log.Timestamp;
+        this.Mods = log.Mods;
+        this.Messages = log.Messages;
+    }
 }
diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs
@@ -6,13 +6,15 @@ namespace StardewModdingAPI.Web.Framework.Storage;
 internal interface IStorageProvider
 {
     /// <summary>Save a text file to storage.</summary>
+    /// <param name="id">The ID with which to save the result.</param>
     /// <param name="content">The content to upload.</param>
     /// <param name="compress">Whether to gzip the text.</param>
     /// <returns>Returns metadata about the save attempt.</returns>
-    Task<UploadResult> SaveAsync(string content, bool compress = true);
+    Task<UploadResult> SaveAsync(string id, string content, bool compress = true);
 
     /// <summary>Fetch raw text from storage.</summary>
-    /// <param name="id">The storage ID returned by <see cref="SaveAsync"/>.</param>
+    /// <param name="id">The storage ID used to upload the file.</param>
     /// <param name="forceRenew">Whether to reset the file expiry.</param>
-    Task<StoredFileInfo> GetAsync(string id, bool forceRenew);
+    /// <param name="forceDownloadContent">Whether to download the file content even if a fetch URI can be provided.</param>
+    Task<StoredFileInfo> GetAsync(string id, bool forceRenew, bool forceDownloadContent = false);
 }
diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs
@@ -6,6 +6,7 @@
 using Azure;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
+using Azure.Storage.Sas;
 using Microsoft.Extensions.Options;
 using StardewModdingAPI.Web.Framework.Clients.Pastebin;
 using StardewModdingAPI.Web.Framework.Compression;
@@ -53,16 +54,14 @@ public StorageProvider(IOptions<ApiClientsConfig> clientsConfig, IPastebinClient
     }
 
     /// <inheritdoc />
-    public async Task<UploadResult> SaveAsync(string content, bool compress = true)
+    public async Task<UploadResult> SaveAsync(string id, string content, bool compress = true)
     {
-        string id = Guid.NewGuid().ToString("N");
-
         // save to Azure
         if (this.HasAzure)
         {
             try
             {
-                using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
+                await using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
                 BlobClient blob = this.GetAzureBlobClient(id);
                 await blob.UploadAsync(stream);
 
@@ -80,42 +79,53 @@ public async Task<UploadResult> SaveAsync(string content, bool compress = true)
             string path = this.GetDevFilePath(id);
             Directory.CreateDirectory(Path.GetDirectoryName(path)!);
 
-            File.WriteAllText(path, content);
+            await File.WriteAllTextAsync(path, content);
             return new UploadResult(id, null);
         }
     }
 
     /// <inheritdoc />
-    public async Task<StoredFileInfo> GetAsync(string id, bool forceRenew)
+    public async Task<StoredFileInfo> GetAsync(string id, bool forceRenew, bool forceDownloadContent = false)
     {
         // fetch from blob storage
-        if (Guid.TryParseExact(id, "N", out Guid _))
+        bool isBlobStorage = id.StartsWith("parsed-")
+            ? Guid.TryParseExact(id.Substring("parsed-".Length), "N", out _)
+            : Guid.TryParseExact(id, "N", out _);
+        if (isBlobStorage)
         {
             // Azure Blob storage
             if (this.HasAzure)
             {
                 try
                 {
-                    // get client
-                    BlobClient blob = this.GetAzureBlobClient(id);
-
-                    // fetch file
-                    Response<BlobDownloadInfo> response = await blob.DownloadAsync();
-                    using BlobDownloadInfo result = response.Value;
-                    using StreamReader reader = new(result.Content);
-                    DateTimeOffset oldExpiry = this.GetExpiry(result.Details.LastModified);
-                    string content = this.GzipHelper.DecompressString(await reader.ReadToEndAsync());
+                    // fetch metadata
+                    BlobClient blobClient = this.GetAzureBlobClient(id);
+                    Response<BlobProperties> properties = await blobClient.GetPropertiesAsync();
+                    DateTimeOffset lastModified = properties.Value.LastModified;
+                    DateTimeOffset oldExpiry = this.GetExpiry(lastModified);
+
+                    // get content or URL file
+                    string? content = null;
+                    string? fetchUri = null;
+                    if (forceDownloadContent)
+                    {
+                        Response<BlobDownloadInfo> response = await blobClient.DownloadAsync();
+                        using StreamReader reader = new(response.Value.Content);
+                        content = this.GzipHelper.DecompressString(await reader.ReadToEndAsync());
+                    }
+                    else
+                        fetchUri = blobClient.GenerateSasUri(BlobSasPermissions.Read, DateTimeOffset.UtcNow.AddMinutes(5)).ToString();
 
                     // extend expiry if needed
                     DateTimeOffset newExpiry = oldExpiry;
-                    if (forceRenew || this.IsWithinAutoRenewalWindow(result.Details.LastModified))
+                    if (forceRenew || this.IsWithinAutoRenewalWindow(lastModified))
                     {
-                        await blob.SetMetadataAsync(new Dictionary<string, string> { ["expiryRenewed"] = DateTime.UtcNow.ToString("O") }); // change the blob's last-modified date (the specific property set doesn't matter)
+                        await blobClient.SetMetadataAsync(new Dictionary<string, string> { ["expiryRenewed"] = DateTime.UtcNow.ToString("O") }); // change the blob's last-modified date (the specific property set doesn't matter)
                         newExpiry = this.GetExpiry(DateTimeOffset.UtcNow);
                     }
 
                     // build model
-                    return new StoredFileInfo(content, oldExpiry, newExpiry);
+                    return new StoredFileInfo(fetchUri, content, oldExpiry, newExpiry);
                 }
                 catch (RequestFailedException ex)
                 {
@@ -135,9 +145,7 @@ public async Task<StoredFileInfo> GetAsync(string id, bool forceRenew)
                 if (file.Exists && file.LastWriteTimeUtc.AddDays(this.ExpiryDays) < DateTime.UtcNow) // expired
                     file.Delete();
                 if (!file.Exists)
-                {
                     return new StoredFileInfo(error: "There's no file with that ID.");
-                }
 
                 // renew
                 if (forceRenew)
@@ -148,7 +156,8 @@ public async Task<StoredFileInfo> GetAsync(string id, bool forceRenew)
 
                 // build model
                 return new StoredFileInfo(
-                    content: await File.ReadAllTextAsync(file.FullName),
+                    fetchUri: null,
+                    fetchedData: await File.ReadAllTextAsync(file.FullName),
                     oldExpiry: null,
                     newExpiry: DateTime.UtcNow.AddDays(this.ExpiryDays),
                     warning: "This file was saved temporarily to the local computer. This should only happen in a local development environment."
@@ -161,7 +170,7 @@ public async Task<StoredFileInfo> GetAsync(string id, bool forceRenew)
         {
             PasteInfo response = await this.Pastebin.GetAsync(id);
             response.Content = this.GzipHelper.DecompressString(response.Content);
-            return new StoredFileInfo(response.Content, null, null, error: response.Error);
+            return new StoredFileInfo(null, response.Content, null, null, error: response.Error);
         }
     }
 
diff --git a/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs b/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs
@@ -10,11 +10,13 @@ internal class StoredFileInfo
     ** Accessors
     *********/
     /// <summary>Whether the file was successfully fetched.</summary>
-    [MemberNotNullWhen(true, nameof(StoredFileInfo.Content))]
-    public bool Success => this.Content != null && this.Error == null;
+    public bool Success => (this.FetchedData != null || this.FetchUri != null) && this.Error == null;
 
-    /// <summary>The fetched file content (if <see cref="Success"/> is <c>true</c>).</summary>
-    public string? Content { get; }
+    /// <summary>The URI from which to fetch the file, if available.</summary>
+    public string? FetchUri { get; }
+
+    /// <summary>The pre-fetched data, if a <see name="FetchUri"/> can't be provided for this file.</summary>
+    public string? FetchedData { get; }
 
     /// <summary>When the file would no longer have been available, before any renewal applied in this request.</summary>
     public DateTimeOffset? OldExpiry { get; }
@@ -33,14 +35,16 @@ internal class StoredFileInfo
     ** Public methods
     *********/
     /// <summary>Construct an instance.</summary>
-    /// <param name="content">The fetched file content (if <see cref="Success"/> is <c>true</c>).</param>
+    /// <param name="fetchUri">The URI from which to fetch the file, if available.</param>
+    /// <param name="fetchedData">The pre-fetched data, if a <paramref name="fetchUri"/> can't be provided for this file.</param>
     /// <param name="oldExpiry">When the file would no longer have been available, before any renewal applied in this request.</param>
     /// <param name="newExpiry">When the file will no longer be available, after any renewal applied in this request.</param>
     /// <param name="warning">The error message if saving succeeded, but a non-blocking issue was encountered.</param>
     /// <param name="error">The error message if saving failed.</param>
-    public StoredFileInfo(string? content, DateTimeOffset? oldExpiry, DateTimeOffset? newExpiry, string? warning = null, string? error = null)
+    public StoredFileInfo(string? fetchUri, string? fetchedData, DateTimeOffset? oldExpiry, DateTimeOffset? newExpiry, string? warning = null, string? error = null)
     {
-        this.Content = content;
+        this.FetchUri = fetchUri;
+        this.FetchedData = fetchedData;
         this.OldExpiry = oldExpiry;
         this.NewExpiry = newExpiry;
         this.Warning = warning;
diff --git a/src/SMAPI.Web/Framework/Storage/UploadResult.cs b/src/SMAPI.Web/Framework/Storage/UploadResult.cs
@@ -9,12 +9,12 @@ internal class UploadResult
     ** Accessors
     *********/
     /// <summary>Whether the file upload succeeded.</summary>
-    [MemberNotNullWhen(true, nameof(UploadResult.ID))]
+    [MemberNotNullWhen(true, nameof(UploadResult.Id))]
     [MemberNotNullWhen(false, nameof(UploadResult.UploadError))]
-    public bool Succeeded => this.ID != null && this.UploadError == null;
+    public bool Succeeded => this.Id != null && this.UploadError == null;
 
     /// <summary>The file ID, if applicable.</summary>
-    public string? ID { get; }
+    public string? Id { get; }
 
     /// <summary>The upload error, if any.</summary>
     public string? UploadError { get; }
@@ -28,7 +28,7 @@ internal class UploadResult
     /// <param name="uploadError">The upload error, if any.</param>
     public UploadResult(string? id, string? uploadError)
     {
-        this.ID = id;
+        this.Id = id;
         this.UploadError = uploadError;
     }
 }
diff --git a/src/SMAPI.Web/ViewModels/LogParserModel.cs b/src/SMAPI.Web/ViewModels/LogParserModel.cs
diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml
diff --git a/src/SMAPI.Web/wwwroot/Content/css/main.css b/src/SMAPI.Web/wwwroot/Content/css/main.css
diff --git a/src/SMAPI.Web/wwwroot/Content/js/log-parser.js b/src/SMAPI.Web/wwwroot/Content/js/log-parser.js
