Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions Analyzer/Resources/Init.sql
Original file line number Diff line number Diff line change
Expand Up @@ -151,12 +151,14 @@ WHERE m.type = 'Material';
-- see SerializedFileSQLiteWriter for details
INSERT INTO types (id, name) VALUES (-1, 'Scene');

-- Database schema version. Bump when the schema changes in a way that tools relying on it
-- (e.g. find-refs) cannot read from an older database. 1 = normalized refs table (issue #44);
-- 2 = renamed assets/asset_dependencies tables to assetbundle_assets/preload_dependencies
-- (issue #82); 3 = renamed asset_bundles table to archives and the asset_bundle column/alias
-- to archive (issue #68); databases produced before versioning report 0.
PRAGMA user_version = 3;
-- Database schema version. Bump on any schema change so the version records which schema a given
-- analyze.db was produced with. Commands that read an existing database (currently only find-refs)
-- can compare against it to give a clean error instead of failing on a missing table or column.
-- 1 = normalized refs table (issue #44); 2 = renamed assets/asset_dependencies tables to
-- assetbundle_assets/preload_dependencies (issue #82); 3 = renamed asset_bundles table to archives
-- and the asset_bundle column/alias to archive (issue #68); 4 = build_report_packed_asset_contents_view
-- type column changed from numeric id to type name (issue #55); databases produced before versioning report 0.
PRAGMA user_version = 4;

PRAGMA synchronous = OFF;
PRAGMA journal_mode = MEMORY;
5 changes: 4 additions & 1 deletion Analyzer/Resources/PackedAssets.sql
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ SELECT
pa.path,
pac.packed_assets_id,
pac.object_id,
pac.type,
-- Show the type name when known (populated from TypeIdRegistry or TypeTree analysis),
-- otherwise fall back to the numeric class id as text.
COALESCE(t.name, CAST(pac.type AS TEXT)) as type,
pac.size,
pac.offset,
sa.source_asset_guid,
Expand All @@ -56,6 +58,7 @@ LEFT JOIN build_report_packed_assets pa ON pac.packed_assets_id = pa.id
LEFT JOIN objects o ON o.id = pa.id
INNER JOIN serialized_files sf ON o.serialized_file = sf.id
LEFT JOIN build_report_source_assets sa ON pac.source_asset_id = sa.id
LEFT JOIN types t ON pac.type = t.id
LEFT JOIN objects br_obj ON o.serialized_file = br_obj.serialized_file AND br_obj.type = 1125
LEFT JOIN build_report_archive_contents brac ON br_obj.id = brac.build_report_id AND pa.path = brac.archive_content;

8 changes: 7 additions & 1 deletion Analyzer/SQLite/Commands/AbstractCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ internal abstract class AbstractCommand

protected virtual string DDLSource { get => null; }

// Conflict-resolution clause for the INSERT (e.g. "OR IGNORE"). Defaults to none, so a
// duplicate primary key surfaces as an error - some tables rely on that to detect problems
// such as the same SerializedFile being analyzed twice.
protected virtual string ConflictClause => "";

// run data definition language commands to create
// tables and views, run once at the beginning of creating
// the database
Expand All @@ -31,7 +36,8 @@ public void CreateCommand(SqliteConnection database)
RunDDL(database);

m_Command = database.CreateCommand();
var commandText = new StringBuilder($"INSERT INTO {TableName} (");
var insert = string.IsNullOrEmpty(ConflictClause) ? "INSERT" : $"INSERT {ConflictClause}";
var commandText = new StringBuilder($"{insert} INTO {TableName} (");
commandText.Append(string.Join(", ", Fields.Keys));
commandText.Append(") VALUES (@");
commandText.Append(string.Join(", @", Fields.Keys));
Expand Down
9 changes: 9 additions & 0 deletions Analyzer/SQLite/Commands/SerializedFile/AddType.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ internal class AddType : AbstractCommand

protected override string DDLSource => null;

// The BuildReport PackedAssetsHandler may have already inserted a type by numeric id (with a
// name from TypeIdRegistry), so ignore the conflict rather than crashing when both a report
// and its build output are analyzed together. First insert wins: the name for a given id is
// the same whether it comes from TypeIdRegistry or the TypeTree, so it does not matter which
// path runs first (a mismatch would indicate a deeper problem). The case that matters is a
// type missing from TypeIdRegistry (e.g. an older UnityDataTool reading a newer file); the
// handler skips those, so the name still arrives here from TypeTree analysis.
protected override string ConflictClause => "OR IGNORE";

protected override Dictionary<string, SqliteType> Fields => new()
{
{ "id", SqliteType.Integer },
Expand Down
33 changes: 28 additions & 5 deletions Analyzer/SQLite/Handlers/PackedAssetsHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using Microsoft.Data.Sqlite;
using UnityDataTools.Analyzer.SerializedObjects;
using UnityDataTools.FileSystem;
using UnityDataTools.FileSystem.TypeTreeReaders;

namespace UnityDataTools.Analyzer.SQLite.Handlers;
Expand All @@ -12,8 +13,14 @@ public class PackedAssetsHandler : ISQLiteHandler
private SqliteCommand m_InsertSourceAssetCommand;
private SqliteCommand m_GetSourceAssetIdCommand;
private SqliteCommand m_InsertContentsCommand;
private SqliteCommand m_InsertTypeCommand;
private Dictionary<(string guid, string path), long> m_SourceAssetCache = new();

// Type ids we have already tried to add to the types table. A single BuildReport can list
// thousands of objects sharing a handful of types, so this skips the redundant INSERT calls.
// The handler instance lives for the whole analyze, so this dedups across all reports.
private HashSet<int> m_InsertedTypes = new();

public void Init(SqliteConnection db)
{
using var command = db.CreateCommand();
Expand Down Expand Up @@ -59,6 +66,15 @@ public void Init(SqliteConnection db)
m_InsertContentsCommand.Parameters.Add("@size", SqliteType.Integer);
m_InsertContentsCommand.Parameters.Add("@offset", SqliteType.Integer);
m_InsertContentsCommand.Parameters.Add("@source_asset_id", SqliteType.Integer);

// A BuildReport records object types by numeric id only. When we can map an id to a
// human-readable name via TypeIdRegistry, we add it to the shared types table so the
// build report views can show the name. INSERT OR IGNORE keeps any name already inserted
// by TypeTree analysis of the actual build output (which is authoritative).
m_InsertTypeCommand = db.CreateCommand();
m_InsertTypeCommand.CommandText = "INSERT OR IGNORE INTO types(id, name) VALUES(@id, @name)";
m_InsertTypeCommand.Parameters.Add("@id", SqliteType.Integer);
m_InsertTypeCommand.Parameters.Add("@name", SqliteType.Text);
}

public void Process(Context ctx, long objectId, RandomAccessReader reader, out string name, out long streamDataSize)
Expand Down Expand Up @@ -93,14 +109,20 @@ public void Process(Context ctx, long objectId, RandomAccessReader reader, out s
m_SourceAssetCache[cacheKey] = sourceAssetId;
}

// Populate the types table so views can display the type name even when the build
// output (and its TypeTrees) is not analyzed alongside the report.
if (m_InsertedTypes.Add(content.Type) &&
TypeIdRegistry.TryGetTypeName(content.Type, out var typeName))
{
m_InsertTypeCommand.Transaction = ctx.Transaction;
Comment thread
SkowronskiAndrew marked this conversation as resolved.
m_InsertTypeCommand.Parameters["@id"].Value = content.Type;
m_InsertTypeCommand.Parameters["@name"].Value = typeName;
m_InsertTypeCommand.ExecuteNonQuery();
}

m_InsertContentsCommand.Transaction = ctx.Transaction;
m_InsertContentsCommand.Parameters["@packed_assets_id"].Value = objectId;
m_InsertContentsCommand.Parameters["@object_id"].Value = content.ObjectID;

// TODO: Ideally we would also populate the type table if the content.Type is
// not already in that table, and if we have a string value for it in TypeIdRegistry. That would
// make it possible to view object types as strings, for the most common types, when importing a BuildReport
// without the associated built content.
m_InsertContentsCommand.Parameters["@type"].Value = content.Type;
m_InsertContentsCommand.Parameters["@size"].Value = (long)content.Size;
m_InsertContentsCommand.Parameters["@offset"].Value = (long)content.Offset;
Expand All @@ -122,6 +144,7 @@ void IDisposable.Dispose()
m_InsertSourceAssetCommand?.Dispose();
m_GetSourceAssetIdCommand?.Dispose();
m_InsertContentsCommand?.Dispose();
m_InsertTypeCommand?.Dispose();
}
}

15 changes: 7 additions & 8 deletions Documentation/buildreport.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,21 +258,20 @@ For consistency and clarity, database columns use slightly different names than
| `build_report_packed_assets.path` | `PackedAssets.ShortPath` | Filename of the SerializedFile, .resS, or .resource file ("short" was redundant since only one path is recorded) |
| `build_report_packed_assets.file_header_size` | `PackedAssets.Overhead` | Size of the file header (zero for .resS and .resource files) |
| `build_report_packed_asset_info.object_id` | `PackedAssetInfo.fileID` | Local file ID of the object (renamed for consistency with `objects.object_id`) |
| `build_report_packed_asset_info.type` | `PackedAssetInfo.classID` | Unity object type (renamed for consistency with `objects.type`) |
| `build_report_packed_asset_info.type` | `PackedAssetInfo.classID` | Unity object type as a numeric [Class ID](https://docs.unity3d.com/Manual/ClassIDReference.html) (renamed for consistency with `objects.type`). `build_report_packed_asset_contents_view` exposes this as the type name. |

## Limitations

**Duplicate filenames:** Multiple build reports cannot be imported into the same database if they share the same filename. This is a general UnityDataTool limitation that assumes unique SerializedFile names.

**Type names:** While `build_report_packed_asset_info.type` records valid [Class IDs](https://docs.unity3d.com/Manual/ClassIDReference.html), the string type name may not exist in the `types` table. The `types` table is only populated when processing object instances (during TypeTree analysis). Analyzing both build output **and** build report together ensures types are fully populated; otherwise only numeric IDs are available.
**Duplicate filenames:** Multiple build reports cannot be imported into the same database if they share the same filename. This is a general UnityDataTool limitation that assumes unique SerializedFile names. The BuildReport files in the build history (introduced in Unity 6.6) intentionally have unique file names (incorporating the build session GUID), so analyze does not hit this issue when analyzing multiple entries from the BuildHistory folder. When analyzing multiple AssetBundle build reports, or build reports from older versions of Unity, be sure to assign a unique filename to each. See also issue #36.

### Information Not Exported

Currently, only the most useful BuildReport data is extracted to SQL. Additional data may be added as needed:

* [Code stripping](https://docs.unity3d.com/ScriptReference/Build.Reporting.BuildReport-strippingInfo.html) appendix (IL2CPP Player builds)
* [ScenesUsingAssets](https://docs.unity3d.com/ScriptReference/Build.Reporting.BuildReport-scenesUsingAssets.html) (detailed build reports)
* `BuildReport.m_BuildSteps` array (SQL may not be ideal for this hierarchical data)
* `BuildAssetBundleInfoSet` appendix (undocumented object listing files in each AssetBundle; `build_report_archive_contents` currently derives this from the File list)
* [Code stripping](https://docs.unity3d.com/ScriptReference/Build.Reporting.BuildReport-strippingInfo.html) appendix (available only for IL2CPP Player builds)
* [ScenesUsingAssets](https://docs.unity3d.com/ScriptReference/Build.Reporting.BuildReport-scenesUsingAssets.html) (available only in detailed build reports for Player builds)
* `BuildReport.m_BuildSteps` array. Supporting this is low priority - SQL is not an ideal representation for this hierarchical data. Starting in Unity 6.6 the build steps are also reported in the BuildLog.jsonl file and Trace Event Profile (TEP) files, which are easy to parse.
* `BuildAssetBundleInfoSet` appendix (undocumented object listing files in each AssetBundle; `build_report_archive_contents` currently derives this from the File list without relying on that data)
* Analytics-only appendices (unlikely to be valuable for analysis)

Tip: All this additional BuildReport data can be viewed using the `dump` command.
15 changes: 12 additions & 3 deletions UnityDataTool.Tests/BuildReportTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -311,10 +311,10 @@ public async Task Analyze_BuildReport_AssetBundle_ContainsPackedAssetsData()
// Verify the specific content row (data[3] from the dump)
const long objectId = -1350043613627603771;
var contentRow = SQLTestHelper.QueryInt(db,
$@"SELECT COUNT(*) FROM build_report_packed_asset_contents_view
WHERE packed_assets_id = {packedAssetId}
$@"SELECT COUNT(*) FROM build_report_packed_asset_contents_view
WHERE packed_assets_id = {packedAssetId}
AND object_id = {objectId}
AND type = 28
AND type = 'Texture2D'
AND size = 204
AND offset = 11840
AND source_asset_guid = '8826f464101b93c4bb006e15a9aff317'
Expand All @@ -323,6 +323,15 @@ public async Task Analyze_BuildReport_AssetBundle_ContainsPackedAssetsData()
Assert.AreEqual(1, contentRow,
"Expected exactly one packed_asset_contents row matching the specified criteria");

// The type column shows the human-readable name (from TypeIdRegistry) even though the
// build report records only the numeric class id and no build output was analyzed here.
SQLTestHelper.AssertQueryString(db,
$@"SELECT type FROM build_report_packed_asset_contents_view
WHERE packed_assets_id = {packedAssetId}
AND object_id = {objectId}",
"Texture2D",
"Expected type name 'Texture2D' in build_report_packed_asset_contents_view");

// Verify the view works correctly for this content row
SQLTestHelper.AssertQueryString(db,
$@"SELECT source_asset_guid FROM build_report_packed_asset_contents_view
Expand Down
10 changes: 10 additions & 0 deletions UnityFileSystem/TypeIdRegistry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -272,5 +272,15 @@ public static string GetTypeName(int typeId)
? name
: typeId.ToString();
}

/// <summary>
/// Looks up the type name for a TypeId, returning false when the id is not a known type.
/// Unlike GetTypeName, this does not fall back to the numeric id, so callers can tell
/// whether a real name is available.
/// </summary>
public static bool TryGetTypeName(int typeId, out string name)
{
return s_KnownTypes.TryGetValue(typeId, out name);
}
}

Loading