Skip to content

Commit 74bf384

Browse files
SkowronskiAndrewSkowronskiAndrew
authored
Document project structure; split Archive and SerializedFile into their own libraries (#88)
* More documentation of the project structure in main README * Restructure Archive and SerializedFile code into their own libraries For consistency with the other command implementations - it didn't make much sense that some commands were implemented directly inside UnityDataTool and others as separate libraries.
1 parent df445a6 commit 74bf384

13 files changed

Lines changed: 223 additions & 59 deletions

File tree

‎AGENTS.md‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,22 @@ UnityDataTool find-refs database.db -n "ObjectName" -t "Texture2D"
9090

9191
### Component Hierarchy
9292
```
93-
UnityDataTool (CLI executable)
93+
UnityDataTool (CLI executable) — wires the feature libraries into commands
9494
├── Analyzer → SQLite database generation
9595
├── TextDumper → Human-readable text output
96-
├── ReferenceFinder → Object reference chain tracing
97-
└── UnityFileSystem → C# wrapper for native library
98-
└── UnityFileSystemApi (native .dll/.dylib/.so)
96+
├── ReferenceFinder → Object reference chain tracing (queries the Analyzer database)
97+
├── Archive → inspect / extract Unity Archives (the `archive` command)
98+
├── SerializedFile → inspect SerializedFile header/metadata/objects (the `serialized-file` command)
99+
└── (base libraries, used by the feature libraries above)
100+
├── UnityBinaryFormat → C# parsers & helpers for Archives / SerializedFiles
101+
├── UnityDataModels → schemas for JSON build-report formats
102+
└── UnityFileSystem → C# wrapper for native library
103+
└── UnityFileSystemApi (native .dll/.dylib/.so)
99104
```
100105

106+
The feature libraries each build on UnityBinaryFormat and UnityFileSystem. See the
107+
"Repository content" section of `README.md` for a diagram of the dependencies.
108+
101109
### Key Architectural Patterns
102110

103111
**Native Interop**: UnityFileSystem wraps UnityFileSystemApi (native library from Unity Editor) via P/Invoke in `DllWrapper.cs`. The native library reads Unity Archive and SerializedFile formats.

‎Archive/Archive.csproj‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Library</OutputType>
5+
<TargetFramework>net9.0</TargetFramework>
6+
<ApplicationIcon />
7+
<StartupObject />
8+
<LangVersion>latest</LangVersion>
9+
</PropertyGroup>
10+
11+
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
12+
<PlatformTarget>AnyCPU</PlatformTarget>
13+
</PropertyGroup>
14+
15+
<PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
16+
<PlatformTarget>AnyCPU</PlatformTarget>
17+
</PropertyGroup>
18+
19+
<ItemGroup>
20+
<PackageReference Include="System.IO.Packaging" Version="7.0.0" />
21+
</ItemGroup>
22+
23+
<ItemGroup>
24+
<ProjectReference Include="..\UnityBinaryFormat\UnityBinaryFormat.csproj" />
25+
<ProjectReference Include="..\UnityFileSystem\UnityFileSystem.csproj" />
26+
</ItemGroup>
27+
28+
</Project>

UnityDataTool/Archive.cs renamed to Archive/ArchiveTool.cs

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,17 @@
55
using UnityDataTools.BinaryFormat;
66
using UnityDataTools.FileSystem;
77

8-
namespace UnityDataTools.UnityDataTool;
8+
namespace UnityDataTools.Archive;
99

10-
public static class Archive
10+
public static class ArchiveTool
1111
{
12-
public static int HandleExtract(FileInfo filename, DirectoryInfo outputFolder, string filter = null)
12+
public enum OutputFormat
13+
{
14+
Text,
15+
Json
16+
}
17+
18+
public static int ExtractContent(FileInfo filename, DirectoryInfo outputFolder, string filter = null)
1319
{
1420
try
1521
{
@@ -39,7 +45,7 @@ err is NotSupportedException
3945
return 0;
4046
}
4147

42-
public static int HandleList(FileInfo filename, OutputFormat format)
48+
public static int ListContent(FileInfo filename, OutputFormat format)
4349
{
4450
try
4551
{
@@ -70,7 +76,7 @@ err is NotSupportedException
7076
return 0;
7177
}
7278

73-
public static int HandleHeader(FileInfo filename, OutputFormat format)
79+
public static int PrintHeader(FileInfo filename, OutputFormat format)
7480
{
7581
var path = filename.ToString();
7682

@@ -94,7 +100,7 @@ public static int HandleHeader(FileInfo filename, OutputFormat format)
94100
return 0;
95101
}
96102

97-
public static int HandleBlocks(FileInfo filename, OutputFormat format)
103+
public static int ListBlocks(FileInfo filename, OutputFormat format)
98104
{
99105
var path = filename.ToString();
100106

@@ -124,7 +130,7 @@ public static int HandleBlocks(FileInfo filename, OutputFormat format)
124130
return 0;
125131
}
126132

127-
public static int HandleInfo(FileInfo filename, OutputFormat format)
133+
public static int PrintSummary(FileInfo filename, OutputFormat format)
128134
{
129135
var path = filename.ToString();
130136

‎Archive/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Archive
2+
3+
See [Documentation/command-archive.md](../Documentation/command-archive.md)

UnityDataTool/WebBundleHelper.cs renamed to Archive/WebBundleHelper.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
using System.Text;
77
using System.Text.Json;
88

9-
namespace UnityDataTools.UnityDataTool;
9+
namespace UnityDataTools.Archive;
1010

1111
public static class WebBundleHelper
1212
{
@@ -50,14 +50,14 @@ public static void Extract(FileInfo filename, DirectoryInfo outputFolder, string
5050
Console.WriteLine($"Extracted {extracted} out of {total} files.");
5151
}
5252

53-
public static void List(FileInfo filename, OutputFormat format)
53+
public static void List(FileInfo filename, ArchiveTool.OutputFormat format)
5454
{
5555
using var fileStream = File.Open(filename.ToString(), FileMode.Open);
5656
using var stream = GetStream(filename, fileStream);
5757
using var reader = new BinaryReader(stream, Encoding.UTF8);
5858
var fileDescriptions = ParseWebBundleHeader(reader);
5959

60-
if (format == OutputFormat.Json)
60+
if (format == ArchiveTool.OutputFormat.Json)
6161
{
6262
var jsonArray = new object[fileDescriptions.Count];
6363
for (int i = 0; i < fileDescriptions.Count; i++)

‎README.md‎

Lines changed: 79 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,84 @@ New to Unity's data files or to UnityDataTool? These topics are a good place to
3737

3838
## Repository content
3939

40-
The repository contains the following items:
41-
* [UnityDataTool](Documentation/unitydatatool.md): a command-line tool providing access to the Analyzer, TextDumper and other class libraries.
42-
* [Analyzer](Documentation/analyzer.md): a class library that can be used to extract key information
43-
from Unity data files and output it into a SQLite database.
44-
* [TextDumper](Documentation/textdumper.md): a class library that can be used to dump SerializedFiles into
45-
a human-readable format (similar to binary2text).
46-
* [ReferenceFinder](Documentation/referencefinder.md): a class library that can be used to find
47-
reference chains from objects to other objects using a database created by the Analyzer
48-
* UnityFileSystem: source code and binaries of a .NET class library exposing the functionalities or the
49-
UnityFileSystemApi native library.
50-
* UnityFileSystem.Tests: test suite for the UnityFileSystem library.
51-
* UnityProjects: Unity projects used to generate some of the test data.
52-
* TestCommon: a helper library used by the test projects.
40+
The solution is organised in three layers. At the base are the libraries that read Unity's binary
41+
formats. On top of those sit the feature libraries that turn that raw data into something useful
42+
(a database, a text dump, a reference graph). And at the top is the command-line tool that exposes
43+
all of that functionality to users.
44+
45+
```mermaid
46+
flowchart TD
47+
CLI["<b>UnityDataTool</b><br/>command-line tool"]
48+
49+
Analyzer["<b>Analyzer</b><br/>SQLite database"]
50+
TextDumper["<b>TextDumper</b><br/>human-readable dump"]
51+
ReferenceFinder["<b>ReferenceFinder</b><br/>reference chains"]
52+
Archive["<b>Archive</b><br/>inspect / extract archives"]
53+
SerializedFile["<b>SerializedFile</b><br/>inspect SerializedFiles"]
54+
55+
UnityFileSystem["<b>UnityFileSystem</b><br/>C# wrapper +<br/>UnityFileSystemApi (native)"]
56+
UnityBinaryFormat["<b>UnityBinaryFormat</b><br/>C# parsers &amp; helpers<br/>for Archives / SerializedFiles"]
57+
UnityDataModels["<b>UnityDataModels</b><br/>schemas for build reporting file formats"]
58+
59+
CLI --> Analyzer
60+
CLI --> TextDumper
61+
CLI --> ReferenceFinder
62+
CLI --> Archive
63+
CLI --> SerializedFile
64+
65+
Analyzer --> UnityFileSystem
66+
Analyzer --> UnityBinaryFormat
67+
Analyzer --> UnityDataModels
68+
TextDumper --> UnityFileSystem
69+
TextDumper --> UnityBinaryFormat
70+
Archive --> UnityFileSystem
71+
Archive --> UnityBinaryFormat
72+
SerializedFile --> UnityFileSystem
73+
SerializedFile --> UnityBinaryFormat
74+
UnityBinaryFormat --> UnityFileSystem
75+
76+
ReferenceFinder -. "reads the SQLite database" .-> Analyzer
77+
```
78+
79+
**Command-line tool**
80+
* [UnityDataTool](Documentation/unitydatatool.md): the command-line tool. It wires the feature
81+
libraries together and exposes them as commands (`analyze`, `dump`, `find-refs`, `archive`,
82+
`serialized-file`, and more), so most users only ever interact with this executable.
83+
84+
**Feature libraries**
85+
* [Analyzer](Documentation/analyzer.md): extracts key information from Unity data files into a SQLite
86+
database for detailed analysis. It also parses Addressables build reports.
87+
* [TextDumper](Documentation/textdumper.md): dumps SerializedFiles into a human-readable format
88+
(similar to Unity's binary2text).
89+
* [ReferenceFinder](Documentation/referencefinder.md): finds reference chains from one object to
90+
another by querying a database produced by the Analyzer (a data dependency rather than a code one).
91+
* [Archive](Documentation/command-archive.md): inspects and extracts the contents of Unity Archives
92+
(AssetBundles and web platform `.data` files) — the `archive` command.
93+
* [SerializedFile](Documentation/command-serialized-file.md): inspects the header, metadata, object
94+
list, and external references of a SerializedFile — the `serialized-file` command.
95+
96+
**Base libraries**
97+
* UnityFileSystem: a .NET class library, with source and binaries, that wraps the native
98+
`UnityFileSystemApi` to mount Unity Archives and read SerializedFiles.
99+
* UnityBinaryFormat: C# parsers and helpers for reading data out of Unity Archives and SerializedFiles.
100+
* UnityDataModels: shared C# models for the reading JSON format files produced by the build (Addressables BuildLayout.json, Content Directory ContentLayout.json).
101+
102+
## Purpose of UnityFileSystemApi
103+
104+
UnityFileSystemApi is compiled from the Unity source code and exposes the core functionality to open and read the Unity Archive and Serialized File formats as a flexible, performant library. It exposes the ability to navigate the TypeTrees inside a SerializedFile so objects can be read generically, without hardcoded type knowledge.
105+
106+
It enables custom tools for binary2text-like output and efficient SQLite database generation.
107+
108+
### Tests and test data
109+
110+
The automated tests are not shown in the diagram above. Each library has its own test project, and the
111+
shared test data doubles as convenient sample content for ad hoc use of the tool.
112+
113+
* TestCommon: a helper library whose `Data/` folder holds small reference files extracted from real
114+
Unity builds (Player, AssetBundles, content directory etc). These back the automated tests and are also handy for trying out the tool by hand.
115+
* Analyzer.Tests, UnityDataTool.Tests, UnityFileSystem.Tests: the per-library test suites.
116+
* UnityProjects: two Unity projects (`Baseline` and `LeadingEdge`) used to regenerate some of the test
117+
data as Unity evolves.
53118

54119
## Downloads
55120

@@ -63,7 +128,7 @@ Refer to the [commit history](https://github.com/Unity-Technologies/UnityDataToo
63128

64129
## Getting UnityFileSystemApi
65130

66-
UnityDataTool uses the native `UnityFileSystemApi` library to read Unity Archives and SerializedFiles. **Normally you don't need to do anything with this library.** The repository already includes a recent Windows, Mac, and Linux copy in the [`UnityFileSystem/`](https://github.com/Unity-Technologies/UnityDataTools/tree/main/UnityFileSystem) directory, and using that bundled copy is the recommended way to run the tool.
131+
UnityDataTool uses the pre-compiled `UnityFileSystemApi` library to read Unity Archives and SerializedFiles. **Normally you don't need to do anything with this library.** The repository already includes a recent Windows, Mac, and Linux copy in the [`UnityFileSystem/`](https://github.com/Unity-Technologies/UnityDataTools/tree/main/UnityFileSystem) directory, and using that bundled copy is the recommended way to run the tool.
67132

68133
The library is backward compatible but not forward compatible: a given version can read content from the same or older Unity versions, but may be unable to read content produced by a newer Unity Editor than the library itself. The bundled copy is updated periodically as Unity evolves, so in practice it can read content from just about any Unity version.
69134

@@ -84,10 +149,6 @@ On Windows, the executable is written to `UnityDataTool\bin\Release\net9.0`. Add
84149

85150
See the [command-line tool documentation](./Documentation/unitydatatool.md) for usage instructions.
86151

87-
## Purpose of UnityFileSystemApi
88-
89-
UnityFileSystemApi exposes the core functionality of WebExtract and binary2text to open and read the Unity Archive and Serialized File formats, exposed as a flexible, performant library. It enables custom tools for binary2text-like output and efficient SQLite database generation.
90-
91152
## Origins
92153

93154
This tool is the evolution of the [AssetBundle Analyzer](https://github.com/faelenor/asset-bundle-analyzer)

‎SerializedFile/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# SerializedFile
2+
3+
See [Documentation/command-serialized-file.md](../Documentation/command-serialized-file.md)

‎SerializedFile/SerializedFile.csproj‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Library</OutputType>
5+
<TargetFramework>net9.0</TargetFramework>
6+
<ApplicationIcon />
7+
<StartupObject />
8+
<LangVersion>latest</LangVersion>
9+
</PropertyGroup>
10+
11+
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
12+
<PlatformTarget>AnyCPU</PlatformTarget>
13+
</PropertyGroup>
14+
15+
<PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
16+
<PlatformTarget>AnyCPU</PlatformTarget>
17+
</PropertyGroup>
18+
19+
<ItemGroup>
20+
<ProjectReference Include="..\UnityBinaryFormat\UnityBinaryFormat.csproj" />
21+
<ProjectReference Include="..\UnityFileSystem\UnityFileSystem.csproj" />
22+
</ItemGroup>
23+
24+
</Project>

UnityDataTool/SerializedFileCommands.cs renamed to SerializedFile/SerializedFileTool.cs

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,17 @@
55
using UnityDataTools.BinaryFormat;
66
using UnityDataTools.FileSystem;
77

8-
namespace UnityDataTools.UnityDataTool;
8+
namespace UnityDataTools.SerializedFile;
99

10-
public static class SerializedFileCommands
10+
public static class SerializedFileTool
1111
{
12-
public static int HandleExternalRefs(FileInfo filename, OutputFormat format)
12+
public enum OutputFormat
13+
{
14+
Text,
15+
Json
16+
}
17+
18+
public static int ListExternalRefs(FileInfo filename, OutputFormat format)
1319
{
1420
// External references are read directly from the parsed metadata rather than via UnityFileSystemApi.
1521
//
@@ -44,10 +50,10 @@ public static int HandleExternalRefs(FileInfo filename, OutputFormat format)
4450
return 0;
4551
}
4652

47-
public static int HandleObjectList(FileInfo filename, OutputFormat format)
53+
public static int ListObjects(FileInfo filename, OutputFormat format)
4854
{
4955
// The object list is read directly from the parsed metadata rather than via UnityFileSystemApi.
50-
// (See comment in HandleExternalRefs() for the reasons for doing it that way)
56+
// (See comment in ListExternalRefs() for the reasons for doing it that way)
5157
if (!ValidateSerializedFile(filename.FullName, out var fileInfo))
5258
return 1;
5359

@@ -72,7 +78,7 @@ public static int HandleObjectList(FileInfo filename, OutputFormat format)
7278
return 0;
7379
}
7480

75-
public static int HandleHeader(FileInfo filename, OutputFormat format)
81+
public static int PrintHeader(FileInfo filename, OutputFormat format)
7682
{
7783
if (!ValidateSerializedFile(filename.FullName, out var fileInfo))
7884
return 1;
@@ -85,7 +91,7 @@ public static int HandleHeader(FileInfo filename, OutputFormat format)
8591
return 0;
8692
}
8793

88-
public static int HandleMetadata(FileInfo filename, OutputFormat format)
94+
public static int PrintMetadata(FileInfo filename, OutputFormat format)
8995
{
9096
if (!ValidateSerializedFile(filename.FullName, out var fileInfo))
9197
return 1;

‎UnityDataTool.Tests/WebBundleSupportTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Linq;
66
using System.Threading.Tasks;
77
using NUnit.Framework;
8+
using UnityDataTools.Archive;
89
using UnityDataTools.FileSystem;
910

1011
namespace UnityDataTools.UnityDataTool.Tests;

0 commit comments

Comments
 (0)