Skip to content

Commit ee372f1

Browse files
francis-pagefrancis-page
committed
Initial commit
1 parent 6c35c3b commit ee372f1

File tree

108 files changed

+28808
-45
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+28808
-45
lines changed

Diff for: ‎.gitignore

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[Aa]rtifacts/
2+
[Bb]uild/
3+
[Ll]ibrary/
4+
[Oo]bj/
5+
[Tt]emp/
6+
[Ll]og/
7+
[Ll]ogs/
8+
9+
**/*.csproj.user
10+
**/*.suo
11+
**/.vs
12+
**/.vsconfig
13+
**/*.vcxproj.user
14+
**/*.sdf
15+
**/*.opensdf
16+
**/*.VC.db
17+
**/*.VC.opendb
18+
**/*.sln.DotSettings.user
19+
**/*.userprefs
20+
**/project.lock.json
21+
**/*.gen.csproj
22+
**/*.gen.sln
23+
**/.vscode
24+
**/bin/
25+
**/obj/
26+
**/.idea/
27+
**/launchSettings.json
28+
**/PublishProfiles/
29+
30+
UnityFileSystemTestData/AssetBundles/
31+
UnityFileSystemTestData/**/*.csproj
32+
UnityFileSystemTestData/**/*.sln

Diff for: ‎Analyzer/Analyzer.csproj

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net5.0</TargetFramework>
5+
</PropertyGroup>
6+
7+
<ItemGroup>
8+
<PackageReference Include="System.Data.SQLite" Version="1.0.115" />
9+
</ItemGroup>
10+
11+
<ItemGroup>
12+
<ProjectReference Include="..\UnityFileSystem\UnityFileSystem.csproj" />
13+
</ItemGroup>
14+
15+
<ItemGroup>
16+
<Compile Update="Properties\Resources.Designer.cs">
17+
<DependentUpon>Resources.resx</DependentUpon>
18+
<DesignTime>True</DesignTime>
19+
<AutoGen>True</AutoGen>
20+
</Compile>
21+
</ItemGroup>
22+
23+
<ItemGroup>
24+
<EmbeddedResource Update="Properties\Resources.resx">
25+
<SubType>Designer</SubType>
26+
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
27+
<Generator>ResXFileCodeGenerator</Generator>
28+
</EmbeddedResource>
29+
</ItemGroup>
30+
31+
</Project>

Diff for: ‎Analyzer/AnalyzerTool.cs

+294
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Data;
4+
using System.Data.SQLite;
5+
using System.Diagnostics;
6+
using System.IO;
7+
using UnityDataTools.FileSystem;
8+
using UnityDataTools.FileSystem.TypeTreeReaders;
9+
10+
11+
namespace UnityDataTools.Analyzer
12+
{
13+
public class AnalyzerTool
14+
{
15+
HashSet<int> m_TypeSet = new HashSet<int>();
16+
17+
int m_NextAssetBundleId = 0;
18+
int m_NextSerializedFileId = 0;
19+
long m_NextObjectId = 0;
20+
bool m_extractReferences = false;
21+
22+
Dictionary<string, int> m_SerializedFilenameToId = new Dictionary<string, int>();
23+
Dictionary<(int, long), long> m_PPtrToId = new Dictionary<(int, long), long>();
24+
Dictionary<string, Processors.IProcessor> m_Processors = new Dictionary<string, Processors.IProcessor>();
25+
26+
SQLiteCommand m_AddReferenceCommand;
27+
28+
public int GetSerializedFileId(string filename)
29+
{
30+
if (m_SerializedFilenameToId.TryGetValue(filename, out var id))
31+
{
32+
return id;
33+
}
34+
35+
m_SerializedFilenameToId.Add(filename, m_NextSerializedFileId);
36+
37+
return m_NextSerializedFileId++;
38+
}
39+
40+
public long GetObjectId(int fileId, long pathId)
41+
{
42+
if (m_PPtrToId.TryGetValue((fileId, pathId), out var id))
43+
{
44+
return id;
45+
}
46+
47+
m_PPtrToId.Add((fileId, pathId), m_NextObjectId);
48+
49+
return m_NextObjectId++;
50+
}
51+
52+
public void AddProcessor(string typeName, Processors.IProcessor processor)
53+
{
54+
m_Processors.Add(typeName, processor);
55+
}
56+
57+
public int Analyze(string path, string databaseName, string searchPattern, bool extractReferences)
58+
{
59+
m_extractReferences = extractReferences;
60+
61+
using var db = new SQLiteConnection($"Data Source={databaseName};Version=3;New=True;Foreign Keys=False;");
62+
try
63+
{
64+
SQLiteConnection.CreateFile(databaseName);
65+
db.Open();
66+
}
67+
catch (Exception e)
68+
{
69+
Console.Error.WriteLine($"Error creating database: {e.Message}");
70+
return 1;
71+
}
72+
73+
using var command = db.CreateCommand();
74+
command.CommandText = Properties.Resources.Init;
75+
command.ExecuteNonQuery();
76+
77+
foreach (var processor in m_Processors.Values)
78+
{
79+
processor.Init(db);
80+
}
81+
82+
using var addAssetBundleCommand = db.CreateCommand();
83+
addAssetBundleCommand.CommandText = "INSERT INTO asset_bundles (id, name, file_size) VALUES (@id, @name, @file_size)";
84+
addAssetBundleCommand.Parameters.Add("@id", DbType.Int32);
85+
addAssetBundleCommand.Parameters.Add("@name", DbType.String);
86+
addAssetBundleCommand.Parameters.Add("@file_size", DbType.Int64);
87+
88+
using var addSerializedFileCommand = db.CreateCommand();
89+
addSerializedFileCommand.CommandText = "INSERT INTO serialized_files (id, asset_bundle, name) VALUES (@id, @asset_bundle, @name)";
90+
addSerializedFileCommand.Parameters.Add("@id", DbType.Int32);
91+
addSerializedFileCommand.Parameters.Add("@asset_bundle", DbType.Int32);
92+
addSerializedFileCommand.Parameters.Add("@name", DbType.String);
93+
94+
m_AddReferenceCommand = db.CreateCommand();
95+
m_AddReferenceCommand.CommandText = "INSERT INTO refs (object, referenced_object, property_path) VALUES (@object, @referenced_object, @property_path)";
96+
m_AddReferenceCommand.Parameters.Add("@object", DbType.Int64);
97+
m_AddReferenceCommand.Parameters.Add("@referenced_object", DbType.Int64);
98+
m_AddReferenceCommand.Parameters.Add("@property_path", DbType.String);
99+
100+
var timer = new Stopwatch();
101+
timer.Start();
102+
103+
var files = Directory.GetFiles(path, searchPattern, SearchOption.AllDirectories);
104+
int i = 0;
105+
int lastLength = 0;
106+
foreach (var file in files)
107+
{
108+
try
109+
{
110+
try
111+
{
112+
using var archive = UnityFileSystem.MountArchive(file, "/");
113+
var assetBundleId = m_NextAssetBundleId++;
114+
var assetBundleName = Path.GetRelativePath(path, file);
115+
116+
var message = $"Processing { i * 100 / files.Length}% ({ i}/{ files.Length}) { assetBundleName}";
117+
Console.Write($"\r{message}{new string(' ', Math.Max(0, lastLength - message.Length))}");
118+
lastLength = message.Length;
119+
120+
addAssetBundleCommand.Parameters["@id"].Value = assetBundleId;
121+
addAssetBundleCommand.Parameters["@name"].Value = assetBundleName;
122+
addAssetBundleCommand.Parameters["@file_size"].Value = new FileInfo(file).Length;
123+
addAssetBundleCommand.ExecuteNonQuery();
124+
125+
foreach (var node in archive.Nodes)
126+
{
127+
if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile))
128+
{
129+
using var transaction = db.BeginTransaction();
130+
131+
try
132+
{
133+
int serializedFileId = GetSerializedFileId(node.Path.ToLower());
134+
addSerializedFileCommand.Parameters["@id"].Value = serializedFileId;
135+
addSerializedFileCommand.Parameters["@asset_bundle"].Value = assetBundleId;
136+
addSerializedFileCommand.Parameters["@name"].Value = node.Path;
137+
addSerializedFileCommand.ExecuteNonQuery();
138+
139+
ProcessSerializedFile("/" + node.Path, serializedFileId, db);
140+
transaction.Commit();
141+
}
142+
catch
143+
{
144+
transaction.Rollback();
145+
throw;
146+
}
147+
}
148+
}
149+
}
150+
catch (NotSupportedException)
151+
{
152+
using var transaction = db.BeginTransaction();
153+
154+
try
155+
{
156+
Console.SetCursorPosition(0, Console.CursorTop);
157+
Console.Write(new string(' ', Console.BufferWidth));
158+
Console.Write($"\rProcessing {i * 100 / files.Length}% ({i}/{files.Length}) {file}");
159+
160+
int serializedFileId = GetSerializedFileId(file.ToLower());
161+
addSerializedFileCommand.Parameters["@id"].Value = serializedFileId;
162+
addSerializedFileCommand.Parameters["@asset_bundle"].Value = null;
163+
addSerializedFileCommand.Parameters["@name"].Value = file;
164+
addSerializedFileCommand.ExecuteNonQuery();
165+
166+
ProcessSerializedFile("/" + file, serializedFileId, db);
167+
transaction.Commit();
168+
}
169+
catch
170+
{
171+
transaction.Rollback();
172+
throw;
173+
}
174+
}
175+
}
176+
catch (Exception)
177+
{
178+
Console.Error.WriteLine();
179+
Console.Error.WriteLine($"Error processing file {file}.");
180+
}
181+
182+
++i;
183+
}
184+
185+
Console.WriteLine();
186+
Console.WriteLine("Finalizing database...");
187+
using var finalizeCommand = db.CreateCommand();
188+
finalizeCommand.CommandText = Properties.Resources.Finalize;
189+
finalizeCommand.ExecuteNonQuery();
190+
191+
timer.Stop();
192+
Console.WriteLine();
193+
Console.WriteLine($"Total time: {(timer.Elapsed.TotalMilliseconds / 1000.0):F3} s");
194+
195+
m_AddReferenceCommand.Dispose();
196+
197+
return 0;
198+
}
199+
200+
void ProcessSerializedFile(string path, int serializedFileId, SQLiteConnection db)
201+
{
202+
using var reader = new UnityFileReader(path, 64 * 1024 * 1024);
203+
using var sf = UnityFileSystem.OpenSerializedFile(path);
204+
205+
// Used to map PPtr fileId to its corresponding serialized file id in the database.
206+
var localToDbFileId = new Dictionary<int, int>();
207+
208+
using var addObjectCommand = db.CreateCommand();
209+
addObjectCommand.CommandText = "INSERT INTO objects (id, object_id, serialized_file, type, name, game_object, size) VALUES (@id, @object_id, @serialized_file, @type, @name, @game_object, @size)";
210+
addObjectCommand.Parameters.Add("@id", DbType.Int64);
211+
addObjectCommand.Parameters.Add("@object_id", DbType.Int64);
212+
addObjectCommand.Parameters.Add("@serialized_file", DbType.Int32);
213+
addObjectCommand.Parameters.Add("@type", DbType.Int32);
214+
addObjectCommand.Parameters.Add("@name", DbType.String);
215+
addObjectCommand.Parameters.Add("@game_object", DbType.Int64);
216+
addObjectCommand.Parameters.Add("@size", DbType.Int64);
217+
218+
using var addTypeCommand = db.CreateCommand();
219+
addTypeCommand.CommandText = "INSERT INTO types (id, name) VALUES (@id, @name)";
220+
addTypeCommand.Parameters.Add("@id", DbType.Int32);
221+
addTypeCommand.Parameters.Add("@name", DbType.String);
222+
223+
int localId = 0;
224+
localToDbFileId.Add(localId++, serializedFileId);
225+
foreach (var extRef in sf.ExternalReferences)
226+
{
227+
localToDbFileId.Add(localId++, GetSerializedFileId(extRef.Path.Substring(extRef.Path.LastIndexOf('/') + 1).ToLower()));
228+
}
229+
230+
foreach (var obj in sf.Objects)
231+
{
232+
var currentObjectId = GetObjectId(serializedFileId, obj.Id);
233+
234+
var root = sf.GetTypeTreeRoot(obj.Id);
235+
var offset = obj.Offset;
236+
237+
if (!m_TypeSet.Contains(obj.TypeId))
238+
{
239+
addTypeCommand.Parameters["@id"].Value = obj.TypeId;
240+
addTypeCommand.Parameters["@name"].Value = root.Type;
241+
addTypeCommand.ExecuteNonQuery();
242+
243+
m_TypeSet.Add(obj.TypeId);
244+
}
245+
246+
var randomAccessReader = new RandomAccessReader(root, reader, offset);
247+
248+
string name = null;
249+
long streamedDataSize = 0;
250+
251+
if (m_Processors.TryGetValue(root.Type, out var processor))
252+
{
253+
processor.Process(this, currentObjectId, localToDbFileId, randomAccessReader, out name, out streamedDataSize);
254+
}
255+
else if (randomAccessReader.HasChild("m_Name"))
256+
{
257+
name = randomAccessReader["m_Name"].GetValue<string>();
258+
}
259+
260+
if (randomAccessReader.HasChild("m_GameObject"))
261+
{
262+
var pptr = randomAccessReader["m_GameObject"];
263+
var fileId = localToDbFileId[pptr["m_FileID"].GetValue<int>()];
264+
addObjectCommand.Parameters["@game_object"].Value = GetObjectId(fileId, pptr["m_PathID"].GetValue<long>());
265+
}
266+
else
267+
{
268+
addObjectCommand.Parameters["@game_object"].Value = null;
269+
}
270+
271+
addObjectCommand.Parameters["@id"].Value = currentObjectId;
272+
addObjectCommand.Parameters["@object_id"].Value = obj.Id;
273+
addObjectCommand.Parameters["@serialized_file"].Value = serializedFileId;
274+
addObjectCommand.Parameters["@type"].Value = obj.TypeId;
275+
addObjectCommand.Parameters["@name"].Value = name;
276+
addObjectCommand.Parameters["@size"].Value = obj.Size + streamedDataSize;
277+
addObjectCommand.ExecuteNonQuery();
278+
279+
if (m_extractReferences)
280+
{
281+
var pptrReader = new PPtrReader(root, reader, offset, (fileId, pathId, propertyPath) => AddReference(currentObjectId, GetObjectId(localToDbFileId[fileId], pathId), propertyPath));
282+
}
283+
}
284+
}
285+
286+
void AddReference(long objectId, long referencedObjectId, string propertyPath)
287+
{
288+
m_AddReferenceCommand.Parameters["@object"].Value = objectId;
289+
m_AddReferenceCommand.Parameters["@referenced_object"].Value = referencedObjectId;
290+
m_AddReferenceCommand.Parameters["@property_path"].Value = propertyPath;
291+
m_AddReferenceCommand.ExecuteNonQuery();
292+
}
293+
}
294+
}

0 commit comments

Comments
 (0)