Skip to content

Commit efbf386

Browse files
JustPlayerDEJustPlayerDE
committed
A small refactor i guess
1 parent 205ce55 commit efbf386

File tree

3 files changed

+175
-132
lines changed

3 files changed

+175
-132
lines changed

‎FastDL Generator/FastDL Generator.csproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@
6868
<ApplicationManifest>Properties\app.manifest</ApplicationManifest>
6969
</PropertyGroup>
7070
<ItemGroup>
71-
<Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
72-
<HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
71+
<Reference Include="ICSharpCode.SharpZipLib, Version=1.3.1.9, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
72+
<HintPath>..\packages\SharpZipLib.1.3.1\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
7373
</Reference>
7474
<Reference Include="System" />
7575
<Reference Include="System.Core" />

‎FastDL Generator/Program.cs

+172-129
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Collections.Concurrent;
34
using System.IO;
45
using System.Threading;
56
using ICSharpCode.SharpZipLib.BZip2;
@@ -9,194 +10,236 @@ namespace FastDL_Generator
910
class Program
1011
{
1112

12-
public static void Main(string[] args)
13+
public static int Main(string[] args)
1314
{
1415
Console.Title = "FastDL Generator";
15-
int RunningTasks = 0;
16-
int MaxTasks = 2;
17-
int HardLimit = 4;
18-
// Define Filetypes
1916
string[] SearchFor = new string[] {
20-
"materials/*.vmt",
21-
"materials/*.vtf",
22-
"materials/*.png",
23-
"materials/*.gif",
24-
"materials/*.jpg",
25-
"sound/*.wav",
26-
"sound/*.mp3",
27-
"sound/*.ogg",
28-
"maps/*.bsp",
29-
"maps/graphs/*.ain",
30-
"models/*.mdl",
31-
"models/*.vtx",
32-
"models/*.dx80.vtx",
33-
"models/*.dx90.vtf",
34-
"models/*.xbox.vtx",
35-
"models/*.sw.vtx",
36-
"models/*.vvd",
37-
"models/*.phy",
38-
"resource/*.ttf",
39-
"particles/*.pcf"
17+
"materials\\*.vmt",
18+
"materials\\*.vtf",
19+
"materials\\*.png",
20+
"materials\\*.gif",
21+
"materials\\*.jpg",
22+
"sound\\*.wav",
23+
"sound\\*.mp3",
24+
"sound\\*.ogg",
25+
"maps\\*.bsp",
26+
"maps\\graphs\\*.ain",
27+
"models\\*.mdl",
28+
"models\\*.vtx",
29+
"models\\*.dx80.vtx",
30+
"odels\\*.dx90.vtf",
31+
"models\\*.xbox.vtx",
32+
"models\\*.sw.vtx",
33+
"models\\*.vvd",
34+
"models\\*.phy",
35+
"resource\\*.ttf",
36+
"particles\\*.pcf"
4037
};
38+
string Path = "";
39+
string PathOutput = "";
40+
int Threads = 24;
41+
int RunningThreads = 0;
4142

42-
// Setting mainpath if arg[0] (first argument given to programm) is set
43-
string MainPath;
44-
try
45-
{
46-
MainPath = args[0];
47-
}
48-
catch (Exception)
43+
// Parsing arguments
44+
if(args.Length == 0)
4945
{
50-
Console.WriteLine("Please add a Path as first arg");
51-
return;
52-
}
46+
Console.WriteLine("Usage: ");
47+
Console.WriteLine("fastdlgen.exe <path to addon dir> (output path) (Threads) (-c)");
48+
Console.WriteLine("If no output path is given it will be put into '<path to addon dir>/upload_dir'");
49+
Console.WriteLine("If no amount of Threads are given it will default to 2 threads.");
50+
Console.WriteLine("The Output directory will always be cleaned.");
5351

54-
// Same as above but instead quitting just set default path as the target path
55-
string copyPath = MainPath + "/../FastDL_Upload/";
56-
try
52+
return 1;
53+
} else
5754
{
58-
if (Directory.Exists(args[1]))
59-
copyPath = args[1];
60-
}
61-
catch (Exception) { }
62-
63-
// if it exists Clear it because we need a clear folder
64-
if (Directory.Exists(copyPath))
65-
{
66-
Directory.Delete(copyPath, true);
67-
}
68-
69-
if (!Directory.Exists(MainPath))
70-
{
71-
Console.WriteLine("The given path doesnt exists: {0}", MainPath);
72-
return;
73-
}
55+
// Input path
56+
if (Directory.Exists(args[0]))
57+
{
58+
Path = args[0];
59+
Console.WriteLine($"Using {Path} as source.");
60+
PathOutput = $"{Path}/upload_dir";
61+
} else
62+
{
63+
Console.WriteLine($"{Path} was not found!");
64+
return 1;
65+
}
7466

7567

76-
List<string> IndexedFiles = new List<string>();
68+
// Output path
69+
if (args.Length >= 2)
70+
{
71+
try
72+
{
73+
PathOutput = args[1];
74+
} catch(Exception) { }
75+
}
76+
// Clean it up before use.
77+
if (Directory.Exists(PathOutput))
78+
{
79+
try
80+
{
81+
Directory.Delete(PathOutput, true);
82+
}
83+
catch (Exception)
84+
{
85+
Console.WriteLine($"Error creating {PathOutput}: Could not delete existing directory for cleanup.");
86+
return 1;
87+
}
88+
}
7789

78-
// Indexing files
79-
foreach (var Type in SearchFor)
80-
{
81-
string[] Data = TreeScan(MainPath, Type);
82-
foreach (var item in Data)
90+
try
8391
{
84-
IndexedFiles.Add(item.Substring(MainPath.Length + 1).Replace('\\', '/'));
92+
Directory.CreateDirectory(PathOutput);
93+
} catch(Exception e)
94+
{
95+
Console.WriteLine($"Error creating {PathOutput}: {e.Message}");
96+
return 1;
8597
}
86-
}
98+
Console.WriteLine($"Using {PathOutput} as output.");
8799

88-
if(IndexedFiles.Count > 2000)
89-
{
90-
MaxTasks = IndexedFiles.Count / 1000;
91-
if(MaxTasks > HardLimit)
100+
// Threads
101+
if (args.Length >= 3)
92102
{
93-
MaxTasks = 4;
103+
try
104+
{
105+
Threads = Int32.Parse(args[2]);
106+
}
107+
catch (Exception) { }
94108
}
109+
Console.WriteLine($"Using {Threads} Threads.");
95110
}
96111

97112
// Define first 2 lines for fastdl.lua
98113
string FileData = "// fastdl.lua generated by FastDL Generator.\n" +
99114
"if (SERVER) then\n";
100115

101-
// While copy files to target folder add line to File
102-
foreach (var item in IndexedFiles)
116+
// Indexing Files for copy
117+
var CopyQueue = new ConcurrentQueue<string>();
118+
var CompressQueue = new ConcurrentQueue<string>();
119+
120+
foreach (var Type in SearchFor)
103121
{
104-
Console.WriteLine("Copy > " + item);
105-
FileData = FileData + " resource.AddFile(\"" + item + "\")\n";
106-
CopyFile(item, MainPath, copyPath);
122+
string[] Data = TreeScan(Path, Type);
123+
foreach (var item in Data)
124+
{
125+
string path = item.Substring(Path.Length + 1).Replace('\\', '/');
126+
CopyQueue.Enqueue(path);
127+
CompressQueue.Enqueue(path);
128+
FileData = FileData + " resource.AddFile(\"" + path + "\")\n";
129+
}
107130
}
108131

109-
// and the end
110-
FileData = FileData + "end";
132+
Console.WriteLine($"Found {CopyQueue.Count} files to copy.");
133+
FileData = FileData + "end"; // Done i guess
134+
File.WriteAllText(PathOutput + "/fastdl.lua", FileData);
135+
111136

137+
// Copy files
138+
for (int i = 0; i < Threads; i++)
139+
{
140+
Thread temp = new Thread(new ThreadStart(ThreadedCopy));
141+
temp.Start();
142+
RunningThreads++;
143+
Console.WriteLine("Thread #{0} Started", temp.ManagedThreadId);
144+
}
112145

113-
// Bzip2 any file in the Index (in the target folder)
114-
Console.Title = "Bzipping...";
146+
// Lazy way of waiting for threads i guess but it works
147+
while (RunningThreads > 0) {
148+
Thread.Sleep(10);
149+
}
115150

116-
// Because the threads should NOT try to edit the same files at the same time ever
117-
List<string> ChangedFiles = new List<string> { };
118-
List<Thread> CurrentThreads = new List<Thread> { };
151+
Console.WriteLine("Compressing files...");
119152

120-
for (int i = 0; i < MaxTasks; i++)
153+
// Compress files
154+
for (int i = 0; i < Threads; i++)
121155
{
122156
Thread temp = new Thread(new ThreadStart(ThreadedCompressing));
123157
temp.Start();
124-
Console.WriteLine("Thread #{0} Started", i);
125-
CurrentThreads.Add(temp);
126-
RunningTasks++;
158+
RunningThreads++;
159+
Console.WriteLine("Thread #{0} Started", temp.ManagedThreadId);
127160
}
128161

129-
void ThreadedCompressing()
162+
// Lazy way of waiting for threads i guess but it works
163+
while (RunningThreads > 0)
164+
{
165+
Thread.Sleep(100);
166+
}
167+
168+
void ThreadedCopy()
130169
{
131-
long Edited = 0;
132-
foreach (string item in IndexedFiles)
170+
string currentFile;
171+
while (CopyQueue.TryDequeue(out currentFile))
133172
{
134-
if (!ChangedFiles.Contains(item))
173+
if (!Directory.Exists(PathOutput + "/" + currentFile))
135174
{
136-
ChangedFiles.Add(item);
137-
Edited++;
138-
} else
139-
{
140-
// if (ChangedFiles.Contains(item)) Console.WriteLine(" >>>Item: <" + item + "> Already changed..");
141-
continue;
175+
try
176+
{
177+
Directory.CreateDirectory(PathOutput + "/" + currentFile);
178+
Directory.Delete(PathOutput + "/" + currentFile); // hacky way
179+
}
180+
catch (Exception) { } // Ignore that
142181
}
182+
183+
// Copy file into new directory
143184
try
144185
{
145-
string Path = copyPath + "/" + item;
146-
147-
148-
if (File.Exists(Path) )
149-
{
150-
Console.WriteLine("compressing > " + item);
151-
BzipFile(Path);
152-
153-
Console.Title = " Compressed "+ChangedFiles.Count+" / "+IndexedFiles.Count+" Files, Running Threads: "+CurrentThreads.Count;
154-
}
186+
File.Copy($"{Path}/{currentFile}", $"{PathOutput}/{currentFile}", true);
187+
Console.WriteLine($"#{Thread.CurrentThread.ManagedThreadId} Copied: {Path}/{currentFile}");
155188
}
156189
catch (Exception)
157190
{
158-
191+
Console.WriteLine($"Error at Copy:\n{Path}/{currentFile} >>> {PathOutput}/{currentFile}");
159192
}
193+
Console.Title = $"FastDL Generator ({CopyQueue.Count} items in queue, {RunningThreads}/{Threads} Threads active)";
160194
}
161-
RunningTasks--;
162-
Console.WriteLine("Thread Killed with {0} compressed files. {1} Threads remaining please be patient...", Edited,RunningTasks);
163-
if(RunningTasks <= 0)
195+
RunningThreads--;
196+
Console.WriteLine($"Thread {Thread.CurrentThread.ManagedThreadId} Stopped");
197+
}
198+
199+
void ThreadedCompressing()
200+
{
201+
string currentFile;
202+
while(CompressQueue.TryDequeue(out currentFile))
164203
{
165-
Console.WriteLine("All Threads Killed, Generator closed");
204+
string filePath = PathOutput + "/" + currentFile;
166205

167-
// Save the fastdl.lua in the target folder
168-
File.WriteAllText(copyPath + "/fastdl.lua", FileData);
206+
try
207+
{
208+
if (File.Exists(filePath))
209+
{
210+
Console.WriteLine($"#{Thread.CurrentThread.ManagedThreadId} Compressing: {currentFile}");
211+
BzipFile(filePath);
212+
}
213+
} catch(Exception e)
214+
{
215+
Console.WriteLine($"#{Thread.CurrentThread.ManagedThreadId} Error: {e.Message}");
216+
CompressQueue.Enqueue(currentFile);
217+
if(File.Exists(filePath + ".bzip"))
218+
{
219+
File.Delete(filePath + ".bzip"); // At least dont download broken files
220+
}
221+
}
222+
Console.Title = $"FastDL Generator ({CompressQueue.Count} items in queue, {RunningThreads}/{Threads} Threads active)";
169223
}
224+
RunningThreads--;
225+
Console.WriteLine($"Thread {Thread.CurrentThread.ManagedThreadId} Stopped");
170226
}
171227

228+
Console.WriteLine("Done");
229+
Console.ReadKey();
230+
return 0;
172231
}
173232

174-
175-
private static void CopyFile(string Filee,string oldFolder, string NewFolder)
176-
{
177-
string oldFile = oldFolder+"/"+ Filee;
178-
string newFile = NewFolder+"/"+ Filee;
179-
Directory.CreateDirectory(newFile);
180-
Directory.Delete(newFile); // hacky way
181-
try
182-
{
183-
File.Copy(oldFile, newFile, true);
184-
}
185-
catch (Exception)
186-
{
187-
Console.WriteLine("Error at Copy:\n" + oldFile + " >>> " + newFile);
188-
}
189-
}
190233

191234
private static string[] TreeScan(string mainDir, string search)
192235
{
193236
try
194237
{
195-
return Directory.GetFiles(mainDir, search, SearchOption.AllDirectories);
238+
return Directory.GetFiles(mainDir, search, SearchOption.AllDirectories);
196239
}
197240
catch (Exception)
198241
{
199-
return new string[] { };
242+
return new string[] { };
200243
}
201244

202245
}
@@ -216,7 +259,7 @@ private static bool BzipFile(string Path)
216259
try
217260
{
218261
BZip2.Compress(fileToBeZippedAsStream, zipTargetAsStream, true, 4096);
219-
System.IO.File.Delete(Path);
262+
File.Delete(Path);
220263
}
221264
catch (Exception ex)
222265
{

‎FastDL Generator/packages.config

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<packages>
3-
<package id="SharpZipLib" version="0.86.0" targetFramework="net461" />
3+
<package id="SharpZipLib" version="1.3.1" targetFramework="net461" />
44
</packages>

0 commit comments

Comments
 (0)