Skip to content

Commit

Permalink
Completed the loader program
Browse files Browse the repository at this point in the history
  • Loading branch information
@BlueSkeye
BlueSkeye committed Oct 4, 2016
1 parent 3dff3c9 commit 01e2a9a
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 40 deletions.
94 changes: 88 additions & 6 deletions SSUpdate/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,19 @@ private static string BuildSqlServerTableCreationInstruction(string tableName)
return builder.ToString();
}

private static void CleanUpDb()
{
using(SqlConnection connection = ConnectToSqlServer()) {
for(int index = CleanUpScripts.Length - 1; 0 <= index; index--) {
using (SqlCommand command = new SqlCommand(CleanUpScripts[index], connection)) {
command.CommandTimeout = 60;
command.ExecuteNonQuery();
Console.Write(".");
}
}
}
}

private static SQLiteConnection ConnectToSQLite()
{
SQLiteConnection result = TryConnectToSQLite();
Expand All @@ -219,6 +232,7 @@ private static void DisplayUsage(string programName)
{
Console.WriteLine("{0} [-f <database path> -s <target SQL Server> -d <Sql Server database>|-h]");
Console.WriteLine();
Console.WriteLine("-c : (optional) Perform cleanup before loading.");
Console.WriteLine("-d : (optional) Sql Server vFeed database name (default vFeed).");
Console.WriteLine("-f : path to vFeed database.");
Console.WriteLine("-h : display this help notice.");
Expand Down Expand Up @@ -299,7 +313,7 @@ private static bool ImportData()
string insertCommand = GetInsertCommand(tableName, out parametersCount);
using (SQLiteCommand command = new SQLiteCommand(readCommand, sqliteConnection)) {
using(SQLiteDataReader reader = command.ExecuteReader()) {
Transfer(sqlServerConnection, reader, insertCommand, parametersCount);
Transfer(tableName, sqlServerConnection, reader, insertCommand, parametersCount);
}
}
}
Expand Down Expand Up @@ -358,6 +372,11 @@ public static int Main(string[] args)
DisplayUsage(programName);
return result;
}
if (_doCleanup) {
Console.WriteLine("Cleaning up.");
CleanUpDb();
Console.WriteLine();
}
if (!LoadSQLiteSchema()) { return 2; }
if (!ImportData()) { return 3; }
return result;
Expand All @@ -373,6 +392,9 @@ private static bool ParseArgs(string[] args)
}
else {
switch (scannedArg.Substring(1).ToUpper()) {
case "C":
_doCleanup = true;
break;
case "F":
string dbPath = TryGetAdditionalArgument(args, ref argIndex);
if (null == dbPath) { return false; }
Expand Down Expand Up @@ -425,8 +447,8 @@ private static bool ParseArgs(string[] args)
return true;
}

private static void Transfer(SqlConnection sqlServerConnection, SQLiteDataReader reader,
string insertCommandText, int parametersCount)
private static void Transfer(string tableName, SqlConnection sqlServerConnection,
SQLiteDataReader reader, string insertCommandText, int parametersCount)
{
int linesCount = 0;
using (SqlCommand insertCommand = new SqlCommand(insertCommandText, sqlServerConnection)) {
Expand All @@ -438,7 +460,25 @@ private static void Transfer(SqlConnection sqlServerConnection, SQLiteDataReader
for(int index = 0; index < parametersCount; index++) {
parameters[index].Value = reader.GetValue(index);
}
insertCommand.ExecuteNonQuery();
Retry:
try { insertCommand.ExecuteNonQuery(); }
catch (SqlException e) {
switch (e.Number) {
case 0x223:
Console.WriteLine("Foreign key reference error. Faulting value is {0}",
parameters[parameters.Length - 1].Value);
break;
case 0x1FD8:
if ("map_cve_mskb" != tableName) { goto default; }
if (!(parameters[0].Value is string) || (parameters[1].Value is int)) { goto default; }
object trash = parameters[0].Value;
parameters[0].Value = parameters[1].Value;
parameters[1].Value = trash;
goto Retry;
default:
throw;
}
}
if (0 == (++linesCount % 1000)) { Console.Write("."); }
}
Console.WriteLine("\r\n\r{0} lines imported.", linesCount);
Expand Down Expand Up @@ -522,13 +562,12 @@ private static void VerifyTableDescription(SQLiteConnection connection,
private const int TableInfoPrimaryKeyFlagColumIndex = 5;
private static FileInfo _database;
private static bool _displayUsage;
private static bool _doCleanup;
#if DEBUG
/// <summary>For debugging purpose only. This allow us to bypass
/// already loaded tables and quicker fix a failing table.</summary>
/// <remarks>This table should be empty during normal use.</remarks>
private static readonly string[] _ignoreTables = new string[] {
"nvd_db", "cwe_db", "cve_cwe", "cwe_category", "cwe_capec", "cve_cpe",
"cve_reference", "map_cve_aixapar"
};
#endif
private static List<string> _inInsertionOrderTableNames;
Expand All @@ -548,5 +587,48 @@ private struct ColumnInfo
internal int PrimaryKey { get; set; }
internal string Type { get; set; }
}

private static readonly string[] CleanUpScripts = new string[] {
"DELETE nvd_db;",
"DELETE cwe_db;",
"DELETE cve_cwe;",
"DELETE cwe_category;",
"DELETE cwe_capec;",
"DELETE cve_cpe;",
"DELETE cve_reference;",
"DELETE map_cve_aixapar;",
"DELETE map_cve_redhat;",
"DELETE map_redhat_bugzilla;",
"DELETE map_cve_suse;",
"DELETE map_cve_debian;",
"DELETE map_cve_mandriva;",
"DELETE map_cve_saint;",
"DELETE map_cve_milw0rm;",
"DELETE map_cve_osvdb;",
"DELETE map_cve_nessus;",
"DELETE map_cve_msf;",
"DELETE map_cve_openvas;",
"DELETE map_cve_scip;",
"DELETE map_cve_iavm;",
"DELETE map_cve_cisco;",
"DELETE map_cve_ubuntu;",
"DELETE map_cve_gentoo;",
"DELETE map_cve_fedora;",
"DELETE map_cve_certvn;",
"DELETE map_cve_ms;",
"DELETE map_cve_mskb;",
"DELETE map_cve_snort;",
"DELETE map_cve_suricata;",
"DELETE map_cve_vmware;",
"DELETE map_cve_bid;",
"DELETE map_cve_hp;",
"DELETE stat_new_cve;",
"DELETE map_cve_exploitdb;",
"DELETE map_cve_nmap;",
"DELETE map_cve_oval;",
"DELETE map_cve_d2;",
"DELETE stat_vfeed_kpi;",
"DELETE capec_db;"
};
}
}
68 changes: 34 additions & 34 deletions SSUpdate/SqlServerSchema.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ internal static class SqlServerSchema
" [redhatid] sysname NULL," +
" [redhatovalid] sysname NULL," +
" [redhatupdatedesc] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveRedhatNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_redhat_bugzilla] (" +
" [advisory_dateissue] sysname NULL," +
" [bugzillaid] sysname NULL," +
" [bugzillatitle] sysname NULL," +
" [bugzillatitle] nvarchar(512) NULL," +
" [redhatid] text NOT NULL);" +
"CREATE TABLE[map_cve_suse] (" +
" [suseid] sysname NULL," +
Expand All @@ -65,59 +65,59 @@ internal static class SqlServerSchema
" [saintexploitid] sysname NULL," +
" [saintexploittitle] sysname NULL," +
" [saintexploitlink] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveSaintNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_milw0rm] (" +
" [milw0rmid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveMilw0rmNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
"CREATE TABLE[map_cve_osvdb] (" +
" [osvdbid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveOsvdbNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_nessus] (" +
" [nessus_script_id] sysname NULL," +
" [nessus_script_file] sysname NULL," +
" [nessus_script_name] sysname NULL," +
" [nessus_script_name] nvarchar(512) NULL," +
" [nessus_script_family] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveNessusNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_msf] (" +
" [msfid] sysname NULL," +
" [msf_script_file] sysname NULL," +
" [msf_script_name] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveMsfNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_openvas] (" +
" [openvas_script_id] sysname NULL," +
" [openvas_script_file] sysname NULL," +
" [openvas_script_name] sysname NULL," +
" [openvas_script_name] varchar(512) NULL," +
" [openvas_script_family] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveOpenvasNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_scip] (" +
" [scipid] sysname NULL," +
" [sciplink] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveScipNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_iavm] (" +
" [iavmid] sysname NULL," +
" [disakey] sysname NULL," +
" [iavmtitle] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveIavmNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_cisco] (" +
" [ciscoid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveCiscoNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_ubuntu] (" +
" [ubuntuid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveUbuntuNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_gentoo] (" +
" [gentooid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveGentooNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_fedora] (" +
" [fedoraid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveFedoraNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_certvn] (" +
" [certvuid] sysname NULL," +
" [certvulink] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveCertvnNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_ms] (" +
" [msid] sysname NULL," +
" [mstitle] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveMsNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [mstitle] nvarchar(512) NULL," +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_mskb] (" +
" [mskbid] sysname NULL," +
" [mskbtitle] sysname NULL," +
Expand All @@ -126,43 +126,43 @@ internal static class SqlServerSchema
" [snort_id] sysname NULL," +
" [snort_sig] sysname NULL," +
" [snort_classtype] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveSnortNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_suricata] (" +
" [suricata_id] sysname NULL," +
" [suricata_sig] sysname NULL," +
" [suricata_sig] nvarchar(2048) NULL," +
" [suricata_classtype] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveSuricataNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_vmware] (" +
" [vmwareid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveVmwareNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_bid] (" +
" [bidid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveBidNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_hp] (" +
" [hpid] sysname NULL," +
" [hplink] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveHpNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [hplink] nvarchar(2048) NULL," +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[stat_new_cve] (" +
" [new_cve_id] sysname NULL CONSTRAINT CveNewNvd FOREIGN KEY REFERENCES [nvd_db]([cveid])," +
" [new_cve_summary] sysname NULL);" +
" [new_cve_id] sysname NULL," +
" [new_cve_summary] nvarchar(2048) NULL);" +
"CREATE TABLE[map_cve_exploitdb] (" +
" [exploitdbid] sysname NULL," +
" [exploitdbscript] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveExploitdbNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_nmap] (" +
" [nmap_script_id] sysname NULL," +
" [nmap_script_cat] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveNmapNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_oval] (" +
" [ovalid] sysname NULL," +
" [ovalclass] sysname NULL," +
" [ovaltitle] sysname NULL," +
" [ovaltitle] nvarchar(2048) NULL," +
" [cpeid] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveOvalNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[map_cve_d2] (" +
" [d2_script_name] sysname NULL," +
" [d2_script_file] sysname NULL," +
" [cveid] sysname NOT NULL CONSTRAINT CveD2Nvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" +
" [cveid] sysname NOT NULL);" +
"CREATE TABLE[stat_vfeed_kpi] (" +
" [db_version] sysname NULL," +
" [total_cve] sysname NULL," +
Expand Down Expand Up @@ -204,11 +204,11 @@ internal static class SqlServerSchema
" [capectitle] sysname NULL," +
" [attack] sysname NULL);" +
"CREATE TABLE[capec_mit] (" +
" [mitigation] sysname NULL," +
" [mitigation] nvarchar(2048) NULL," +
" [capecid] text NOT NULL);" +
"CREATE TABLE[cwe_wasc] (" +
" [wascname] sysname NULL," +
" [wascid] sysname NULL," +
" [cweid] sysname NOT NULL CONSTRAINT CveWascNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));";
" [cweid] sysname);";
}
}

0 comments on commit 01e2a9a

Please sign in to comment.