diff --git a/SSUpdate/Program.cs b/SSUpdate/Program.cs index a5f1284..e7c0de6 100644 --- a/SSUpdate/Program.cs +++ b/SSUpdate/Program.cs @@ -201,6 +201,19 @@ private static string BuildSqlServerTableCreationInstruction(string tableName) return builder.ToString(); } + private static void CleanUpDb() + { + using(SqlConnection connection = ConnectToSqlServer()) { + for(int index = CleanUpScripts.Length - 1; 0 <= index; index--) { + using (SqlCommand command = new SqlCommand(CleanUpScripts[index], connection)) { + command.CommandTimeout = 60; + command.ExecuteNonQuery(); + Console.Write("."); + } + } + } + } + private static SQLiteConnection ConnectToSQLite() { SQLiteConnection result = TryConnectToSQLite(); @@ -219,6 +232,7 @@ private static void DisplayUsage(string programName) { Console.WriteLine("{0} [-f -s -d |-h]"); Console.WriteLine(); + Console.WriteLine("-c : (optional) Perform cleanup before loading."); Console.WriteLine("-d : (optional) Sql Server vFeed database name (default vFeed)."); Console.WriteLine("-f : path to vFeed database."); Console.WriteLine("-h : display this help notice."); @@ -299,7 +313,7 @@ private static bool ImportData() string insertCommand = GetInsertCommand(tableName, out parametersCount); using (SQLiteCommand command = new SQLiteCommand(readCommand, sqliteConnection)) { using(SQLiteDataReader reader = command.ExecuteReader()) { - Transfer(sqlServerConnection, reader, insertCommand, parametersCount); + Transfer(tableName, sqlServerConnection, reader, insertCommand, parametersCount); } } } @@ -358,6 +372,11 @@ public static int Main(string[] args) DisplayUsage(programName); return result; } + if (_doCleanup) { + Console.WriteLine("Cleaning up."); + CleanUpDb(); + Console.WriteLine(); + } if (!LoadSQLiteSchema()) { return 2; } if (!ImportData()) { return 3; } return result; @@ -373,6 +392,9 @@ private static bool ParseArgs(string[] args) } else { switch (scannedArg.Substring(1).ToUpper()) { + case "C": + _doCleanup = true; + break; case "F": string dbPath = TryGetAdditionalArgument(args, ref argIndex); if (null == dbPath) { return false; } @@ -425,8 +447,8 @@ private static bool ParseArgs(string[] args) return true; } - private static void Transfer(SqlConnection sqlServerConnection, SQLiteDataReader reader, - string insertCommandText, int parametersCount) + private static void Transfer(string tableName, SqlConnection sqlServerConnection, + SQLiteDataReader reader, string insertCommandText, int parametersCount) { int linesCount = 0; using (SqlCommand insertCommand = new SqlCommand(insertCommandText, sqlServerConnection)) { @@ -438,7 +460,25 @@ private static void Transfer(SqlConnection sqlServerConnection, SQLiteDataReader for(int index = 0; index < parametersCount; index++) { parameters[index].Value = reader.GetValue(index); } - insertCommand.ExecuteNonQuery(); + Retry: + try { insertCommand.ExecuteNonQuery(); } + catch (SqlException e) { + switch (e.Number) { + case 0x223: + Console.WriteLine("Foreign key reference error. Faulting value is {0}", + parameters[parameters.Length - 1].Value); + break; + case 0x1FD8: + if ("map_cve_mskb" != tableName) { goto default; } + if (!(parameters[0].Value is string) || (parameters[1].Value is int)) { goto default; } + object trash = parameters[0].Value; + parameters[0].Value = parameters[1].Value; + parameters[1].Value = trash; + goto Retry; + default: + throw; + } + } if (0 == (++linesCount % 1000)) { Console.Write("."); } } Console.WriteLine("\r\n\r{0} lines imported.", linesCount); @@ -522,13 +562,12 @@ private static void VerifyTableDescription(SQLiteConnection connection, private const int TableInfoPrimaryKeyFlagColumIndex = 5; private static FileInfo _database; private static bool _displayUsage; + private static bool _doCleanup; #if DEBUG /// For debugging purpose only. This allow us to bypass /// already loaded tables and quicker fix a failing table. /// This table should be empty during normal use. private static readonly string[] _ignoreTables = new string[] { - "nvd_db", "cwe_db", "cve_cwe", "cwe_category", "cwe_capec", "cve_cpe", - "cve_reference", "map_cve_aixapar" }; #endif private static List _inInsertionOrderTableNames; @@ -548,5 +587,48 @@ private struct ColumnInfo internal int PrimaryKey { get; set; } internal string Type { get; set; } } + + private static readonly string[] CleanUpScripts = new string[] { + "DELETE nvd_db;", + "DELETE cwe_db;", + "DELETE cve_cwe;", + "DELETE cwe_category;", + "DELETE cwe_capec;", + "DELETE cve_cpe;", + "DELETE cve_reference;", + "DELETE map_cve_aixapar;", + "DELETE map_cve_redhat;", + "DELETE map_redhat_bugzilla;", + "DELETE map_cve_suse;", + "DELETE map_cve_debian;", + "DELETE map_cve_mandriva;", + "DELETE map_cve_saint;", + "DELETE map_cve_milw0rm;", + "DELETE map_cve_osvdb;", + "DELETE map_cve_nessus;", + "DELETE map_cve_msf;", + "DELETE map_cve_openvas;", + "DELETE map_cve_scip;", + "DELETE map_cve_iavm;", + "DELETE map_cve_cisco;", + "DELETE map_cve_ubuntu;", + "DELETE map_cve_gentoo;", + "DELETE map_cve_fedora;", + "DELETE map_cve_certvn;", + "DELETE map_cve_ms;", + "DELETE map_cve_mskb;", + "DELETE map_cve_snort;", + "DELETE map_cve_suricata;", + "DELETE map_cve_vmware;", + "DELETE map_cve_bid;", + "DELETE map_cve_hp;", + "DELETE stat_new_cve;", + "DELETE map_cve_exploitdb;", + "DELETE map_cve_nmap;", + "DELETE map_cve_oval;", + "DELETE map_cve_d2;", + "DELETE stat_vfeed_kpi;", + "DELETE capec_db;" + }; } } diff --git a/SSUpdate/SqlServerSchema.cs b/SSUpdate/SqlServerSchema.cs index 95a3306..1f86910 100644 --- a/SSUpdate/SqlServerSchema.cs +++ b/SSUpdate/SqlServerSchema.cs @@ -46,11 +46,11 @@ internal static class SqlServerSchema " [redhatid] sysname NULL," + " [redhatovalid] sysname NULL," + " [redhatupdatedesc] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveRedhatNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_redhat_bugzilla] (" + " [advisory_dateissue] sysname NULL," + " [bugzillaid] sysname NULL," + - " [bugzillatitle] sysname NULL," + + " [bugzillatitle] nvarchar(512) NULL," + " [redhatid] text NOT NULL);" + "CREATE TABLE[map_cve_suse] (" + " [suseid] sysname NULL," + @@ -65,59 +65,59 @@ internal static class SqlServerSchema " [saintexploitid] sysname NULL," + " [saintexploittitle] sysname NULL," + " [saintexploitlink] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveSaintNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_milw0rm] (" + " [milw0rmid] sysname NULL," + " [cveid] sysname NOT NULL CONSTRAINT CveMilw0rmNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + "CREATE TABLE[map_cve_osvdb] (" + " [osvdbid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveOsvdbNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_nessus] (" + " [nessus_script_id] sysname NULL," + " [nessus_script_file] sysname NULL," + - " [nessus_script_name] sysname NULL," + + " [nessus_script_name] nvarchar(512) NULL," + " [nessus_script_family] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveNessusNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_msf] (" + " [msfid] sysname NULL," + " [msf_script_file] sysname NULL," + " [msf_script_name] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveMsfNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_openvas] (" + " [openvas_script_id] sysname NULL," + " [openvas_script_file] sysname NULL," + - " [openvas_script_name] sysname NULL," + + " [openvas_script_name] varchar(512) NULL," + " [openvas_script_family] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveOpenvasNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_scip] (" + " [scipid] sysname NULL," + " [sciplink] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveScipNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_iavm] (" + " [iavmid] sysname NULL," + " [disakey] sysname NULL," + " [iavmtitle] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveIavmNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_cisco] (" + " [ciscoid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveCiscoNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_ubuntu] (" + " [ubuntuid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveUbuntuNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_gentoo] (" + " [gentooid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveGentooNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_fedora] (" + " [fedoraid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveFedoraNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_certvn] (" + " [certvuid] sysname NULL," + " [certvulink] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveCertvnNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_ms] (" + " [msid] sysname NULL," + - " [mstitle] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveMsNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [mstitle] nvarchar(512) NULL," + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_mskb] (" + " [mskbid] sysname NULL," + " [mskbtitle] sysname NULL," + @@ -126,43 +126,43 @@ internal static class SqlServerSchema " [snort_id] sysname NULL," + " [snort_sig] sysname NULL," + " [snort_classtype] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveSnortNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_suricata] (" + " [suricata_id] sysname NULL," + - " [suricata_sig] sysname NULL," + + " [suricata_sig] nvarchar(2048) NULL," + " [suricata_classtype] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveSuricataNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_vmware] (" + " [vmwareid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveVmwareNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_bid] (" + " [bidid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveBidNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_hp] (" + " [hpid] sysname NULL," + - " [hplink] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveHpNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [hplink] nvarchar(2048) NULL," + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[stat_new_cve] (" + - " [new_cve_id] sysname NULL CONSTRAINT CveNewNvd FOREIGN KEY REFERENCES [nvd_db]([cveid])," + - " [new_cve_summary] sysname NULL);" + + " [new_cve_id] sysname NULL," + + " [new_cve_summary] nvarchar(2048) NULL);" + "CREATE TABLE[map_cve_exploitdb] (" + " [exploitdbid] sysname NULL," + " [exploitdbscript] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveExploitdbNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_nmap] (" + " [nmap_script_id] sysname NULL," + " [nmap_script_cat] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveNmapNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_oval] (" + " [ovalid] sysname NULL," + " [ovalclass] sysname NULL," + - " [ovaltitle] sysname NULL," + + " [ovaltitle] nvarchar(2048) NULL," + " [cpeid] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveOvalNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[map_cve_d2] (" + " [d2_script_name] sysname NULL," + " [d2_script_file] sysname NULL," + - " [cveid] sysname NOT NULL CONSTRAINT CveD2Nvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));" + + " [cveid] sysname NOT NULL);" + "CREATE TABLE[stat_vfeed_kpi] (" + " [db_version] sysname NULL," + " [total_cve] sysname NULL," + @@ -204,11 +204,11 @@ internal static class SqlServerSchema " [capectitle] sysname NULL," + " [attack] sysname NULL);" + "CREATE TABLE[capec_mit] (" + - " [mitigation] sysname NULL," + + " [mitigation] nvarchar(2048) NULL," + " [capecid] text NOT NULL);" + "CREATE TABLE[cwe_wasc] (" + " [wascname] sysname NULL," + " [wascid] sysname NULL," + - " [cweid] sysname NOT NULL CONSTRAINT CveWascNvd FOREIGN KEY REFERENCES [nvd_db]([cveid]));"; + " [cweid] sysname);"; } }