BLD initial commit

Author: eigenfoo

.gitignore

@@ -0,0 +1,51 @@
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.out
+*.so
+*.pyc
+*.pyx
+__pycache__/
+*.gch
+*.pch
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+.ipynb_checkpoints/
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Temporary files #
+###################
+*~
+*.swp
+*.swo
+*.dSYM/

README.md

@@ -1 +1,3 @@
-# textcat
+# TextCat
+
+A simple text categorizer.

data/analyze.pl

@@ -0,0 +1,185 @@
+# Analyze results
+
+$ARGC = @ARGV;
+if ($ARGC != 2) {
+    print STDERR "Usage: analyze.pl PREDICTIONS ACTUAL\n";
+    print STDERR "   PREDICTION: list of system's predictions\n";
+    print STDERR "   ACTUAL: list of the actual labels\n";
+    exit;
+}
+
+$results = $ARGV[0];
+$actual = $ARGV[1];
+
+# Process results file.
+
+print STDERR "Processing answer file...\n";
+
+if (!(open(ANSWER, "<$actual"))) {
+    print STDERR "Error: could not open $actual for input.\n";
+    exit;
+}
+
+# Load the category information
+
+# Stores size of each category
+%cat_size = ();
+# Set to true for seen categories
+%have_seen = ();
+# Stores the actual category of every document.
+%actual_cat = ();
+
+while ($line = <ANSWER>) {
+    if (!($line =~ /([^ ]*) (.*)/)) {
+        print STDERR "Error: Line is actual file not in expected format!\n";
+        exit;
+    }
+    $number = $1;
+    $actual_cat{$number} = $2;
+    $have_seen{$2} = 1;
+    $cat_size{$2} += 1;
+}
+
+# Stores list of categories
+@categories = keys %have_seen;
+$num_categories = @categories;
+
+print "Found $num_categories categories:";
+foreach $cat (@categories) {
+    print " $cat";
+}
+print "\n";
+
+# Process results file.
+
+print STDERR "Processing prediction file...\n";
+
+if (!(open(RESULTS, "<$results"))) {
+    print STDERR "Error: could not open $results for input.\n";
+    exit;
+}
+
+# Stores the predicted category of every document.
+%predicted_cat = ();
+# First dimension of contingency table (row) is system's prediction,
+# second dimension (column) is actual category.
+%contingency = ();
+
+while ($line = <RESULTS>) {
+    if (!($line =~ /([^ ]*) (.*)/)) {
+	print STDERR "Error: Line in results file not in expected format!\n";
+	exit;
+    }
+    $number = $1;
+    $predicted_cat{$number} = $2;
+
+    $contingency{$predicted_cat{$number}}{$actual_cat{$number}} += 1;
+}
+
+# Determine overall accuracy.
+$correct = $incorrect = 0;
+foreach $cat (@categories) {
+    foreach $cat2 (@categories) {
+	if ($cat eq $cat2) {
+	    $correct += $contingency{$cat}{$cat2};
+	}
+	else {
+	    $incorrect += $contingency{$cat}{$cat2};
+	}
+    }
+}
+if ($correct + $incorrect > 0) {
+    $ratio = $correct / ($correct + $incorrect);
+}
+else {
+    $ratio = "UNDEFINED";
+}
+print "\n$correct CORRECT, $incorrect INCORRECT, RATIO = $ratio.\n\n";
+
+# Display contingency table, calculate precisions, recalls, and F_1s.
+
+# Header row.
+print "CONTINGENCY TABLE:\n        ";
+foreach $cat (@categories) {
+    if (length($cat) > 7) {
+	$header = substr($cat, 0, 7);
+    }
+    else {
+	$header = $cat;
+    }
+    printf ("%-8s", $header);
+}
+print "PREC\n";
+
+# Category rows.
+%precision = ();
+foreach $cat (@categories) {
+    if (length($cat) > 7) {
+	$header = substr($cat, 0, 7);
+    }
+    else {
+	$header = $cat;
+    }
+    printf ("%-8s", $header);
+
+    $correct = $incorrect = 0;
+    foreach $cat2 (@categories) {
+	if ($contingency{$cat}{$cat2} eq "") {
+	    $contingency{$cat}{$cat2} = 0;
+	}
+	printf ("%-8d", $contingency{$cat}{$cat2});
+	if ($cat eq $cat2) {
+	    $correct += $contingency{$cat}{$cat2};
+	}
+	else {
+	    $incorrect += $contingency{$cat}{$cat2};
+	}
+    }
+    if ($correct + $incorrect > 0) {
+	$prec = $correct / ($correct + $incorrect);
+    }
+    else {
+	$prec = 0;
+    }
+    $precision{$cat} = $prec;
+    printf("%.2f\n", $prec);
+}
+
+# Recall row.
+%recall = ();
+print "RECALL  ";
+foreach $cat (@categories) {
+    $correct = $incorrect = 0;
+    foreach $cat2 (@categories) {
+	# Now $cat is the column and $cat2 is the row.
+	if ($cat eq $cat2) {
+	    $correct += $contingency{$cat2}{$cat};
+	}
+	else {
+	    $incorrect += $contingency{$cat2}{$cat};
+	}	
+    }
+    if ($correct + $incorrect > 0) {
+	$rec = $correct / ($correct + $incorrect);
+    }
+    else {
+	$rec = 0;
+    }
+    $recall{$cat} = $rec;
+    printf("%-8.2f", $rec);    
+}
+print "\n\n";
+
+# F_1 values.
+%f1 = ();
+foreach $cat (@categories) {
+    if ($precision{$cat} + $recall{$cat} > 0) {
+	$f1{$cat} = (2 * $precision{$cat} * $recall{$cat}) /
+	    ($precision{$cat} + $recall{$cat});
+    }
+    else {
+	$f1{$cat} = 0;
+    }
+    print "F_1($cat) = $f1{$cat}\n";
+}
+print "\n";

data/corpus1/test/03785.article

@@ -0,0 +1,35 @@
+  	  				 
+	 LONDON (Reuter) - Anti-terrorist police Saturday sifted  
+through debris for clues after an explosion in a west London 
+litter bin, but there was still no indication of who was 
+responsible. 
+	 No one was injured, but two cars and windows in nearby shops  
+and homes were damaged in the blast just after midnight outside 
+a cemetery in Old Brompton Road, near the vast Earl's Court 
+exhibition hall. 
+	 There was no advance warning of the blast, no claim of  
+responsibility and no obvious target. 
+	 London has been in a state of high alert since the Irish  
+Republican Army last month ended a 17-month cease-fire in its 
+campaign of violence against British rule in Northern Ireland 
+and set off bombs in east London's Docklands redevelopment area 
+and a double-decker bus in the city center. 
+	 ``There was no warning. We have had no reports of injuries.  
+The damage was confined to a couple of cars and windows in 
+buildings opposite,'' police chief superintendent Peter Rice 
+told reporters after the litter bin bombing. 
+	 Police declined to make any link with the IRA, saying their  
+forensic work was continuing. 
+	 A large area around the blast scene was cordoned off during  
+the night and sniffer dogs were brought in to search for 
+possible secondary devices in the area, which is well known for 
+its nightclubs. 
+	 The IRA, frustrated by the lack of progress toward Northern  
+Irish peace talks, ended its cease-fire on February 9 by 
+detonating a huge bomb in the eastern Docklands financial 
+district. Two people died in the blast. 
+	 Nine days later an IRA guerrilla died when the device he was  
+carrying exploded prematurely on a London bus. 
+	 The guerrillas said this week they were ready for another 25  
+years of war if London and Dublin failed to come up with a ``new 
+deal'' for peace in Northern Ireland. 

data/corpus1/test/04043.article

@@ -0,0 +1,70 @@
+  	  				 
+	 WASHINGTON (Reuter) - The United States Tuesday gave another  
+sharp warning to China against invading Taiwan, and the House of 
+Representatives approved a resolution stating that Washington 
+should help defend Taiwan in such an attack. 
+	 A senior U.S. official also said Taiwan had asked the United  
+States in day-long talks here to provide diesel submarines to 
+Taipei for naval defense, a request rejected by Washington last 
+year. 
+	 The official, who asked not to be identified, said the  
+request came in annual talks on possible new arms sales to 
+Taipei as tension grew over Chinese maneuvers off the island in 
+advance of Taiwan's presidential elections on Saturday. 
+	 The Pentagon Tuesday put China on notice that ``America has  
+the best damned Navy in the world'' and could sail the Taiwan 
+Strait if it chose to, but the State Department hastened to say 
+this was not meant to provoke Beijing. 
+	 The defense resolution, overwhelmingly passed by the House  
+despite White House opposition, is not binding on the Clinton 
+administration. It was drafted in response to Chinese military 
+exercises n the Taiwan Strait. 
+	 ``America has the best damned Navy in the world, and no one  
+should ever forget that,'' said Defense Secretary William Perry 
+in an obvious warning to Beijing not to launch an attack on 
+Taiwan. 
+	 ``Beijing should know -- and this (a gathering U.S. armada)  
+will remind them -- that, while they are a great military power, 
+that the premier, the strongest military power, in the western 
+Pacific is the United States,'' Perry said in a speech to 
+current and former members of Congress. 
+	 He referred to a U.S. naval task force gathering in the  
+Taiwan region -- although well away from the strait separating 
+China from what it regards as its renegade province -- to be 
+centered around two aircraft carriers. 
+	 At the State Department, where a reporter asked about the  
+apparent escalation in rhetoric, spokesman Glyn Davies said 
+there was no such intention on the U.S. side. 
+	 Davies said Perry's comments were ``about his pride in the  
+United States armed forces and the United States Navy.'' 
+	 ``That was just a statement of fact, that it is the best  
+navy in the world. But there's no racheting up here, from our 
+standpoint, of the rhetoric at all.'' 
+	 At a separate Pentagon briefing, however, Defense spokesman  
+Ken Bacon noted that the carrier Nimitz would arrive off Taiwan 
+on Saturday or Sunday to join the carrier Independence -- and he 
+he rejected China's warning against sending U.S. warships 
+through the Taiwan Strait. 
+	 ``We reserve the right to sail in international waters,''  
+Bacon said. ``Whether or not we sail through the Strait of 
+Taiwan has not been decided.'' 
+	 The United States and China, meanwhile, took steps that  
+might help ease tensions, although not immediately. 
+	 In Prague, where Secretary of State Warren Christopher was  
+traveling, State Department spokesman Nicholas Burns said 
+Christopher would meet his Chinese counterpart, Qian Qichen, in 
+The Hague on April 21 to discuss Taiwan and other issues. 
+	 At the Pentagon, Bacon said Chinese Defense Minister Chi  
+Haotian was scheduled to visit for talks with Perry sometime in 
+April. 
+	 U.S. officials refused to characterize the tone of the  
+one-day arms meeting between Taiwan officials and 
+representatives of the Defense and State Departments. And they 
+would not say when any decision could come out of the talks. 
+	 The Washington Times reported Tuesday that Taiwan wanted six  
+German-designed diesel submarines to be built in the United 
+States as well as new P-3 anti-submarine aircraft and 
+air-launched Harpoon anti-ship missiles. 
+	 But the senior U.S. official told Reuters that Taiwan  
+officials did not ask for the anti-submarine planes or the 
+missiles.