komconfstats

#!/usr/bin/python
# -*- coding: iso-8859-1 -*-
# Collect some [1] statistics about a conference
# $Id: komconfstats,v 1.20 2008/09/30 21:21:57 kent Exp $
# (C) 2000-2002 Kent Engström. Released under GPL.
#
# SAKER ATT FIXA:
# - Mer statistik
#   - Mottagartyp: vanlig, extra kopia, bcc
#   - Antal mottagare
#   - Mera aux-items (t.ex. fast-replies)
# - Annat
#   - Förklarande text under rapporten
#   - Redovisa begränsningar (max_displayed) i lång titel
#   - Grafik via Gnuplot eller liknande
#   - Transponera (a,b) -> (b,a) istället för att samla flera samtidigt?
#
# The rest of the comments should be in English. Don't ask why,
# it just happens to be like that.
#
# --
# [1] or rather "more than you even knew you wanted"
#


import kom
import komparam
import komconfwalker
import sys
import getopt
import string
import re
import time

import jddate # From the lyspython collection
import tabulator

# Get revision number from RCS/CVS
vc_revision = "$Revision: 1.20 $"
revision = vc_revision[11:-2]

# Error/sucess reporting
exit_code = 0
def error(str, code = 2, exit_now = 0):
    global exit_code
    exit_code = max(exit_code, code)
    sys.stderr.write("ERROR: " + str + "\n")
    if exit_now:
        exit()
        
def success(str):
    sys.stderr.write("OK: " + str + "\n")

def exit():
    sys.exit(exit_code)

#
# Split a text into lower case words (removing the subject line!)
#

upper2lower = string.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZÅÄÖ",
                               "abcdefghijklmnopqrstuvwxyzåäö")
def split_text_dehyphen(s):
    return re.sub("(^-*)|(-*$)", "", s)

def split_text_into_words(text):
    first_nl = string.find(text,"\n")
    if first_nl <> -1:
        text = text[first_nl+1:]
    text = string.translate(text, upper2lower)
    words = filter(None, map(split_text_dehyphen,
                             re.split("[^a-zåäö-]+", text)))
    return words

# Palindrome check
def is_palindrome(str):
    l1 = list(str)
    l2 = list(str)
    l2.reverse()
    return l1 == l2


# Base classes and useful comments have been refactored into tabulator.py

class ConfNameTabulator(tabulator.Tabulator):
    # Stores conference numbers but displays them as names.

    def key_to_display(self, pers_no):
        return conn.conf_name(pers_no)

class CommentTabulator(tabulator.Tabulator):
    # Stores pairs of conference numbers but displays them as names.
    
    def key_to_display(self, pers_tuple):
        return conn.conf_name(pers_tuple[0]) + " " + self.prop.separator + " " + \
               conn.conf_name(pers_tuple[1])

re_zero = re.compile(".*[1-9]([0-9]+)$")
class EvenNumberTabulator(tabulator.Tabulator):
    # Stores the number of zero digits at the end of an article
    # number.
    
    def data_to_key(self, str):
        m = re_zero.match(str)
        return -len(m.group(1))

    def key_to_display(self, x):
        return "Jämna 1" + "0" * (-x) + "-tal"

class WordTabulator(tabulator.Tabulator):
    def report_extra_header(self, l):
        r = random.choice(self.dict.keys())
        l.append("Denna rapports slumpmässigt valda ord är:")
        l.append('    "%s", använt %s' % (r,
                                          tabulator.get_count_with_noun(self.dict[r].count,
                                                                        "gång","gånger")))
        l.append("")

class HourHistogram(tabulator.Histogram):
    def key_to_display(self, hour):
        return "%02d" % hour

swedish_weekdays = ["Måndag","Tisdag","Onsdag","Torsdag","Fredag","Lördag","Söndag"]
class WeekdayHistogram(tabulator.Histogram):
    def key_to_display(self, wd):
        return swedish_weekdays[wd]
    
class DateHistogram(tabulator.Histogram):
    def key_to_display(self, jd):
        (y, w, d) = jd.GetYWD()
        return jd.GetString_YYYY_MM_DD() + " " + \
               swedish_weekdays[d-1][:2]

class SizeLogHistogram(tabulator.LogHistogram):
    def __init__(self, key = None, properties = None):
        self.logbase = 2
        tabulator.LogHistogram.__init__(self, key, properties) 

class TimeBinHistogram(tabulator.BinHistogram):
    def __init__(self, key = None, properties = None):
        self.bins = [(      0, " 0-15 s"),
                     (     15, "15-30 s"),
                     (     30, "30-45 s"),
                     (     45, "45-60 s"),
                     (     60, " 1- 2 m"),
                     (    120, " 2- 3 m"),
                     (    180, " 3- 5 m"),
                     (    300, " 5-10 m"),
                     (    600, "10-15 m"),
                     (    900, "15-30 m"),
                     (   1800, "30-45 m"),
                     (   3600, " 1- 2 h"),
                     (   7200, " 2- 4 h"),
                     (  14400, " 4- 6 h"),
                     (  21600, " 6-12 h"),
                     (  43200, "12-24 h"),
                     (  86400, " 1- 2 d"),
                     ( 172800, " 2- 4 d"),
                     ( 345600, " 4- 7 d"),
                     ( 604800, " 1- 2 w"),
                     (1209600, " 2- 4 w"),
                     (2419200, " 4-   w"),
                     ]
        tabulator.BinHistogram.__init__(self, key, properties)

#
# CONFSTATS CLASS
#
# This is the real worker. An object of this class is used
# to read the articles of the conference, organize the data
# using all the nice little Tabulators and then produce all
# the reports (as text on stdout or as articles in a conference).
#
# Newsflash: The walking logic has been isolated to a separate
# class, from which this one inherits.

class ConfStats(komconfwalker.ConfTemporalWalker):
    def __init__(self, conn, person, conference, period,
                 mark_earlier = 0, mark_processed = 0,
                 full_text = 0, post_to_conference = 0,
                 exclude_self = 0,
                 debug = 0):
        komconfwalker.ConfTemporalWalker.__init__(self, conn,
                                                  person, conference, period,
                                                  mark_earlier, mark_processed,
                                                  debug)
        
        self.full_text = full_text
        self.post_to_conference = post_to_conference
        self.exclude_self = exclude_self

        self.reports = [] # The reports to produce, in order
        self.suppressed_reports = [] # The reports not to produce, in order
        
    def add_report(self, tabulator, suppress = 0):
        if not suppress:
            self.reports.append(tabulator)
        else:
            self.suppressed_reports.append(tabulator)
        
    def gather(self):

        # First, declare tabulators. This is the part you should
        # change if you want to delete, move or add reports.

        self.at = ConfNameTabulator()
        self.at.prop.what = "författare"
        self.at.prop.title = "Inlägg"
        self.at.prop.singularis = "inlägg"
        sub = self.at.prop.add_sub(tabulator.Tabulator)
        sub.what = "ärenderad"
        sub.max_displayed = 10
        self.add_report(tabulator.Report(self.at, 1))
        # 2002-08-01: Not interesting enough
        self.add_report(self.at, suppress = 1)
        
        self.st = tabulator.Tabulator()
        self.st.prop.title = "Inlägg"
        self.st.prop.singularis = "inlägg"
        self.st.prop.what = "ärenderad"
        self.st.prop.max_displayed = 20
        sub = self.st.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 10
        self.add_report(tabulator.Report(self.st, 1))
        self.add_report(self.st)
     
        self.gat = ConfNameTabulator()
        self.gat.prop.what = "författare"
        self.gat.prop.title = "Inlägg av gästförfattare"
        self.gat.prop.singularis = "inlägg"
        sub = self.gat.prop.add_sub(tabulator.Tabulator)
        sub.what = "ärenderad"
        self.add_report(self.gat)
      
        self.oat = ConfNameTabulator()
        self.oat.prop.what = "författare"
        self.oat.prop.title = "Originalinlägg"
        self.oat.prop.singularis = "inlägg"
        sub = self.oat.prop.add_sub(tabulator.Tabulator)
        sub.what = "ärenderad"
        sub = sub.add_sub(tabulator.Tabulator)
        sub.what = "inläggsnummer"
        self.add_report(self.oat, suppress = 1)
      
        self.sat = ConfNameTabulator()
        self.sat.prop.title = "Kommentarer med ändrad ärenderad"
        self.sat.prop.singularis = "inlägg"
        self.sat.prop.what = "författare"
        sub = self.sat.prop.add_sub(tabulator.Tabulator)
        sub.what = "gammal ärenderad"
        sub = sub.add_sub(tabulator.Tabulator)
        sub.what = "ny ärenderad"        
        self.add_report(self.sat)
      
        self.mat = ConfNameTabulator()
        self.mat.prop.title = "Kommentarer till texter i andra möten"
        self.mat.prop.singularis = "inlägg"
        self.mat.prop.what = "författare"
        self.mat.prop.max_displayed = 5
        sub = self.mat.prop.add_sub(ConfNameTabulator)
        sub.what = "möte"
        sub = sub.add_sub(tabulator.Tabulator)
        sub.what = "ärenderad"
        # self.add_report(tabulator.Report(self.mat, levels = 2))
        self.add_report(self.mat)
      
        # 2002-08-01: Not interesting enough
        self.rmat = ConfNameTabulator()
        self.rmat.prop.title = "Kommentarer till texter i andra möten"
        self.rmat.prop.singularis = "inlägg"
        self.rmat.prop.what = "möte"
        sub = self.rmat.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub = sub.add_sub(tabulator.Tabulator)
        sub.what = "ärenderad"
        self.add_report(self.rmat, suppress = 1)
      
        #2002-08-01: Not interesting enough
        self.crt = ConfNameTabulator()
        self.crt.prop.title = "Inlägg i andra möten också"
        self.crt.prop.singularis = "inlägg"
        self.crt.prop.what = "möte"
        sub = self.crt.prop.add_sub(tabulator.Tabulator)
        sub.what = "ärenderad"
        sub = sub.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 10
        self.add_report(self.crt, suppress = 1)

        # 2002-08-01: Not interesting enough
        self.rcrt = tabulator.Tabulator()
        self.rcrt.prop.title = "Inlägg i andra möten också"
        self.rcrt.prop.singularis = "inlägg"
        self.rcrt.prop.what = "ärenderad"
        sub = self.rcrt.prop.add_sub(ConfNameTabulator)
        sub.what = "möte"
        sub = sub.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 10
        self.add_report(self.rcrt, suppress = 1)

        # 2002-08-01: Not interesting enough
        # 2003-04-10: Enabled again
        self.prt = ConfNameTabulator()
        self.prt.prop.title = "Inlägg mottagna av personer"
        self.prt.prop.singularis = "inlägg"
        self.prt.prop.what = "person"
        self.prt.prop.max_displayed = 10
        sub = self.prt.prop.add_sub(tabulator.Tabulator)
        sub.what = "ärenderad"
        sub.max_displayed = 5
        sub = sub.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 5
        self.add_report(self.prt)

        # 2002-08-01: Not interesting enough
        self.rprt = tabulator.Tabulator()
        self.rprt.prop.title = "Inlägg mottagna av personer"
        self.rprt.prop.singularis = "inlägg"
        self.rprt.prop.what = "ärenderad"
        sub = self.rprt.prop.add_sub(ConfNameTabulator)
        sub.what = "person"
        sub = sub.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 10
        self.add_report(self.rprt, suppress = 1)

        self.fnat = ConfNameTabulator()
        self.fnat.prop.title = "Fotnoter"
        self.fnat.prop.singularis = "inlägg"
        self.fnat.prop.what = "författare"
        self.add_report(self.fnat)
      
        self.scat = ConfNameTabulator()
        self.scat.prop.title = "Kommentarer till egna inlägg exkl. fotnoter"
        self.scat.prop.singularis = "inlägg"
        self.scat.prop.what = "författare"
        self.add_report(self.scat)
      
        self.cat0 = ConfNameTabulator()
        self.cat0.prop.title = "Inlägg utan kommentarer"
        self.cat0.prop.singularis = "inlägg"
        self.cat0.prop.what = "författare"
        self.add_report(self.cat0)
        self.add_report(tabulator.Report(self.cat0, relative = 1, min_total_count = 10))
      
        self.cat1 = ConfNameTabulator()
        self.cat1.prop.title = "Inlägg med minst 1 kommentar"
        self.cat1.prop.singularis = "inlägg"
        self.cat1.prop.what = "författare"
        self.add_report(self.cat1, suppress=1)
        self.add_report(tabulator.Report(self.cat1, relative = 1, min_total_count = 10), suppress=1)
        
        self.cat2 = ConfNameTabulator()
        self.cat2.prop.title = "Inlägg med minst 2 kommentarer"
        self.cat2.prop.singularis = "inlägg"
        self.cat2.prop.what = "författare"
        self.add_report(self.cat2, suppress=1)
        self.add_report(tabulator.Report(self.cat2, relative = 1, min_total_count = 10), suppress=1)
 
        self.cat3 = ConfNameTabulator()
        self.cat3.prop.title = "Inlägg med minst 3 kommentarer"
        self.cat3.prop.singularis = "inlägg"
        self.cat3.prop.what = "författare"
        self.add_report(self.cat3, suppress=1)
        self.add_report(tabulator.Report(self.cat3, relative = 1, min_total_count = 10), suppress=1)
 
        self.cat4 = ConfNameTabulator()
        self.cat4.prop.title = "Inlägg med minst 4 kommentarer"
        self.cat4.prop.singularis = "inlägg"
        self.cat4.prop.what = "författare"
        self.add_report(self.cat4, suppress=1)
        self.add_report(tabulator.Report(self.cat4, relative = 1, min_total_count = 10), suppress=1)
        
        self.cat5 = ConfNameTabulator()
        self.cat5.prop.title = "Inlägg med minst 5 kommentarer"
        self.cat5.prop.singularis = "inlägg"
        self.cat5.prop.what = "författare"
        self.add_report(self.cat5, suppress=1)
        self.add_report(tabulator.Report(self.cat5, relative = 1, min_total_count = 10), suppress=1)

        self.cat10 = ConfNameTabulator()
        self.cat10.prop.title = "Inlägg med minst 10 kommentarer"
        self.cat10.prop.singularis = "inlägg"
        self.cat10.prop.what = "författare"
        sub = self.cat10.prop.add_sub(tabulator.Tabulator)
        sub.what = "inläggsnummer"
        self.add_report(self.cat10)
        self.add_report(tabulator.Report(self.cat10, relative = 1, min_total_count = 10), suppress = 1)

        # 2002-08-01: Not interesting enough
        self.ct = ConfNameTabulator()
        self.ct.prop.title = "Kommentarer"
        self.ct.prop.what = "författare"
        self.ct.prop.singularis = "kommentering"
        self.ct.prop.pluralis = "kommenteringar"
        sub = self.ct.prop.add_sub(ConfNameTabulator)
        sub.what = "kommenterad författare"
        sub.max_displayed = 10
        self.add_report(self.ct, suppress = 1)

        # 2002-08-01: Not interesting enough
        self.rct = ConfNameTabulator()
        self.rct.prop.title = "Kommentarer"
        self.rct.prop.what = "kommenterad författare"
        self.rct.prop.singularis = "kommentering"
        self.rct.prop.pluralis = "kommenteringar"
        sub = self.rct.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 10
        self.add_report(self.rct, suppress = 1)
      
        self.dct = CommentTabulator()
        self.dct.prop.title = "Kommentarer"
        self.dct.prop.what = "personpar (varje riktning för sig)"
        self.dct.prop.separator = "kommenterar"
        self.dct.prop.singularis = "kommentering"
        self.dct.prop.pluralis = "kommenteringar"
        self.dct.prop.max_displayed = 50
        self.add_report(self.dct)
      
        self.nct = CommentTabulator()
        self.nct.prop.title = "Kommentarer"
        self.nct.prop.what = "personpar (oavsett riktning)"
        self.nct.prop.separator = "och"
        self.nct.prop.singularis = "kommentering"
        self.nct.prop.pluralis = "kommenteringar"
        self.nct.prop.max_displayed = 50
        self.add_report(self.nct)

        self.vt = tabulator.Tabulator()
        self.vt.prop.title = "Inlägg"
        self.vt.prop.singularis = "inlägg"
        self.vt.prop.what = "klient"
        sub = self.vt.prop.add_sub(tabulator.Tabulator)
        sub.what = "version"
        self.add_report(self.vt)
     
        self.ctt = tabulator.Tabulator()
        self.ctt.prop.title = "Inlägg"
        self.ctt.prop.singularis = "inlägg"
        self.ctt.prop.what = "content-type"
        sub = self.ctt.prop.add_sub(tabulator.Tabulator)
        sub.what = "options"
        self.add_report(self.ctt, suppress = 1)
     
        self.ent = EvenNumberTabulator()
        self.ent.prop.title = "Jubelinlägg"
        self.ent.prop.singularis = "inlägg"
        self.ent.prop.what = "jubeltyp"
        sub = self.ent.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub = sub.add_sub(tabulator.Tabulator)
        sub.what = "inläggsnummer"
        self.add_report(self.ent)

        self.lent = EvenNumberTabulator()
        self.lent.prop.title = "Lokaljubelinlägg"
        self.lent.prop.singularis = "inlägg"
        self.lent.prop.what = "jubeltyp"
        sub = self.lent.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub = sub.add_sub(tabulator.Tabulator)
        sub.what = "inläggsnummer"
        self.add_report(self.lent, suppress = 1)

        self.pal = ConfNameTabulator()
        self.pal.prop.title = "Palindromnummerinlägg"
        self.pal.prop.singularis = "inlägg"
        self.pal.prop.what = "författare"
        sub = self.pal.prop.add_sub(tabulator.Tabulator)
        sub.what = "inläggsnummer"
        self.add_report(self.pal)
     
        self.lpal = ConfNameTabulator()
        self.lpal.prop.title = "Lokalpalindromnummerinlägg"
        self.lpal.prop.singularis = "inlägg"
        self.lpal.prop.what = "författare"
        sub = self.lpal.prop.add_sub(tabulator.Tabulator)
        sub.what = "inläggsnummer"
        self.add_report(self.lpal, suppress = 1)
     
        if self.full_text:
            self.wt = WordTabulator()
            self.wt.prop.title = "Ord i texter"
            self.wt.prop.what = "ord"
            self.wt.prop.singularis = "ord"
            self.wt.prop.max_displayed = 200
            self.add_report(self.wt)

        # Histograms
        
        self.hh = HourHistogram()
        self.hh.prop.title = "Inlägg"
        self.hh.prop.singularis = "inlägg"
        self.hh.prop.what = "timme på dygnet"
        self.hh.prop.set_histogram_range(0, 23)
        self.hh.prop.histogram_hide_zero = 1
        sub = self.hh.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 5
        self.add_report(tabulator.Report(self.hh, 1))
        self.add_report(self.hh)
            
        self.wh = WeekdayHistogram()
        self.wh.prop.title = "Inlägg"
        self.wh.prop.singularis = "inlägg"
        self.wh.prop.what = "veckodag"
        self.wh.prop.set_histogram_range(0,6)
        self.wh.prop.histogram_hide_zero = 1
        sub = self.wh.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 5
        self.add_report(tabulator.Report(self.wh, 1), suppress = 1)
        self.add_report(self.wh, suppress = 1)
            
        self.dh = DateHistogram()
        self.dh.prop.title = "Inlägg"
        self.dh.prop.singularis = "inlägg"
        self.dh.prop.what = "datum"
        self.dh.prop.histogram_hide_zero = 1
        sub = self.dh.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 5
        self.add_report(tabulator.Report(self.dh, 1))
        self.add_report(self.dh)

        # Inte intressant nog 2004-09-01    
        self.dst = tabulator.Tabulator()
        self.dst.prop.title = "Inlägg"
        self.dst.prop.singularis = "inlägg"
        self.dst.prop.what = "ärenderad"
        self.dst.prop.max_displayed = 20
        sub = self.dst.prop.add_sub(DateHistogram)
        sub.what = "datum"
        self.add_report(self.dst, suppress=1)
     
        self.kh = tabulator.Histogram()
        self.kh.prop.title = "Inlägg"
        self.kh.prop.singularis = "inlägg"
        self.kh.prop.what = "antal kommentarer"
        self.kh.prop.histogram_hide_zero = 1
        self.add_report(self.kh)
            
        self.csh = SizeLogHistogram()
        self.csh.prop.title = "Inlägg"
        self.csh.prop.singularis = "inlägg"
        self.csh.prop.what = "antal tecken"
        self.csh.prop.histogram_hide_zero = 1
        self.add_report(self.csh, suppress = 1)
            
        self.lsh = SizeLogHistogram()
        self.lsh.prop.title = "Inlägg"
        self.lsh.prop.singularis = "inlägg"
        self.lsh.prop.what = "antal rader"
        self.lsh.prop.histogram_hide_zero = 1
        sub = self.lsh.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 5
        self.add_report(tabulator.Report(self.lsh, 1))
        self.add_report(self.lsh)
            
        self.ah = TimeBinHistogram()
        self.ah.prop.title = "Kommentarer"
        self.ah.prop.singularis = "inlägg"
        self.ah.prop.what = "antal sekunder sedan det kommenterade"
        self.ah.prop.histogram_hide_zero = 1
        sub = self.ah.prop.add_sub(ConfNameTabulator)
        sub.what = "författare"
        sub.max_displayed = 5
        self.add_report(tabulator.Report(self.ah, 1))
        self.add_report(self.ah)

        # Stop! The declaration of tabulators end here. Below
        # lies Real Code <tm>.
            
        # Learn the members of the conference
        # FIXME: What maximum number should be used?
        self.members = map(lambda x: x.member,
                           kom.ReqGetMembers(self.conn,
                                             self.conference,
                                             0, 16384).response())
        
        # Traverse the texts.
        self.walk()

        # That's it. The reporting is done in a separate method.


    def process(self, loc_no, global_no, ts):
        # Overrides the dummy method in ConfTemporalWalker


        # Skip texts by myself
        if self.exclude_self and ts.author == self.person:
            return

        # Tabulate simple things that only depend on this text
        subject = self.conn.subjects[global_no]
        self.at.tabulate([ts.author, subject])
        self.st.tabulate([subject, ts.author])
        if ts.author not in self.members:
            self.gat.tabulate([ts.author, subject])
        self.hh.tabulate([ts.creation_time.hours, ts.author])
        self.wh.tabulate([(ts.creation_time.day_of_week-1)%7, ts.author])
        self.dh.tabulate([jddate.FromUnixTime(ts.creation_time.to_python_time()), ts.author])
        self.dst.tabulate([subject, jddate.FromUnixTime(ts.creation_time.to_python_time())])
        self.csh.tabulate([ts.no_of_chars])
        self.lsh.tabulate([ts.no_of_lines, ts.author])

        # Even numbers ("jubel")
        # FIXME: This is base 10. Should we do base-2 also?
        
        numstr = str(global_no)
        if numstr[-3:] == "000":
            self.ent.tabulate([numstr, ts.author, numstr])

        lnumstr = str(loc_no)
        if lnumstr[-3:] == "000":
            self.lent.tabulate([lnumstr, ts.author, lnumstr + ": " + numstr])

        # Palindromes
        if is_palindrome(numstr):
            self.pal.tabulate([ts.author, numstr])

        if is_palindrome(lnumstr):
            self.lpal.tabulate([ts.author, lnumstr + ": " + numstr])

        # Tabulate things that depend on the recipient list
        for mir in ts.misc_info.recipient_list:
            if mir.recpt <> self.conference:
                if self.conn.uconferences[mir.recpt].type.letterbox:
                    self.prt.tabulate([mir.recpt, subject, ts.author])
                    self.rprt.tabulate([subject, mir.recpt, ts.author])

                else:
                    self.crt.tabulate([mir.recpt, subject, ts.author])
                    self.rcrt.tabulate([subject, mir.recpt, ts.author])

        # Tabulate things that depend on the aux-items
        # CREATING SOFTWARE
        ai_c_s = kom.first_aux_items_with_tag(ts.aux_items,
                                              kom.AI_CREATING_SOFTWARE)
        if ai_c_s is not None:
            (c_s,c_s_version) = (string.split(ai_c_s.data)+[""])[:2]
            self.vt.tabulate([c_s, c_s_version])
        else:
            self.vt.tabulate(["Ej specificerad", "Ingen version"])

        # CONTENT_TYPE
        ai_c_t = kom.first_aux_items_with_tag(ts.aux_items,
                                              kom.AI_CONTENT_TYPE)
        if ai_c_t is not None:
            (c_t,c_t_options) = (string.split(ai_c_t.data,";")+[""])[:2]
            self.ctt.tabulate([c_t, c_t_options])
        else:
            self.ctt.tabulate(["Ej specificerad","Inga optioner"])
            
        # Tabulate things that depend on the commented text(s)

        # NOTICE: Things tallied inside the for loop could get
        # rather confused (e.g. an article counted multiple times)
        # when an article is a comment to more than one other article.
        # However, this may be a small price to pay to get some more
        # interesting information for all "normal" chains.
        
        if len(ts.misc_info.comment_to_list) == 0:
            self.oat.tabulate([ts.author, subject, str(global_no)])
        else:
            recipients_of_commented_includes_this_conference = 0
            this_is_a_footnote = 0
            this_is_a_self_comment= 0
            for entry in ts.misc_info.comment_to_list:
                try:
                    ts_commented = self.conn.textstats[entry.text_no]
                except kom.NoSuchText:
                    continue
                
                # Subject change?
                old_subject = self.conn.subjects[entry.text_no]
                if old_subject <> subject:
                    self.sat.tabulate([ts.author, old_subject, subject])

                # Who comments whom? First both directions separately...
                a1 = ts.author
                a2 = ts_commented.author
                self.ct.tabulate([a1,a2])
                self.rct.tabulate([a2,a1])
                self.dct.tabulate([(a1,a2)])
                # ... then mixed together
                if a1 > a2:
                    (a1, a2) = (a2, a1)
                self.nct.tabulate([(a1,a2)])

                # Find out if the commented text is also in
                # our conference
                recpts_of_commented = map(lambda x: x.recpt,
                                          ts_commented.misc_info.recipient_list)
                if not self.conference in recpts_of_commented:
                    # FIXME: Here, we will count each article once
                    # for every conference the parent was in...
                    for c in recpts_of_commented:
                        self.mat.tabulate([ts.author, c, subject])
                        self.rmat.tabulate([c, ts.author, subject])

                # Footnote?
                if entry.type == kom.MIC_FOOTNOTE:
                    this_is_a_footnote = 1

                # Self-comment?
                if entry.type == kom.MIC_COMMENT and \
                   ts_commented.author == ts.author:
                    this_is_a_self_comment = 1

                # Time between parent and comment
                created = ts.creation_time.to_python_time()
                created_commented = ts_commented.creation_time.to_python_time()
                age = int(created - created_commented)
                if age >= 0:
                    self.ah.tabulate([age, ts.author])

            # Now, tally the things we do not want/need to
            # tally inside the loop
            if this_is_a_footnote:
                self.fnat.tabulate([ts.author])

            if this_is_a_self_comment:
                self.scat.tabulate([ts.author])
                
        # Tabulate things that depend on the comment(s)
        no_c = len(ts.misc_info.comment_in_list)
        self.kh.tabulate([no_c])
        self.cat0.tabulate([ts.author], normal = 0)
        self.cat1.tabulate([ts.author], normal = 0)
        self.cat2.tabulate([ts.author], normal = 0)
        self.cat3.tabulate([ts.author], normal = 0)
        self.cat4.tabulate([ts.author], normal = 0)
        self.cat5.tabulate([ts.author], normal = 0)
        self.cat10.tabulate([ts.author, str(global_no)], normal = 0)
        if no_c == 0:
            self.cat0.tabulate([ts.author], total = 0)
        else:
            if no_c >= 1:
                self.cat1.tabulate([ts.author], total = 0)
                if no_c >= 2:
                    self.cat2.tabulate([ts.author], total = 0)
                    if no_c >= 3:
                        self.cat3.tabulate([ts.author], total = 0)
                        if no_c >= 4:
                            self.cat4.tabulate([ts.author], total = 0)
                            if no_c >= 5:
                                self.cat5.tabulate([ts.author], total = 0)
                                if no_c >= 10:
                                    self.cat10.tabulate([ts.author], total = 0)

        # Get the text and tabulate the words (max size=1024*1024)

        if self.full_text:
            text = kom.ReqGetText(self.conn, global_no, 0, 1024*1024).response()
            for word in split_text_into_words(text):
                self.wt.tabulate([word])
      

    def report(self):
        title = "Statistik för " + conn.conf_name(self.conference)

        lines = ["Statistik för " + conn.conf_name(self.conference),
                 "Perioden är " + self.period.format()]

        if self.exclude_self:
            lines.append("Inlägg av %s är inte medräknade" % conn.conf_name(self.person))
            
        lines.extend([ "", 
                       "Följande rapporter följer som kommentarer till detta inlägg:",
                 ""])
        
        empty_lines = []
        
        for r in self.reports:
            if r.is_empty():
                empty_lines.append("- " + r.get_title()) 
            else:
                lines.append("- " + r.get_title()) 

        if len(empty_lines) > 0:
            lines.append("")
            lines.append("Följande rapporter är tomma och kommer inte att visas:")
            lines.append("")
            lines.extend(empty_lines)

        if len(self.suppressed_reports) > 0:
            lines.append("")
            lines.append("Följande rapporter är bortvalda och kommer inte att visas:")
            lines.append("")
            for r in self.suppressed_reports:
                lines.append("- " + r.get_title()) 

        body = string.strip(string.join(lines, "\n"))
        no = self.write_article(title, body)
            
        for r in self.reports:
            if r.is_empty(): continue
            
            title = r.get_title()
            lines = ["Statistik för " + conn.conf_name(self.conference),
                     ""] + r.report()
            body = string.strip(string.join(lines, "\n"))
            self.write_article(title, body, comment_to = no)

    def write_article(self, title, body, comment_to = None):
        if self.post_to_conference:
            misc_info = kom.CookedMiscInfo()

            mir = kom.MIRecipient(kom.MIR_TO, self.conference)
            misc_info.recipient_list.append(mir)
            
            if comment_to is not None:
                mic = kom.MICommentTo(kom.MIC_COMMENT, comment_to)
                misc_info.comment_to_list.append(mic)

            aux_items = []
            ai = kom.AuxItem(kom.AI_CREATING_SOFTWARE)
            ai.data = "komconfstat %s" % revision
            aux_items.append(ai)

            text = title + "\n" + body
            text_no = kom.ReqCreateText(conn, text,
                                        misc_info, aux_items).response()
            return text_no
        
        else:
            print title
            print "=" * len(title)
            print
            print body
            print

# MAIN
# Connect and log in
# Notice: conn is a global variable used in a lot of places

param = komparam.Parameters(sys.argv[1:])
(conn, conn_error) = param.connect_and_login(kom.CachedConnection)
if conn is None:
    sys.stderr.write("%s: %s\n" % (sys.argv[0], conn_error))
    sys.exit(1)
conn.enable_req_histo()

#
# CHECK FOR OPTIONS
#

conference = None
period = komconfwalker.Period()
mark_earlier = 0
mark_processed = 0
full_text = 0
post_to_conference = 0
exclude_self = 0
debug = 0

options, arguments = getopt.getopt(param.get_arguments(),
                                   "",
                                   ["conference=",
                                    "today",
                                    "yesterday",
                                    "this-week",
                                    "last-week",
                                    "this-month",
                                    "last-month",
                                    "begin=",
                                    "end=",
                                    "mark-earlier", 
                                    "mark-processed",
                                    "full-text",
                                    "post-to-conference",
                                    "exclude-self",
                                    "debug",
                                    ])

for (opt, optarg) in options:
    if opt == "--conference":
        matches = conn.lookup_name(optarg, 1, 1)
        if len(matches) == 0:
            error("%s -- conference not found" % optarg)
        elif len(matches) <> 1:
            error("%s -- ambiguous recipient" % optarg)
        else:
            conference = matches[0][0]
    elif opt == "--today":
        period.set_today()
    elif opt == "--yesterday":
        period.set_yesterday()
    elif opt == "--this-week":
        period.set_this_week()
    elif opt == "--last-week":
        period.set_last_week()
    elif opt == "--this-month":
        period.set_this_month()
    elif opt == "--last-month":
        period.set_last_month()
    elif opt == "--begin":
        period.set_begin(optarg)
    elif opt == "--end":
        period.set_end(optarg)
    elif opt == "--mark-earlier":
        mark_earlier = 1
    elif opt == "--mark-processed":
        mark_processed = 1
    elif opt == "--full-text":
        full_text = 1
    elif opt == "--post-to-conference":
        post_to_conference = 1
    elif opt == "--exclude-self":
        exclude_self = 1
    elif opt == "--debug":
        debug = 1

    else:
        error("Option %s not handled (internal error)" % opt, exit_now = 1)

if conference is None:
    error("No conference given!", exit_now = 1)
    
cstats = ConfStats(conn,
                   param.get_person_no(), conference, period,
                   mark_earlier = mark_earlier,
                   mark_processed = mark_processed,
                   full_text = full_text,
                   post_to_conference = post_to_conference,
                   exclude_self = exclude_self,
                   debug = debug)

cstats.gather()
cstats.report()

# People that are curiosity-challenged will comment out the following
# two lines:
conn.report_cache_usage()
conn.show_req_histo()

# Exit successfully
exit()