Remove remaining mx.DateTime uses.

filtersentence_xml.py

@@ -1,10 +1,9 @@
 #! /usr/bin/python
 
+from datetime import datetime
 import re
 import string
 
-import mx.DateTime
-
 from lxml import etree
 
 from contextexception import ContextException
@@ -40,7 +39,7 @@
 reqnum = re.compile("\s*\[(\d+)\]\s*$")
 refqnum = re.compile("\s*\[(\d+)\]\s*")
 
-redatephraseval = re.compile('(?:(?:%s),?)?(\d+(?: | )*(?:%s)(\d+)?)' % (parlPhrases.daysofweek, parlPhrases.monthsofyear))
+redatephraseval = re.compile('(?:(?:%s),? )?(\d+(?: | )*(?:%s)( \d+)?)' % (parlPhrases.daysofweek, parlPhrases.monthsofyear))
 
 
 def TokenDate(ldate, phrtok):
@@ -49,9 +48,8 @@ def TokenDate(ldate, phrtok):
     if not ldate.group(2):
         tdate += " %s" % sdate_year
     try:
-        lldate = mx.DateTime.DateTimeFrom(tdate)
-        ldate = lldate.date
-        phrtok.lastdate = ldate
+        lldate = datetime.strptime(tdate, '%A, %d %B %Y')
+        phrtok.lastdate = lldate.date
     except:
         phrtok.lastdate = ''
     return ('phrase', ' class="date" code="%s"' % phrtok.lastdate)
@@ -166,10 +164,10 @@ def TokenOffRepWDate(qoffrep, phrtok):
     m = re.match('(\d+)/(\d+)/(\d+)', date)
     if m:
         lordsdate = True
-        date = mx.DateTime.DateTimeFrom('%s/%s/%s' % (m.group(2), m.group(1), m.group(3))).date
+        date = datetime.strptime(date, '%d/%m/%Y').date
     else:
         lordsdate = False
-        date = mx.DateTime.DateTimeFrom(date).date
+        date = datetime.strptime(date, '%d %B %Y').date
     if qcolprefix:
         qcolprefix = qcolprefix.upper()
     if qcolsuffix:

pyscraper/new_hansard.py

@@ -1,14 +1,14 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import datetime
 import re
 import os
 import sys
 import io
 import tempfile
 from lxml import etree
 import xml.sax
-import mx.DateTime
 import miscfuncs
 
 xmlvalidate = xml.sax.make_parser()
@@ -199,10 +199,8 @@ def reset(self):
 
     def is_pre_new_parser(self):
         is_pre = False
-        parser_start = mx.DateTime.Date(2016, 4, 1)
-        file_date = mx.DateTime.DateTimeFrom(self.date)
-
-        if file_date < parser_start:
+        parser_start = datetime.date(2016, 4, 1)
+        if self.date < parser_start:
             is_pre = True
 
         return is_pre
@@ -312,7 +310,7 @@ def new_speech(self, member, url):
 
     def parse_system_header(self, header):
         sitting = header.xpath('./ns:Sitting', namespaces=self.ns_map)[0]
-        date = mx.DateTime.DateTimeFrom(sitting.get('short-date')).date
+        date = datetime.datetime.strptime(sitting.get('short-date'), '%d %B %Y').date
         if date:
             self.date = date
 
@@ -889,8 +887,8 @@ def parse_time(self, tag):
             minutes = int(matches.group(2) or 0)
             if matches.group(3) == 'pm' and hours < 12:
                 hours += 12
-            time = mx.DateTime.DateTimeFrom(hour=hours, minute=minutes)
-            self.current_time = time.strftime('%H:%M:%S')
+            time = datetime.time(hours, minutes)
+            self.current_time = time.isoformat()
         elif time_txt in ('Noon', 'noon') or re.match('12\s*?noon', time_txt):
             self.current_time = "12:00:00"
         elif re.match('12\s*?midnight', time_txt):

pyscraper/regmem/pullgluepages.py

@@ -1,13 +1,13 @@
 #! /usr/bin/python
 
+from datetime import datetime
 import glob
 import sys
 import urllib
 import urlparse
 import re
 import os.path
 import time
-import mx.DateTime
 import tempfile
 import BeautifulSoup
 
@@ -90,7 +90,7 @@ def GlueByNext(fout, url, regmemdate):
             if not dateinpage:
                 raise Exception, 'Not found date marker'
             dateinpage = dateinpage.group(1).replace(" ", " ")
-            dateinpage = mx.DateTime.DateTimeFrom(dateinpage).date
+            dateinpage = datetime.strptime(dateinpage, '%d %B %Y').date
             if dateinpage != regmemdate:
                 raise Exception, 'Date in page is %s, expected %s - update the URL list in regmempullgluepages.py' % (dateinpage, regmemdate)
             matcheddate = True
@@ -225,7 +225,7 @@ def FindRegmemPages(remote):
                 print alldates
                 raise Exception, 'Date match failed, expected one got %d\n%s' % (len(alldates), url)
 
-            date = mx.DateTime.DateTimeFrom(alldates[0]).date
+            date = datetime.strptime(alldates[0], '%d %B %Y').date
             if (date, ixurl) not in urls:
                 urls.append((date, ixurl))
         elif re.search('Session 201[79]|Session 20[2-9]', content):
@@ -261,28 +261,14 @@ def FindRegmemPages(remote):
                     date = corrections[url_path]
                 else:
                     alldates[0] = re.sub('\s+', ' ', alldates[0])
-                    date = mx.DateTime.DateTimeFrom(alldates[0]).date
+                    alldates[0] = re.sub('(?<=\d)(st|nd|rd|th)', '', alldates[0])
+                    date = datetime.strptime(alldates[0], '%d %B %Y').date
 
                 if (date, url) not in urls:
                     urls.append((date, url))
 
     return urls
 
-def FindLordRegmemPages():
-    urls = [('2004-10-01', 'http://www.publications.parliament.uk/pa/ld200304/ldreg/reg01.htm')]
-    ixurl = 'http://www.publications.parliament.uk/pa/ld/ldreg.htm'
-    ur = opener.open(ixurl)
-    content = ur.read()
-    ur.close();
-
-    allurls = re.findall('<a href="([^>]*reg01[^>]*)">.*?position on (.*?)\)</a>(?i)', content)
-    for match in allurls:
-        url = urlparse.urljoin(ixurl, match[0])
-        date = mx.DateTime.DateTimeFrom(match[1]).date
-        urls.append((date, url))
-
-    return urls
-
 ###############
 # main function
 ###############
@@ -307,8 +293,5 @@ def RegmemPullGluePages(options):
     # third parameter is a regexp, fourth is the filename (%s becomes the date).
     GlueAllType(pwcmregmem, urls, 'regmem%s.html', options.forcescrape, options.remote)
 
-    # urls = FindLordRegmemPages()
-    # GlueAllType(pwldregmem, urls, 'regmem%s.html', forcescrape)
-
 if __name__ == '__main__':
     RegmemPullGluePages(False)

requirements.txt

@@ -1,6 +1,5 @@
 BeautifulSoup==3.2.1
 beautifulsoup4==4.3.2
-egenix-mx-base==3.2.9
 everypolitician==0.0.13
 lxml==3.4.0
 python-dateutil==2.2