diff --git a/baler.py b/baler.py index 1c46803..43e842d 100755 --- a/baler.py +++ b/baler.py @@ -1,6 +1,6 @@ import ConfigParser import csv -import datetime +import datetime as dt import gzip import json import os @@ -11,7 +11,7 @@ def tiq_output(reg_file, enr_file): config = ConfigParser.ConfigParser() config.read('combine.cfg') tiq_dir = os.path.join(config.get('Baler', 'tiq_directory'), 'data') - today = datetime.today().strftime('%Y%m%d') + today = dt.datetime.today().strftime('%Y%m%d') with open(reg_file, 'rb') as f: reg_data = json.load(f) @@ -22,21 +22,27 @@ def tiq_output(reg_file, enr_file): sys.stderr.write('Preparing tiq directory structure under %s\n' % tiq_dir) if not os.path.isdir(tiq_dir): os.makedirs(os.path.join(tiq_dir, 'raw', 'public_inbound')) - os.makedirs(os.path.join(tiq_dir, 'raw', 'public_inbound')) - os.makedirs(os.path.join(tiq_dir, 'enriched', 'public_outbound')) + os.makedirs(os.path.join(tiq_dir, 'raw', 'public_outbound')) + os.makedirs(os.path.join(tiq_dir, 'enriched', 'public_inbound')) os.makedirs(os.path.join(tiq_dir, 'enriched', 'public_outbound')) - inbound_data = [row for row in reg_data if row['direction'] == 'inbound'] - outbound_data = [row for row in reg_data if row['direction'] == 'outbound'] + inbound_data = [row for row in reg_data if row[2] == 'inbound'] + outbound_data = [row for row in reg_data if row[2] == 'outbound'] - bale_reg_csvgz(inbound_data, os.path.join(tiq_dir, 'raw', 'public_inbound', today+'.csv.gz')) - bale_reg_csvgz(outbound_data, os.path.join(tiq_dir, 'raw', 'public_outbound', today+'.csv.gz')) + try: + bale_reg_csvgz(inbound_data, os.path.join(tiq_dir, 'raw', 'public_inbound', today+'.csv.gz')) + bale_reg_csvgz(outbound_data, os.path.join(tiq_dir, 'raw', 'public_outbound', today+'.csv.gz')) + except: + pass - inbound_data = [row for row in enr_data if row['direction'] == 'inbound'] - outbound_data = [row for row in enr_data if row['direction'] == 'outbound'] + inbound_data = [row for row in enr_data if row[2] == 'inbound'] + outbound_data = [row for row in enr_data if row[2] == 'outbound'] - bale_enr_csvgz(inbound_data, os.path.join(tiq_dir, 'enriched', 'public_inbound', today+'.csv.gz')) - bale_enr_csvgz(outbound_data, os.path.join(tiq_dir, 'enriched', 'public_outbound', today+'.csv.gz')) + try: + bale_enr_csvgz(inbound_data, os.path.join(tiq_dir, 'enriched', 'public_inbound', today+'.csv.gz')) + bale_enr_csvgz(outbound_data, os.path.join(tiq_dir, 'enriched', 'public_outbound', today+'.csv.gz')) + except: + pass # oh my god this is such a hack