First

Author: dsjoerg

.gitignore

@@ -0,0 +1,44 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+
+#Translations
+*.mo
+
+#Mr Developer
+.mr.developer.cfg
+
+gg.pid
+
+#emacs poop
+*~
+
+#pip poop
+src
+config
+
+testcache/*

README.md

@@ -0,0 +1,59 @@
+Retrieve jobs queued by ruby in python, and perform the requested parsing work.
+
+Installation
+===============
+
+**TODO** - give commands to set up an acceptable directory structure, with config containing what is needed
+
+Install the various requirements. PIL may require external libraries to be installed.
+
+> $ pip install -r requirements.txt
+
+Set up database configuration
+
+**TODO** - describe how config/database.yml must be symlinked to the containing (esdb) directory.
+**TODO** - give steps to set up AWS for testing
+
+
+Set up your environment variables
+
+* `GGFACTORY_CACHE_DIR` (required): Sets a directory to cache remote files used to load SC2 resources. Use full path.
+* `DJANGO_SECRETKEY` (required): Sets the SECRETKEY in the django settings.py file.
+
+Run the tests to verify a successful setup:
+
+> GGFACTORY_CACHE_DIR=testcache ./manage.py test sc2parse
+
+Run specific tests like this:
+> GGFACTORY_CACHE_DIR=testcache ./manage.py test sc2parse.SC2ReaderToEsdbTestCase.test_close_replays
+
+
+Keeping up to Date
+=====================
+
+When you pull down changes to the `requirements.txt` file:
+
+> $ pip install --upgrade -r requirements.txt
+
+
+
+Working on sc2reader
+========================
+
+Development work on sc2reader requires a manual uninstall of the
+sc2reader version installed by the requirements file followed by
+an installation from a local copy of the ggtracker/sc2reader repo.
+
+> $ pip uninstall sc2reader
+> $ cd path/to/ggtracker/sc2reader
+> $ python setup.py develop
+
+When installed in develop mode sc2reader will automatically reflect
+changes to the code base it was installed from. Please note that
+a worker running as a daemon will *not* automatically reflect changes
+to the code made since the process began running. The worker must be
+shut down and restarted to pick up the code changes.
+
+Git
+===
+Miserable.

__init__.py

@@ -0,0 +1,206 @@
+'''
+  ***
+  Modified generic daemon class
+  ***
+  
+  Author:   http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/
+        www.boxedice.com
+  
+  License:   http://creativecommons.org/licenses/by-sa/3.0/
+  
+  Changes:  23rd Jan 2009 (David Mytton <[email protected]>)
+        - Replaced hard coded '/dev/null in __init__ with os.devnull
+        - Added OS check to conditionally remove code that doesn't work on OS X
+        - Added output to console on completion
+        - Tidied up formatting 
+        11th Mar 2009 (David Mytton <[email protected]>)
+        - Fixed problem with daemon exiting on Python 2.4 (before SystemExit was part of the Exception base)
+        13th Aug 2010 (David Mytton <[email protected]>
+        - Fixed unhandled exception if PID file is empty
+'''
+
+# Core modules
+import atexit
+import os
+import sys
+import time
+import signal
+import errno
+
+class Daemon(object):
+  """
+  A generic daemon class.
+  
+  Usage: subclass the Daemon class and override the run() method
+  """
+  def __init__(self, pidfile, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
+    self.stdin = stdin
+    self.stdout = stdout
+    self.stderr = stderr
+    self.pidfile = pidfile
+    self.home_dir = home_dir
+    self.verbose = verbose
+    self.umask = umask
+    self.daemon_alive = True
+  
+  def daemonize(self):
+    """
+    Do the UNIX double-fork magic, see Stevens' "Advanced 
+    Programming in the UNIX Environment" for details (ISBN 0201563177)
+    http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
+    """
+    try: 
+      pid = os.fork() 
+      if pid > 0:
+        # Exit first parent
+        sys.exit(0) 
+    except OSError, e: 
+      sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
+      sys.exit(1)
+    
+    # Decouple from parent environment
+    os.chdir(self.home_dir)
+    os.setsid() 
+    os.umask(self.umask)
+  
+    # Do second fork
+    try: 
+      pid = os.fork() 
+      if pid > 0:
+        # Exit from second parent
+        sys.exit(0) 
+    except OSError, e: 
+      sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
+      sys.exit(1) 
+  
+    if sys.platform != 'darwin': # This block breaks on OS X
+      # Redirect standard file descriptors
+      sys.stdout.flush()
+      sys.stderr.flush()
+      si = file(self.stdin, 'r')
+      so = file(self.stdout, 'a+')
+      if self.stderr:
+        se = file(self.stderr, 'a+', 0)
+      else:
+        se = so
+      os.dup2(si.fileno(), sys.stdin.fileno())
+      os.dup2(so.fileno(), sys.stdout.fileno())
+      os.dup2(se.fileno(), sys.stderr.fileno())
+
+    def sigtermhandler(signum, frame):
+      self.daemon_alive = False
+    signal.signal(signal.SIGTERM, sigtermhandler)
+    signal.signal(signal.SIGINT, sigtermhandler)
+
+    if self.verbose >= 1:
+      print "Started"
+    
+    # Write pidfile
+    atexit.register(self.delpid) # Make sure pid file is removed if we quit
+    pid = str(os.getpid())
+    file(self.pidfile,'w+').write("%s\n" % pid)
+    
+  def delpid(self):
+    os.remove(self.pidfile)
+
+  def start(self, *args, **kwargs):
+    """
+    Start the daemon
+    """
+    
+    if self.verbose >= 1:
+      print "Starting..."
+    
+    # Check for a pidfile to see if the daemon already runs
+    try:
+      pf = file(self.pidfile,'r')
+      pid = int(pf.read().strip())
+      pf.close()
+    except IOError:
+      pid = None
+    except SystemExit:
+      pid = None
+  
+    if pid:
+      message = "pidfile %s already exists. Is the daemon already running?\n"
+      sys.stderr.write(message % self.pidfile)
+
+      # Remove stale pid files
+      try:
+        os.kill(pid, 0)
+      except OSError, why:
+        if why[0] == errno.ESRCH:
+          print('Removing stale pidfile %s' % self.pidfile)
+          os.remove(self.pidfile)
+        else:
+          sys.exit("Can't check status of PID %s from pidfile %s: %s" %
+                  (pid, self.pidfile, why[1]))
+      else:
+        sys.exit("daemon is already running with PID %s\n" % pid)
+
+    # Start the daemon
+    self.daemonize()    
+    self.run(*args, **kwargs)
+
+  def stop(self):
+    """
+    Stop the daemon
+    """
+    
+    if self.verbose >= 1:
+      print "Stopping..."
+    
+    # Get the pid from the pidfile
+    try:
+      pf = file(self.pidfile,'r')
+      pid = int(pf.read().strip())
+      pf.close()
+    except IOError:
+      pid = None
+    except ValueError:
+      pid = None
+  
+    if not pid:
+      message = "pidfile %s does not exist. Not running?\n"
+      sys.stderr.write(message % self.pidfile)
+      
+      # Just to be sure. A ValueError might occur if the PID file is empty but does actually exist
+      if os.path.exists(self.pidfile):
+        os.remove(self.pidfile)
+      
+      return # Not an error in a restart
+
+    # Try killing the daemon process  
+    try:
+      while 1:
+
+        # Note: kill -SIGTERM will work, doing it from python will not.
+        # I don't feel like finding out why right now. FIXME
+
+        os.kill(pid, signal.SIGKILL)
+        time.sleep(0.1)
+    except OSError, err:
+      err = str(err)
+      if err.find("No such process") > 0:
+        if os.path.exists(self.pidfile):
+          os.remove(self.pidfile)
+      else:
+        print str(err)
+        sys.exit(1)
+    
+    if self.verbose >= 1:
+      print "Stopped"
+
+  def restart(self):
+    """
+    Restart the daemon
+    """
+    self.stop()    
+    self.start()
+
+  def run(self):
+    """
+    You should override this method when you subclass Daemon. It will be called after the process has been
+    daemonized by start() or restart().
+    """
+

daemon.py

@@ -0,0 +1,7 @@
+import sys
+import sc2reader
+from sc2parse import ggfactory
+
+# usage GGFACTORY_CACHE_DIR=testcache python engagement.py <ggtracker match id>
+
+replay = ggfactory.load_replay("http://ggtracker.com/matches/{}/replay".format(sys.argv[1]))

engagement.py

@@ -0,0 +1,36 @@
+NOT OK
+  -rw-r--r--   1 david  staff   6392 Jun 16  2013 settings.py  <--- AWS
+
+FIXED
+  -rw-r--r--   1 david  staff  1335 Jun 16  2013 ggtracker/__init__.py  <--- exceptional api key
+
+OK
+  -rw-r--r--   1 david  staff     43 Jun 16  2013 .git
+  -rw-r--r--   1 david  staff    336 Jun 16  2013 .gitignore
+  -rw-r--r--   1 david  staff   1543 Dec 13  2013 README.md
+  -rw-r--r--   1 david  staff      0 Jun 16  2013 __init__.py
+  drwxr-xr-x   3 david  staff    102 Jul 30  2013 config
+  -rw-r--r--   1 david  staff   5581 Jun 16  2013 daemon.py
+  -rw-r--r--   1 david  staff    234 Jun 16  2013 engagement.py
+  -rw-r--r--   1 david  staff  12359 Jun 16  2013 etracker_fun.txt
+  drwxr-xr-x   4 david  staff    136 Jul 10 20:07 exceptional
+  -rwxr-xr-x   1 david  staff    503 Jun 16  2013 manage.py
+  -rw-r--r--   1 david  staff   2342 Jun 16  2013 plugin_debug.py
+  -rw-r--r--   1 david  staff   1251 Dec 13  2013 profile.py
+  -rw-r--r--   1 david  staff    636 Jun 23 13:39 requirements.txt
+  -rw-r--r--   1 david  staff    384 Jun 16  2013 resque_test.py
+  -rw-r--r--   1 david  staff    137 Jun 16  2013 settings_dev.py
+  -rwxr-xr-x   1 david  staff   1966 Dec 13  2013 worker.py
+  drwxr-xr-x   6 david  staff    204 Jun 28 17:20 src
+  drwxr-xr-x  10 david  staff    340 Jun 28 17:20 testcache
+  -rw-r--r--   1 david  staff  7149 Jun 18 15:01 jobs.py
+  drwxr-xr-x   8 david  staff    272 Jul 10 20:07 ggtracker
+  -rw-r--r--   1 david  staff  2713 Jun 22 08:42 utils.py
+  -rw-r--r--   1 david  staff   3127 Jun 21 19:53 ggfactory.py
+  -rw-r--r--   1 david  staff  10972 Dec 13  2013 models.py
+  -rw-r--r--   1 david  staff  70493 Jun 23 13:40 plugins.py
+  -rw-r--r--   1 david  staff  52966 Jul  2 16:50 sc2reader_to_esdb.py
+  -rw-r--r--   1 david  staff    226 Jun 16  2013 views.py
+  -rw-r--r--   1 david  staff  33213 Dec 13  2013 tests.py
+  drwxr-xr-x  15 david  staff    510 Jul 10 20:07 sc2parse
+  drwxr-xr-x  37 david  staff   1258 Dec 13  2013 testfiles