From dfa8e24ebb2e7c6eca112beb9381c607f67e748a Mon Sep 17 00:00:00 2001 From: Andrew Aldridge Date: Wed, 29 Jul 2015 18:55:22 -0400 Subject: [PATCH] Make stage and test work in project subdirectories --- giza/giza/cmdline.py | 2 +- giza/giza/config/main.py | 4 ++-- giza/giza/config/paths.py | 6 ++++++ giza/giza/operations/stage.py | 38 +++++++++++++++++++---------------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/giza/giza/cmdline.py b/giza/giza/cmdline.py index 1e4412ce2..f21f4f519 100644 --- a/giza/giza/cmdline.py +++ b/giza/giza/cmdline.py @@ -47,7 +47,7 @@ giza.operations.sphinx_cmds.main, giza.operations.deploy.twofa_code, giza.operations.http_serve.start, - giza.operations.stage.start, + giza.operations.stage.main, giza.operations.configuration.report_version, giza.operations.make.main, giza.operations.test.integration_main diff --git a/giza/giza/config/main.py b/giza/giza/config/main.py index 6957aa724..e7c97d085 100644 --- a/giza/giza/config/main.py +++ b/giza/giza/config/main.py @@ -132,7 +132,7 @@ def deploy(self): @deploy.setter def deploy(self, value): - fn = os.path.join(self.paths.global_config, 'deploy.yaml') + fn = os.path.join(self.paths.projectroot, self.paths.global_config, 'deploy.yaml') if os.path.exists(fn): self.state['deploy'] = DeployConfig(fn) else: @@ -146,7 +146,7 @@ def test(self): @test.setter def test(self, value): - fn = os.path.join(self.paths.global_config, 'test-matrix.yaml') + fn = os.path.join(self.paths.projectroot, self.paths.global_config, 'test-matrix.yaml') if os.path.exists(fn): self.state['test'] = TestConfig(fn) else: diff --git a/giza/giza/config/paths.py b/giza/giza/config/paths.py index ae8cf49c2..814f4e886 100644 --- a/giza/giza/config/paths.py +++ b/giza/giza/config/paths.py @@ -203,3 +203,9 @@ def htaccess(self): self.state['htaccess'] = p return self.state['htaccess'] + + @property + def file_changes_database(self): + """Returns a path to the database containing output path mtimes and + hashes to back FileCollector.""" + return os.path.join(self.projectroot, self.output, 'stage-cache.db') diff --git a/giza/giza/operations/stage.py b/giza/giza/operations/stage.py index f17064fba..33ec71174 100644 --- a/giza/giza/operations/stage.py +++ b/giza/giza/operations/stage.py @@ -140,7 +140,7 @@ def run_pool(tasks, n_workers=50): class FileCollector(object): """Database for detecting changed files.""" - def __init__(self, namespace, db_path='build/.stage-cache.db'): + def __init__(self, namespace, db_path): self.namespace = namespace self.conn = sqlite3.connect(db_path) self.conn.row_factory = sqlite3.Row @@ -277,19 +277,22 @@ class Staging(object): S3_OPTIONS = {'reduced_redundancy': True} RESERVED_BRANCHES = {'cache'} - def __init__(self, branch, edition, bucket, username): - if branch in self.RESERVED_BRANCHES: - raise ValueError(branch) - - self.branch = branch - self.edition = edition - self.username = username - + def __init__(self, namespace, bucket, conf): # The S3 prefix for this staging site - self.full_prefix = '/'.join([x for x in (username, branch, edition) if x]) + self.namespace = namespace self.bucket = bucket - self.collector = FileCollector(self.full_prefix) + self.collector = FileCollector(self.namespace, conf.paths.file_changes_database) + + @classmethod + def compute_namespace(cls, username, branch, edition): + """Staging places each stage under a unique namespace computed from an + arbitrary username, branch, and edition. This helper returns such + a namespace, appropriate for constructing a new Staging instance.""" + if branch in cls.RESERVED_BRANCHES: + raise ValueError(branch) + + return '/'.join([x for x in (username, branch, edition) if x]) def purge(self): """Remove all files associated with this branch and edition.""" @@ -297,14 +300,14 @@ def purge(self): # inconsistent state, we want to err on the side of reuploading too much self.collector.purge_now() - keys = [k.key for k in self.bucket.list(prefix='/'.join((self.full_prefix, '')))] + keys = [k.key for k in self.bucket.list(prefix='/'.join((self.namespace, '')))] result = self.bucket.delete_keys(keys) if result.errors: raise SyncException(result.errors) def stage(self, root): """Synchronize the build directory with the staging bucket under - [username]/[branch]/[edition]/""" + the namespace [username]/[branch]/[edition]/""" tasks = [] # Ensure that the root ends with a trailing slash to make future @@ -335,7 +338,7 @@ def stage(self, root): # Remove from staging any files that our FileCollector thinks have been # deleted locally. - remove_keys = ['/'.join((self.full_prefix, path.replace(root, '', 1))) + remove_keys = ['/'.join((self.namespace, path.replace(root, '', 1))) for path in self.collector.removed_files] if remove_keys: LOGGER.info('Removing %s', remove_keys) @@ -348,7 +351,7 @@ def stage(self, root): return def __upload(self, local_path, src_path, file_hash): - full_name = '/'.join((self.full_prefix, local_path)) + full_name = '/'.join((self.namespace, local_path)) k = boto.s3.key.Key(self.bucket) try: @@ -429,7 +432,7 @@ def print_stage_report(url, username, branch, editions): @argh.arg('--builder', '-b', default='html') @argh.named('stage') @argh.expects_obj -def start(args): +def main(args): """Start an HTTP server rooted in the build directory.""" conf = fetch_config(args) staging_config = conf.deploy.get_staging(conf.project.name) @@ -480,7 +483,8 @@ def start(args): try: bucket = conn.get_bucket(staging_config.bucket) for root, edition in zip(roots, editions): - staging = Staging(branch, edition, bucket, username=username) + namespace = Staging.compute_namespace(username, branch, edition) + staging = Staging(namespace, bucket, conf) if args._destage: staging.purge()