Chromium

.hgignore

@@ -10,3 +10,4 @@ REVISION$
 htmlcov$
 .DS_Store$
 workspace.xml$
+mapreduce/.*

Makefile

@@ -1,17 +1,42 @@
 # Makefile to simplify some common AppEngine actions.
 # Use 'make help' for a list of commands.
 
+# Helper code to detect SDK location
+define DETECT_SDK
+import os
+locations = [
+  "../google_appengine",
+  "/usr/local/google_appengine",
+  ".locally/google_appengine",
+]
+for path in locations:
+  if os.path.exists(path):
+    print(path)
+    break
+endef
+# /Helper
+
 APPID?= `cat app.yaml | sed -n 's/^application: *//p'`
+STAGEID= $(APPID)-staging
 
-SDK_PATH ?=
+SDK_PATH ?= $(shell python -c '$(DETECT_SDK)')
 
 DEV_APPSERVER?= $(if $(SDK_PATH), $(SDK_PATH)/,)dev_appserver.py
 DEV_APPSERVER_FLAGS?=
 
 APPCFG?= $(if $(SDK_PATH), $(SDK_PATH)/,)appcfg.py
 APPCFG_FLAGS?=
 
-PYTHON?= python2.5
+# Set dirty suffix depending on output of "hg status".
+dirty=
+ifneq ($(shell hg status),)
+        dirty="-tainted"
+endif
+VERSION_TAG= `hg parents --template='{rev}:{node|short}'`$(dirty)
+# AppEngine version cannot use ':' in its name so use a '-' instead.
+VERSION?= `hg parents --template='{rev}-{node|short}'`$(dirty)
+
+PYTHON?= python2.7
 COVERAGE?= coverage
 
 
@@ -23,40 +48,66 @@ help:
 
 run: serve
 
-serve: update_revision
+serve: update_revision mapreduce
 	@echo "---[Starting SDK AppEngine Server]---"
 	$(DEV_APPSERVER) $(DEV_APPSERVER_FLAGS) .
 
 serve_remote: update_revision
-	$(DEV_APPSERVER) $(DEV_APPSERVER_FLAGS) --address 0.0.0.0 .
+	$(DEV_APPSERVER) $(DEV_APPSERVER_FLAGS) --host 0.0.0.0  --admin_host 0.0.0.0 .
 
 serve_email: update_revision
 	$(DEV_APPSERVER) $(DEV_APPSERVER_FLAGS) --enable_sendmail .
 
 serve_remote_email: update_revision
-	$(DEV_APPSERVER) $(DEV_APPSERVER_FLAGS) --enable_sendmail --address 0.0.0.0 .
+	$(DEV_APPSERVER) $(DEV_APPSERVER_FLAGS) --enable_sendmail --host 0.0.0.0 --admin_host 0.0.0.0 .
 
 update_revision:
 	@echo "---[Updating REVISION]---"
-	@echo "`hg id -n`:`hg id -i`" >REVISION
+	@echo "$(VERSION_TAG)" >REVISION
 
-update: update_revision
+update: update_revision mapreduce update_backend
 	@echo "---[Updating $(APPID)]---"
-	$(APPCFG) $(APPCFG_FLAGS) update . --application $(APPID) --version $(VERSION)
+	$(APPCFG) $(APPCFG_FLAGS) update . --oauth2 --application $(APPID) --version $(VERSION)
+
+update_backend: update_revision mapreduce
+	@echo "---[Updating backend $(APPID)]---"
+	$(APPCFG) $(APPCFG_FLAGS) backends update . --oauth2 --application $(APPID) --version $(VERSION)
 
 upload: update
 
 deploy: update
 
 update_indexes:
-	$(APPCFG) $(APPCFG_FLAGS) update_indexes .
+	$(APPCFG) $(APPCFG_FLAGS) update_indexes . --oauth2 --application $(APPID)
 
 vacuum_indexes:
-	$(APPCFG) $(APPCFG_FLAGS) vacuum_indexes .
+	$(APPCFG) $(APPCFG_FLAGS) vacuum_indexes . --oauth2 --application $(APPID)
 
 test:
-	$(PYTHON) tests/run_tests.py $(SDK_PATH)
+	$(PYTHON) tests/run_tests.py
 
 coverage:
 	$(COVERAGE) run --branch tests/run_tests.py $(SDK_PATH)
 	$(COVERAGE) html --include="codereview/*"
+
+# Checkout mapreduce library and apply a little patch.
+# See https://code.google.com/p/appengine-mapreduce/issues/detail?id=174
+mapreduce:
+	svn co -r 491 http://appengine-mapreduce.googlecode.com/svn/trunk/python/src/mapreduce
+	cd mapreduce/ && patch < ../mapreduce.patch
+
+
+# AppEngine apps can be tested locally and in non-default versions upload to
+# the main app-id, but it is still sometimes useful to have a completely
+# separate app-id.  E.g., for testing inbound email, load testing, or something
+# that might clutter the real datastore.
+stage: update_revision mapreduce stage_backend
+	@echo "---[Staging $(STAGEID)]---"
+	$(APPCFG) $(APPCFG_FLAGS) update . --oauth2 --application $(STAGEID) --version $(VERSION)
+
+stage_backend: update_revision mapreduce
+	@echo "---[Staging backend $(STAGEID)]---"
+	$(APPCFG) $(APPCFG_FLAGS) backends update . --oauth2 --application $(STAGEID) --version $(VERSION)
+
+stage_indexes:
+	$(APPCFG) $(APPCFG_FLAGS) update_indexes . --oauth2 --application $(STAGEID)

README

@@ -1,8 +1,10 @@
 Welcome to Rietveld
 -------------------
 
-This project shows how to create a somewhat substantial web application
-using Django on Google App Engine.  It requires Django version 1.2.
+This project shows how to create a somewhat substantial web
+application using Django on Google App Engine.  It requires Python 2.7
+and Django version 1.3 (although a previous version using Python 2.5
+and Django 1.2 can still be found in the py25 branch in the repository).
 
 In addition, I hope it will serve as a practical tool for the Python
 developer community, and hopefully for other open source communities.
@@ -21,7 +23,7 @@ Links
 
 Mondrian video: http://www.youtube.com/watch?v=sMql3Di4Kgc
 Google App Engine: http://code.google.com/appengine/
-Live app: http://codereview.appspot.com
+Live app: http://codereview.chromium.org
 About code review: http://en.wikipedia.org/wiki/Code_review
 Django: http://djangoproject.com
 Python: http://python.org
@@ -43,20 +45,21 @@ can then run the server using
 (assuming you're on Linux or Mac OS X).  On Windows just use Google
 App Engine Launcher.
 
-Rietveld uses Django 1.2 libraries.  They are included in App Engine
-SDK version 1.4.2 and above.
+Please make sure that you have the most recent version of the App Engine SDK
+installed when running Rietveld locally. That's the version that runs in the
+production environment too and Rietveld often uses new features.
 
 The server is only accessible on http://localhost:8080.  The server in
 the Google App Engine SDK is not designed for serving real traffic.
-The App Engine FAQ at http://code.google.com/appengine/kb/general.html
+The App Engine FAQ at https://developers.google.com/appengine/kb/general
 says about this: "You can override this using the -a <hostname> flag
 when running it, but doing so is not recommended because the SDK has
 not been hardened for security and may contain vulnerabilities."
 
 To deploy your own instance of the app to Google App Engine:
 
   1. Register your own application ID on the App Engine admin site.
-  2. Edit app.yaml to use this app ID instead of 'codereview-hr'.
+  2. Edit app.yaml to use this app ID instead of 'chromiumcodereview'.
   3. Upload using
 
     make update VERSION=123
@@ -65,15 +68,17 @@ To deploy your own instance of the app to Google App Engine:
 you'll get a error message from "appcfg.py update" (called by "make
 update") complaining you don't have the right to administer this app.
 
-*** Don't update the Python runtime in app.yaml to Python 2.7!
-Currently Rietveld doesn't support Python 2.7 on App Engine due to
-some thread-safety issues (see
-http://code.google.com/p/rietveld/issues/detail?id=348).
-
 The VERSION=xxx argument sets the version; the version from the
 app.yaml is not used.  This is to support a convention used for the
-main Rietveld instance (codereview.appspot.com) whereby we never
+main Rietveld instance (codereview.chromium.org) whereby we never
 deploy to the same version twice; the version must be manually picked
 by the developer doing the deployment.  If you don't like this, just
 edit the Makefile to remove "--version $(VERSION)" and edit app.yaml
 to hardcode a version number.
+
+Administration
+--------------
+
+Various jobs to administer an instance are collected in admin_tasks.py. These
+jobs can be run by an instance administrator by visiting http://<your-
+instance>/mapreduce/.

TODO

@@ -23,13 +23,6 @@ Patch Sets
 Edit patch set message
 Delete patch sets
 
-Repositories and Branches
--------------------------
-
-Get rid of the repository feature in favor of always using upload.py?
-Edit/Delete Repository?
-Automatically add Repository entries when a new project uploads stuff?
-
 Searching, Organizing
 ---------------------
 
@@ -67,3 +60,5 @@ Software Engineering
 --------------------
 
 Unittests
+Move library.py to codereview/templatetags/codereview.py to match the default
+Django project layout.

admin_tasks.py

@@ -0,0 +1,29 @@
+"""Collection of mapreduce jobs."""
+
+import logging
+from mapreduce import operation as op
+from codereview.models import Account, Issue
+
+
+def delete_unused_accounts(account):
+  """Delete accounts for uses that don't participate in any reviews."""
+  email = account.user.email()
+  if Issue.query(Issue.owner_email == email).get():
+    return
+  if Issue.query(Issue.cc == email).get():
+    return
+  if Issue.query(Issue.reviewers == email).get():
+    return
+  logging.warn('Deleting %s' % email)
+  yield op.db.Delete(account)
+
+
+def update_account_schema(account):
+  """Update schema for all Accounts by saving them back to the datastore."""
+
+  # Make sure we don't alter the modified time of any accounts. Because of how
+  # mapreduce is designed, we just set this to False on every function
+  # invocation (since there's no convenient once-per-instance place to do it).
+  Account.modified.auto_now = False
+
+  yield op.db.Put(account)

app.yaml

@@ -1,10 +1,17 @@
-application: codereview-hr
+application: chromiumcodereview-hr
 version: use-version-arg
-runtime: python
+runtime: python27
 api_version: 1
+threadsafe: false
+
+includes:
+- mapreduce/include.yaml
 
 default_expiration: 7d  # This is good for images, which never change
 
+# The commented "secure: always" handler can't be enabled for appspot.com
+# applications with a custom domain, since GAE doesn't support HTTPS on custom
+# domains at the moment.
 handlers:
 
 - url: /(robots.txt|favicon.ico)
@@ -16,24 +23,53 @@ handlers:
   upload: static/robots.txt
 
 - url: /static/upload.py
-  script: main.py
+  script: main.application
+#  secure: always
 
+# These are fetched with the rietveld version number in the URL.
 - url: /static/(script.js|styles.css)
   static_files: static/\1
   upload: static/(script.js|styles.css)
-  expiration: 1h  # Shorter expiration, these change often
+#  secure: always
 
 - url: /static
   static_dir: static
+#  secure: always
+
+# Admin-only URLs like tasks or cron jobs. See cron.yaml and queue.yaml.
+- url: /restricted/.*
+  script: main.application
+  login: admin
+
+# These seem to be hit over http and cannot deal with a 302 to https.
+- url: /_ah/(start|stop)
+  script: backends.app
+  login: admin
 
-- url: .*
-  script: main.py
+- url: /.*
+  script: main.application
+#  secure: always
+
+admin_console:
+  pages:
+  - name: Client ID and Secret Settings
+    url: /restricted/set-client-id-and-secret
 
 inbound_services:
 - mail
 - xmpp_message
 - warmup
 
 builtins:
-- appstats: on
+# To reenable appstats, do not forget to update appengine_config.py too.
+#- appstats: on
 - remote_api: on
+
+libraries:
+- name: django
+  version: 1.4
+- name: webapp2
+  version: 2.5.2
+
+env_variables:
+  DJANGO_SETTINGS_MODULE: 'settings'

appengine_config.py

@@ -3,14 +3,16 @@
 import logging
 import os
 import re
+import sys
 
 from google.appengine.ext.appstats import recording
 
 logging.info('Loading %s from %s', __name__, __file__)
 
 # Custom webapp middleware to add Appstats.
 def webapp_add_wsgi_middleware(app):
-  app = recording.appstats_wsgi_middleware(app)
+  # If reenabling appstats, do not forget to reeanble it in app.yaml too.
+  #app = recording.appstats_wsgi_middleware(app)
   return app
 
 # Custom Appstats path normalization.
@@ -31,12 +33,14 @@ def appstats_normalize_path(path):
             return path[:i] + '/X'
     return re.sub(r'\d+', 'X', path)
 
-# Declare the Django version we need.
-from google.appengine.dist import use_library
-use_library('django', '1.2')
+# Segregate Appstats by runtime (python vs. python27).
+appstats_KEY_NAMESPACE = '__appstats_%s__' % os.getenv('APPENGINE_RUNTIME')
 
-# Django 1.2 requires DJANGO_SETTINGS_MODULE environment variable to be set
-# http://code.google.com/appengine/docs/python/tools/libraries.html#Django 
-os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
-# NOTE: All "main" scripts must import webapp.template before django.
+# Enable Interactive Playground.
+appstats_SHELL_OK = True
 
+# Enable RPC cost calculation.
+appstats_CALC_RPC_COSTS = True
+
+# Enable third-party imports
+sys.path.append(os.path.join(os.path.dirname(__file__), 'third_party'))

backends.py

@@ -0,0 +1,23 @@
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import webapp2
+
+class BackendHandler(webapp2.RequestHandler):
+  def get(self):
+    pass
+
+app = webapp2.WSGIApplication(routes=[
+  (r'/_ah/.*', BackendHandler),
+])

backends.yaml

@@ -0,0 +1,4 @@
+backends:
+- name: mapreduce-backend
+  instances: 10
+  options: dynamic