"This is now live" automatic notifications

bin/lib/amazon.py

@@ -206,6 +206,11 @@ def release_for(releases, s3_key):
     return None
 
 
+def get_current_release(cfg: Config) -> Optional[Release]:
+    current = get_current_key(cfg)
+    return release_for(get_releases(), current) if current else None
+
+
 def get_events_file(cfg: Config) -> str:
     try:
         o = s3_client.get_object(Bucket="compiler-explorer", Key=events_file_for(cfg))
@@ -221,6 +226,7 @@ def save_event_file(cfg: Config, contents: str):
         Body=contents,
         ACL="public-read",
         CacheControl="public, max-age=60",
+        ContentType="application/json",
     )
 
 
@@ -355,6 +361,7 @@ def put_bouncelock_file(cfg: Config):
         Body="",
         ACL="public-read",
         CacheControl="public, max-age=60",
+        ContentType="text/plain",
     )
 
 
@@ -368,3 +375,38 @@ def has_bouncelock_file(cfg: Config):
         return True
     except s3_client.exceptions.NoSuchKey:
         return False
+
+
+def notify_file_name():
+    return "ce-notify-file"
+
+
+def has_notify_file():
+    try:
+        s3_client.head_object(Bucket="compiler-explorer", Key=notify_file_name())
+        return True
+    except (s3_client.exceptions.NoSuchKey, botocore.exceptions.ClientError):
+        return False
+
+
+def put_notify_file(body: str):
+    s3_client.put_object(
+        Bucket="compiler-explorer", Key=notify_file_name(), Body=body, ACL="public-read", ContentType="text/plain"
+    )
+
+
+def delete_notify_file():
+    s3_client.delete_object(Bucket="compiler-explorer", Key=notify_file_name())
+
+
+def set_current_notify(sha: str):
+    if not has_notify_file():
+        put_notify_file(sha)
+
+
+def get_current_notify():
+    try:
+        o = s3_client.get_object(Bucket="compiler-explorer", Key=notify_file_name())
+        return o["Body"].read().decode("utf-8")
+    except s3_client.exceptions.NoSuchKey:
+        return None

bin/lib/cli/builds.py

@@ -28,6 +28,7 @@
     put_bouncelock_file,
     delete_bouncelock_file,
     has_bouncelock_file,
+    set_current_notify,
 )
 from lib.cdn import DeploymentJob
 from lib.ce_utils import describe_current_release, are_you_sure, display_releases, confirm_branch, confirm_action
@@ -120,6 +121,9 @@ def builds_set_current(cfg: Config, branch: Optional[str], version: str, raw: bo
             old_deploy_staticfiles(branch, to_set)
         set_current_key(cfg, to_set)
         if release:
+            if cfg.env.value == cfg.env.PROD:
+                print("Logging for notifications")
+                set_current_notify(release.hash.hash)
             print("Marking as a release in sentry...")
             token = get_ssm_param("/compiler-explorer/sentryAuthToken")
             result = requests.post(

bin/lib/cli/environment.py

@@ -2,11 +2,20 @@
 
 import click
 
-from lib.amazon import get_autoscaling_groups_for, as_client
+from lib.amazon import (
+    get_autoscaling_groups_for,
+    as_client,
+    get_current_release,
+    get_current_notify,
+    get_ssm_param,
+    delete_notify_file,
+)
 from lib.ce_utils import are_you_sure, describe_current_release, set_update_message
 from lib.cli import cli
 from lib.env import Config, Environment
 
+from lib.notify import handle_notify
+
 
 @cli.group()
 def environment():
@@ -54,14 +63,17 @@ def environment_start(cfg: Config):
     help="Set the message of the day used during refresh",
     show_default=True,
 )
+@click.option("--notify/--no-notify", help="Send GitHub notifications for newly released PRs", default=True)
 @click.pass_obj
-def environment_refresh(cfg: Config, min_healthy_percent: int, motd: str):
+def environment_refresh(cfg: Config, min_healthy_percent: int, motd: str, notify: bool):
     """Refreshes an environment.
 
     This replaces all the instances in the ASGs associated with an environment with
     new instances (with the latest code), while ensuring there are some left to handle
     the traffic while we update."""
     set_update_message(cfg, motd)
+    current_release = get_current_release(cfg)
+
     for asg in get_autoscaling_groups_for(cfg):
         group_name = asg["AutoScalingGroupName"]
         if asg["DesiredCapacity"] == 0:
@@ -105,6 +117,12 @@ def environment_refresh(cfg: Config, min_healthy_percent: int, motd: str):
                 last_log = log
             if status in ("Successful", "Failed", "Cancelled"):
                 break
+    if cfg.env.value == cfg.env.PROD and notify:
+        current_notify = get_current_notify()
+        if current_notify is not None and current_release is not None:
+            gh_token = get_ssm_param("/compiler-explorer/githubAuthToken")
+            handle_notify(current_notify, current_release.hash.hash, gh_token)
+            delete_notify_file()
     set_update_message(cfg, "")
 
 

bin/lib/cli/notify.py

@@ -0,0 +1,19 @@
+import click
+
+from lib.amazon import has_notify_file, delete_notify_file, set_current_notify
+from lib.cli import cli
+
+
+@cli.group()
+def notify():
+    """Now-live notification manipulation commands."""
+
+
+@notify.command(name="set_base")
+@click.argument("sha", type=str)
+def notify_current(sha: str):
+    """Sets the first commit from which to start notifying to a specific one.
+    The commit hash is not validated, so make sure to add a valid one"""
+    if has_notify_file():
+        delete_notify_file()
+    set_current_notify(sha)

bin/lib/notify.py

@@ -0,0 +1,189 @@
+import urllib.request
+import urllib.parse
+import json
+from typing import List
+
+OWNER_REPO = "compiler-explorer/compiler-explorer"
+USER_AGENT = "CE Live Now Notification Bot"
+
+NOW_LIVE_LABEL = "live"
+NOW_LIVE_MESSAGE = "This is now live"
+
+
+def post(entity: str, token: str, query: dict = None, dry=False) -> dict:
+    try:
+        if query is None:
+            query = {}
+        path = entity
+        querystring = json.dumps(query).encode()
+        print(f"Posting {path}")
+        req = urllib.request.Request(
+            f"https://api.github.com/{path}",
+            data=querystring,
+            headers={
+                "User-Agent": USER_AGENT,
+                "Authorization": f"token {token}",
+                "Accept": "application/vnd.github.v3+json",
+            },
+        )
+        if dry:
+            return {}
+        result = urllib.request.urlopen(req)
+        # It's ok not to check for error codes here. We'll throw either way
+        return json.loads(result.read())
+    except Exception as e:
+        raise RuntimeError(f"Error while posting {entity}") from e
+
+
+def get(entity: str, token: str, query: dict = None) -> dict:
+    try:
+        if query is None:
+            query = {}
+        path = entity
+        if query:
+            querystring = urllib.parse.urlencode(query)
+            path += f"?{querystring}"
+        print(f"Getting {path}")
+        req = urllib.request.Request(
+            f"https://api.github.com/{path}",
+            None,
+            {
+                "User-Agent": USER_AGENT,
+                "Authorization": f"token {token}",
+                "Accept": "application/vnd.github.v3+json",
+            },
+        )
+        result = urllib.request.urlopen(req)
+        # It's ok not to check for error codes here. We'll throw either way
+        return json.loads(result.read())
+    except Exception as e:
+        raise RuntimeError(f"Error while getting {entity}") from e
+
+
+def paginated_get(entity: str, token: str, query: dict = None) -> List[dict]:
+    if query is None:
+        query = {}
+    result: List[dict] = []
+    results_per_page = 50
+    query["page"] = 1
+    query["per_page"] = results_per_page
+    while True:
+        current_page_results = get(entity, token, query)
+        result.extend(current_page_results)
+        if len(current_page_results) == results_per_page:
+            query["page"] += 1
+        else:
+            break
+    return result
+
+
+def list_inbetween_commits(end_commit: str, new_commit: str, token: str) -> List[dict]:
+    commits = get(f"repos/{OWNER_REPO}/compare/{end_commit}...{new_commit}", token=token)
+    return commits["commits"]
+
+
+def get_linked_pr(commit: str, token: str) -> dict:
+    """Returns a list whose items are the PR associated to each commit"""
+    pr = get(f"repos/{OWNER_REPO}/commits/{commit}/pulls", token=token)
+    return pr[0] if len(pr) == 1 else {}
+
+
+def get_linked_issues(pr: str, token: str):
+    query = (
+        """
+query {
+  repository(owner: "compiler-explorer", name: "compiler-explorer") {
+    pullRequest(number: %s) {
+      closingIssuesReferences(first: 10) {
+        edges {
+          node {
+            labels(first: 10) {
+              edges {
+                node {
+                  name
+                }
+              }
+            }
+            number
+          }
+        }
+      }
+    }
+  }
+}
+    """
+        % pr
+    )
+    return post("graphql", token, {"query": query})
+
+
+def get_issue_comments(issue: str, token: str) -> List[dict]:
+    return paginated_get(f"repos/{OWNER_REPO}/issues/{issue}/comments", token)
+
+
+def comment_on_issue(issue: str, msg: str, token: str):
+    result = post(f"repos/{OWNER_REPO}/issues/{issue}/comments", token, {"body": msg})
+    return result
+
+
+def set_issue_labels(issue: str, labels: List[str], token: str):
+    post(f"repos/{OWNER_REPO}/issues/{issue}/labels", token, {"labels": labels})
+
+
+def should_send_comment_to_issue(issue: str, token: str):
+    """Only send a comment to the issue if nothing like the live message is there already"""
+    comments = get_issue_comments(issue, token)
+    return all([NOW_LIVE_MESSAGE not in comment["body"] for comment in comments])
+
+
+def send_live_message(issue: str, token: str):
+    set_issue_labels(issue, [NOW_LIVE_LABEL], token)
+    if should_send_comment_to_issue(issue, token):
+        comment_on_issue(issue, NOW_LIVE_MESSAGE, token)
+
+
+def get_edges(issue: dict) -> List[dict]:
+    return issue["data"]["repository"]["pullRequest"]["closingIssuesReferences"]["edges"]
+
+
+def should_process_pr(pr_labels):
+    """Only process PRs that do not have the live label already set"""
+    return all([label["name"] != NOW_LIVE_LABEL for label in pr_labels])
+
+
+def should_notify_issue(edge) -> bool:
+    """We want to notify the issue if:
+    - there's one linked ("number" in edge) AND
+    - either:
+      - the linked issue has no labels ("labels" not in edge["node"]) OR
+      - the NOW_LIVE_LABEL label is not among its labels"""
+    return "number" in edge and (
+        ("labels" not in edge) or all([label["node"]["name"] != NOW_LIVE_LABEL for label in edge["labels"]["edges"]])
+    )
+
+
+def handle_notify(base, new, token):
+    print(f"Checking for live notifications from {base} to {new}")
+
+    commits = list_inbetween_commits(base, new, token)
+    prs = [get_linked_pr(commit["sha"], token) for commit in commits]
+
+    for pr_data in prs:
+        pr_id = pr_data["number"]
+        if should_process_pr(pr_data["labels"]):
+            print(f"Notifying PR {pr_id}")
+            send_live_message(pr_id, token)
+
+            linked_issues = get_linked_issues(pr_id, token)
+            issues_edges = get_edges(linked_issues)
+            if len(issues_edges) == 1 and "node" in issues_edges[0]:
+                edge = issues_edges[0]["node"]
+                if should_notify_issue(edge):
+                    print(f"Notifying issue {edge['number']}")
+                    send_live_message(edge["number"], token)
+                else:
+                    print(f"Skipping notifying issue {edge['number']}")
+            else:
+                print(f"No issues in which to notify for PR {pr_id}")
+        else:
+            print(f"Skipping notifying PR {pr_id}")