mach_commands.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import logging
import subprocess
import sys
from datetime import datetime, timedelta
from operator import itemgetter

from mach.decorators import Command, CommandArgument, SubCommand
from mozbuild.base import MozbuildObject


def _get_busted_bugs(payload):
    import requests

    payload = dict(payload)
    payload["include_fields"] = "id,summary,last_change_time,resolution"
    payload["blocks"] = 1543241
    response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload)
    response.raise_for_status()
    return response.json().get("bugs", [])


@Command(
    "busted",
    category="misc",
    description="Query known bugs in our tooling, and file new ones.",
)
def busted_default(command_context):
    unresolved = _get_busted_bugs({"resolution": "---"})
    creation_time = datetime.now() - timedelta(days=15)
    creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ")
    resolved = _get_busted_bugs({"creation_time": creation_time})
    resolved = [bug for bug in resolved if bug["resolution"]]
    all_bugs = sorted(
        unresolved + resolved, key=itemgetter("last_change_time"), reverse=True
    )
    if all_bugs:
        for bug in all_bugs:
            print(
                "[%s] Bug %s - %s"
                % (
                    (
                        "UNRESOLVED"
                        if not bug["resolution"]
                        else "RESOLVED - %s" % bug["resolution"]
                    ),
                    bug["id"],
                    bug["summary"],
                )
            )
    else:
        print("No known tooling issues found.")


@SubCommand("busted", "file", description="File a bug for busted tooling.")
@CommandArgument(
    "against",
    help=(
        "The specific mach command that is busted (i.e. if you encountered "
        "an error with `mach build`, run `mach busted file build`). If "
        "the issue is not connected to any particular mach command, you "
        "can also run `mach busted file general`."
    ),
)
def busted_file(command_context, against):
    import webbrowser

    if (
        against != "general"
        and against not in command_context._mach_context.commands.command_handlers
    ):
        print(
            "%s is not a valid value for `against`. `against` must be "
            "the name of a `mach` command, or else the string "
            '"general".' % against
        )
        return 1

    if against == "general":
        product = "Firefox Build System"
        component = "General"
    else:
        import inspect

        import mozpack.path as mozpath

        # Look up the file implementing that command, then cross-refernce
        # moz.build files to get the product/component.
        handler = command_context._mach_context.commands.command_handlers[against]
        sourcefile = mozpath.relpath(
            inspect.getsourcefile(handler.func), command_context.topsrcdir
        )
        reader = command_context.mozbuild_reader(config_mode="empty")
        try:
            res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
            product, component = res.product, res.component
        except TypeError:
            # The file might not have a bug set.
            product = "Firefox Build System"
            component = "General"

    uri = (
        "https://bugzilla.mozilla.org/enter_bug.cgi?"
        "product=%s&component=%s&blocked=1543241" % (product, component)
    )
    webbrowser.open_new_tab(uri)


def pastebin_create_parser():
    # Accept all args so they can be promptly ignored.
    parser = argparse.ArgumentParser()
    parser.add_argument("argv", nargs=argparse.REMAINDER, help=argparse.SUPPRESS)
    return parser


@Command(
    "pastebin",
    category="misc",
    hidden=True,
    parser=pastebin_create_parser,
)
def pastebin(command_context, argv):
    """Obsolete command line interface to `paste.mozilla.org`."""

    print(
        "pastebin.mozilla.org has been decommissioned.\n"
        "Please use your favorite search engine to find alternatives."
    )
    return 1


class PypiBasedTool:
    """
    Helper for loading a tool that is hosted on pypi. The package is expected
    to expose a `mach_interface` module which has `new_release_on_pypi`,
    `parser`, and `run` functions.
    """

    def __init__(self, module_name, pypi_name=None):
        self.name = module_name
        self.pypi_name = pypi_name or module_name

    def _import(self):
        # Lazy loading of the tools mach interface.
        # Note that only the mach_interface module should be used from this file.
        import importlib

        try:
            return importlib.import_module("%s.mach_interface" % self.name)
        except ImportError:
            return None

    def create_parser(self, subcommand=None):
        # Create the command line parser.
        # If the tool is not installed, or not up to date, it will
        # first be installed.
        cmd = MozbuildObject.from_environment()
        cmd.activate_virtualenv()
        tool = self._import()
        if not tool:
            # The tool is not here at all, install it
            cmd.virtualenv_manager.install_pip_package(self.pypi_name)
            print(
                "%s was installed. please re-run your"
                " command. If you keep getting this message please "
                " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name)
            )
        else:
            # Check if there is a new release available
            release = tool.new_release_on_pypi()
            if release:
                print(release)
                # there is one, so install it. Note that install_pip_package
                # does not work here, so just run pip directly.
                subprocess.check_call(
                    [
                        cmd.virtualenv_manager.python_path,
                        "-m",
                        "pip",
                        "install",
                        f"{self.pypi_name}=={release}",
                    ]
                )
                print(
                    "%s was updated to version %s. please"
                    " re-run your command." % (self.pypi_name, release)
                )
            else:
                # Tool is up to date, return the parser.
                if subcommand:
                    return tool.parser(subcommand)
                else:
                    return tool.parser()
        # exit if we updated or installed mozregression because
        # we may have already imported mozregression and running it
        # as this may cause issues.
        sys.exit(0)

    def run(self, **options):
        tool = self._import()
        tool.run(options)


def mozregression_create_parser():
    # Create the mozregression command line parser.
    # if mozregression is not installed, or not up to date, it will
    # first be installed.
    loader = PypiBasedTool("mozregression")
    return loader.create_parser()


@Command(
    "mozregression",
    category="misc",
    description="Regression range finder for nightly and inbound builds.",
    parser=mozregression_create_parser,
)
def run(command_context, **options):
    command_context.activate_virtualenv()
    mozregression = PypiBasedTool("mozregression")
    mozregression.run(**options)


@Command(
    "node",
    category="devenv",
    description="Run the NodeJS interpreter used for building.",
)
@CommandArgument("args", nargs=argparse.REMAINDER)
def node(command_context, args):
    from mozbuild.nodeutil import find_node_executable

    # Avoid logging the command
    command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)

    node_path, _ = find_node_executable()

    return command_context.run_process(
        [node_path] + args,
        pass_thru=True,  # Allow user to run Node interactively.
        ensure_exit_code=False,  # Don't throw on non-zero exit code.
    )


@Command(
    "npm",
    category="devenv",
    description="Run the npm executable from the NodeJS used for building.",
)
@CommandArgument("args", nargs=argparse.REMAINDER)
def npm(command_context, args):
    from mozbuild.nodeutil import find_npm_executable

    # Avoid logging the command
    command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)

    import os

    # Add node and npm from mozbuild to front of system path
    #
    # This isn't pretty, but npm currently executes itself with
    # `#!/usr/bin/env node`, which means it just uses the node in the
    # current PATH. As a result, stuff gets built wrong and installed
    # in the wrong places and probably other badness too without this:
    npm_path, _ = find_npm_executable()
    if not npm_path:
        exit(-1, "could not find npm executable")
    path = os.path.abspath(os.path.dirname(npm_path))
    os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])

    # karma-firefox-launcher needs the path to firefox binary.
    firefox_bin = command_context.get_binary_path(validate_exists=False)
    if os.path.exists(firefox_bin):
        os.environ["FIREFOX_BIN"] = firefox_bin

    return command_context.run_process(
        [npm_path, "--scripts-prepend-node-path=auto"] + args,
        pass_thru=True,  # Avoid eating npm output/error messages
        ensure_exit_code=False,  # Don't throw on non-zero exit code.
    )


def logspam_create_parser(subcommand):
    # Create the logspam command line parser.
    # if logspam is not installed, or not up to date, it will
    # first be installed.
    loader = PypiBasedTool("logspam", "mozilla-log-spam")
    return loader.create_parser(subcommand)


from functools import partial


@Command(
    "logspam",
    category="misc",
    description="Warning categorizer for treeherder test runs.",
)
def logspam(command_context):
    pass


@SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
def report(command_context, **options):
    command_context.activate_virtualenv()
    logspam = PypiBasedTool("logspam")
    logspam.run(command="report", **options)


@SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
def bisect(command_context, **options):
    command_context.activate_virtualenv()
    logspam = PypiBasedTool("logspam")
    logspam.run(command="bisect", **options)


@SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
def create(command_context, **options):
    command_context.activate_virtualenv()
    logspam = PypiBasedTool("logspam")
    logspam.run(command="file", **options)


# mots_loader will be used when running commands and subcommands, as well as
# when creating the parsers.
mots_loader = PypiBasedTool("mots")


def mots_create_parser(subcommand=None):
    return mots_loader.create_parser(subcommand)


def mots_run_subcommand(command, command_context, **options):
    command_context.activate_virtualenv()
    mots_loader.run(command=command, **options)


class motsSubCommand(SubCommand):
    """A helper subclass that reduces repitition when defining subcommands."""

    def __init__(self, subcommand):
        super().__init__(
            "mots",
            subcommand,
            parser=partial(mots_create_parser, subcommand),
        )


@Command(
    "mots",
    category="misc",
    description="Manage module information in-tree using the mots CLI.",
    parser=mots_create_parser,
)
def mots(command_context, **options):
    """The main mots command call."""
    command_context.activate_virtualenv()
    mots_loader.run(**options)


# Define subcommands that will be proxied through mach.
for sc in (
    "clean",
    "check-hashes",
    "export",
    "export-and-clean",
    "module",
    "query",
    "settings",
    "user",
    "validate",
):
    # Pass through args and kwargs, but add the subcommand string as the first argument.
    motsSubCommand(sc)(lambda *a, **kw: mots_run_subcommand(sc, *a, **kw))