[Mac] Add support for butler.py to generate Mac ARM64 deployment zip

Pipfile

@@ -4,6 +4,8 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
+# If you add or bump packages here with native/compiled C/C++/Rust extensions,
+# make sure to also update src/platform_requirements.txt.
 cryptography = "==37.0.4"
 crcmod = "==1.7"
 future = "==0.17.1"

butler.py

@@ -33,6 +33,8 @@
 
 guard.check()
 
+from local.butler import constants
+
 
 class _ArgumentParser(argparse.ArgumentParser):
   """Custom ArgumentParser."""
@@ -125,7 +127,9 @@ def _add_package_subparser(toplevel_subparsers):
   parser_package = toplevel_subparsers.add_parser(
       'package', help='Package clusterfuzz with a staging revision')
   parser_package.add_argument(
-      '-p', '--platform', choices=['linux', 'macos', 'windows', 'all'])
+      '-p',
+      '--platform',
+      choices=list(constants.DEPLOYMENT_TARGETS.keys()) + ['all'])
   parser_package.add_argument(
       '-r',
       '--release',

src/clusterfuzz/_internal/base/utils.py

@@ -315,12 +315,10 @@ def _get_deployment_zip_release_suffix(release):
   return suffix
 
 
-def get_platform_deployment_filename(platform, release):
-  """Return the platform deployment filename."""
-  # Expects linux, macos or windows.
-  base_filename = platform
+def get_deployment_target_filename(deployment_target, release):
+  """Return the deployment filename for the given target."""
   release_filename_suffix = _get_deployment_zip_release_suffix(release)
-  return f'{base_filename}{release_filename_suffix}.zip'
+  return f'{deployment_target}{release_filename_suffix}.zip'
 
 
 def get_remote_manifest_filename(release):

src/clusterfuzz/_internal/bot/tasks/commands.py

@@ -112,8 +112,13 @@ def is_supported_cpu_arch_for_job():
     # No specific cpu architecture requirement specified in job, bail out.
     return True
 
-  # Convert to list just in case anyone specifies value as a single string.
-  supported_cpu_arch_list = list(supported_cpu_arch)
+  # We support specifying CPU_ARCH as a list e.g. ['armeabi', 'armeabi-v7a'] or
+  # as a string e.g. 'x86_64, arm64'.
+  if isinstance(supported_cpu_arch, list):
+    supported_cpu_arch_list = supported_cpu_arch
+  else:
+    supported_cpu_arch_list = utils.parse_delimited(
+        str(supported_cpu_arch), delimiter=',', strip=True, remove_empty=True)
 
   return cpu_arch in supported_cpu_arch_list
 

src/clusterfuzz/_internal/bot/tasks/update_task.py

@@ -66,14 +66,20 @@ def get_source_url():
   """Return the source URL."""
   release = utils.get_clusterfuzz_release()
   platform_name = platform.system()
-  platform_mappings = {
+  deployment_target_mappings = {
       'Linux': 'linux',
       'Windows': 'windows',
       'Darwin': 'macos'
   }
-  platform_name = platform_mappings[platform_name]
+  deployment_target = deployment_target_mappings[platform_name]
+
+  # macOS bots run on both x86_64 and arm64 (Apple Silicon), requiring distinct
+  # deployment bundles. Other desktop platforms currently share a single bundle.
+  if deployment_target == 'macos' and platform.machine().lower() == 'arm64':
+    deployment_target = 'macos_arm64'
+
   return _deployment_file_url(
-      utils.get_platform_deployment_filename(platform_name, release))
+      utils.get_deployment_target_filename(deployment_target, release))
 
 
 def get_source_manifest_url():

src/clusterfuzz/_internal/system/environment.py

@@ -17,6 +17,7 @@
 import enum
 import functools
 import os
+import platform as platform_util
 import re
 import socket
 import subprocess
@@ -227,8 +228,14 @@ def get_cpu_arch():
     from clusterfuzz._internal.platforms import android
     return android.settings.get_cpu_arch()
 
-  # FIXME: Add support for desktop architectures as needed.
-  return None
+  machine = platform_util.machine().lower()
+  if machine in ('arm64', 'aarch64'):
+    return 'arm64'
+  if machine in ('x86_64', 'amd64'):
+    return 'x86_64'
+  if machine in ('i386', 'i686', 'x86'):
+    return 'x86'
+  return machine or None
 
 
 def get_current_memory_tool_var():
@@ -271,7 +278,7 @@ def get_instrumented_libraries_paths():
 
 
 def get_default_tool_path(tool_name):
-  """Get the default tool for this platform (from scripts/ dir)."""
+  """Get the default tool for this platform (from resources/ directory)."""
   if is_android():
     # For android devices, we do symbolization on the host machine, which is
     # linux. So, we use the linux version of llvm-symbolizer.

src/clusterfuzz/_internal/tests/core/bot/tasks/commands_test.py

@@ -295,3 +295,49 @@ def test_timeout_overrides(self):
         'FUZZ_TEST_TIMEOUT = 123\nMAX_TESTCASES = 5\n')
     self.assertEqual(9001, environment.get_value('FUZZ_TEST_TIMEOUT'))
     self.assertEqual(42, environment.get_value('MAX_TESTCASES'))
+
+
+class IsSupportedCpuArchForJobTest(unittest.TestCase):
+  """Tests for is_supported_cpu_arch_for_job."""
+
+  def setUp(self):
+    helpers.patch_environ(self)
+    helpers.patch(self, [
+        'clusterfuzz._internal.system.environment.get_cpu_arch',
+    ])
+
+  def test_no_bot_cpu_arch(self):
+    """Test when no cpu arch is defined on bot."""
+    self.mock.get_cpu_arch.return_value = None
+    environment.set_value('CPU_ARCH', 'arm64')
+    self.assertTrue(commands.is_supported_cpu_arch_for_job())
+
+  def test_no_job_cpu_arch_requirement(self):
+    """Test when job specifies no CPU_ARCH requirement."""
+    self.mock.get_cpu_arch.return_value = 'arm64'
+    environment.set_value('CPU_ARCH', None)
+    self.assertTrue(commands.is_supported_cpu_arch_for_job())
+
+  def test_matching_single_string(self):
+    """Test when job specifies matching single string CPU_ARCH."""
+    self.mock.get_cpu_arch.return_value = 'arm64'
+    environment.set_value('CPU_ARCH', 'arm64')
+    self.assertTrue(commands.is_supported_cpu_arch_for_job())
+
+  def test_non_matching_single_string(self):
+    """Test when job specifies non-matching single string CPU_ARCH."""
+    self.mock.get_cpu_arch.return_value = 'arm64'
+    environment.set_value('CPU_ARCH', 'x86_64')
+    self.assertFalse(commands.is_supported_cpu_arch_for_job())
+
+  def test_matching_comma_separated_string(self):
+    """Test when job specifies comma-separated CPU_ARCH."""
+    self.mock.get_cpu_arch.return_value = 'arm64'
+    environment.set_value('CPU_ARCH', 'x86_64, arm64')
+    self.assertTrue(commands.is_supported_cpu_arch_for_job())
+
+  def test_matching_list(self):
+    """Test when job specifies a list of supported architectures."""
+    self.mock.get_cpu_arch.return_value = 'arm64'
+    environment.set_value('CPU_ARCH', ['x86_64', 'arm64'])
+    self.assertTrue(commands.is_supported_cpu_arch_for_job())

src/clusterfuzz/_internal/tests/core/bot/tasks/update_task_test.py

@@ -268,3 +268,39 @@ def test_archive_execute_permission_is_respected(self):
         filepath = os.path.join(self.temp_directory, member.filename)
         if mode & 0o100:
           self.assertTrue(os.access(filepath, os.X_OK))
+
+
+class GetSourceUrlTest(unittest.TestCase):
+  """Tests for get_source_url."""
+
+  def setUp(self):
+    helpers.patch_environ(self)
+    helpers.patch(self, [
+        'clusterfuzz._internal.config.local_config.ProjectConfig',
+        'clusterfuzz._internal.base.utils.get_clusterfuzz_release',
+        'platform.machine',
+        'platform.system',
+    ])
+    self.mock.ProjectConfig().get.return_value = 'deployment-bucket'
+    self.mock.get_clusterfuzz_release.return_value = 'prod'
+
+  def test_linux(self):
+    """Test get_source_url for linux."""
+    self.mock.system.return_value = 'Linux'
+    self.mock.machine.return_value = 'x86_64'
+    self.assertEqual('gs://deployment-bucket/linux-3.zip',
+                     update_task.get_source_url())
+
+  def test_mac_x86_64(self):
+    """Test get_source_url for mac x86_64."""
+    self.mock.system.return_value = 'Darwin'
+    self.mock.machine.return_value = 'x86_64'
+    self.assertEqual('gs://deployment-bucket/macos-3.zip',
+                     update_task.get_source_url())
+
+  def test_mac_arm64(self):
+    """Test get_source_url for mac arm64."""
+    self.mock.system.return_value = 'Darwin'
+    self.mock.machine.return_value = 'arm64'
+    self.assertEqual('gs://deployment-bucket/macos_arm64-3.zip',
+                     update_task.get_source_url())

src/clusterfuzz/_internal/tests/core/system/environment_test.py

@@ -469,3 +469,72 @@ def test_function():
       return 10
 
     self.assertEqual(None, test_function())
+
+
+class GetCpuArchTest(unittest.TestCase):
+  """Tests for get_cpu_arch."""
+
+  def setUp(self):
+    test_helpers.patch_environ(self)
+    test_helpers.patch(self, [
+        'clusterfuzz._internal.system.environment.is_android',
+        'platform.machine',
+        'clusterfuzz._internal.platforms.android.settings.get_cpu_arch',
+    ])
+    self.mock.is_android.return_value = False
+
+  def test_android(self):
+    """Test Android architecture delegation."""
+    self.mock.is_android.return_value = True
+    self.mock.get_cpu_arch.return_value = 'arm64_v8a'
+    self.assertEqual('arm64_v8a', environment.get_cpu_arch())
+
+  def test_arm64(self):
+    """Test ARM64 architecture detection."""
+    self.mock.machine.return_value = 'arm64'
+    self.assertEqual('arm64', environment.get_cpu_arch())
+
+  def test_aarch64(self):
+    """Test aarch64 normalized to arm64."""
+    self.mock.machine.return_value = 'aarch64'
+    self.assertEqual('arm64', environment.get_cpu_arch())
+
+  def test_x86_64(self):
+    """Test x86_64 architecture detection."""
+    self.mock.machine.return_value = 'x86_64'
+    self.assertEqual('x86_64', environment.get_cpu_arch())
+
+  def test_amd64(self):
+    """Test amd64 normalized to x86_64."""
+    self.mock.machine.return_value = 'AMD64'
+    self.assertEqual('x86_64', environment.get_cpu_arch())
+
+
+class GetDefaultToolPathTest(unittest.TestCase):
+  """Tests for get_default_tool_path."""
+
+  def setUp(self):
+    test_helpers.patch_environ(self)
+    test_helpers.patch(self, [
+        'clusterfuzz._internal.system.environment.is_android',
+        'clusterfuzz._internal.system.environment.platform',
+        'clusterfuzz._internal.system.environment.get_platform_resources_directory',
+    ])
+    self.mock.is_android.return_value = False
+    self.mock.platform.return_value = 'MAC'
+    self.mock.get_platform_resources_directory.return_value = (
+        '/resources/platform/mac')
+
+  def test_desktop(self):
+    """Test getting default tool path on desktop."""
+    self.assertEqual('/resources/platform/mac/llvm-symbolizer',
+                     environment.get_default_tool_path('llvm-symbolizer'))
+
+  def test_android(self):
+    """Test getting default tool path for Android uses host linux directory."""
+    self.mock.is_android.return_value = True
+    self.mock.get_platform_resources_directory.return_value = (
+        '/resources/platform/linux')
+    self.assertEqual('/resources/platform/linux/llvm-symbolizer',
+                     environment.get_default_tool_path('llvm-symbolizer'))
+    self.mock.get_platform_resources_directory.assert_called_once_with('linux')

src/local/butler/common.py

@@ -219,7 +219,10 @@ def _install_chromedriver():
   if plt == 'linux':
     archive_name = 'chromedriver_linux64.zip'
   elif plt == 'macos':
-    archive_name = 'chromedriver_mac64.zip'
+    if platform.machine() == 'arm64' or platform.processor() == 'arm':
+      archive_name = 'chromedriver_mac64_m1.zip'
+    else:
+      archive_name = 'chromedriver_mac64.zip'
   elif plt == 'windows':
     archive_name = 'chromedriver_win32.zip'
 
@@ -285,7 +288,7 @@ def _install_pip(requirements_path, target_path):
 
 def _install_platform_pip(requirements_path, target_path, platform_name):
   """Install platform specific pip packages."""
-  pip_platform = constants.PLATFORMS.get(platform_name)
+  pip_platform = constants.DEPLOYMENT_TARGETS.get(platform_name)
   if not pip_platform:
     raise OSError(f'Unknown platform: {platform_name}.')
 
@@ -299,25 +302,36 @@ def _install_platform_pip(requirements_path, target_path, platform_name):
 
   pip_abi = constants.ABIS[platform_name]
 
-  for pip_platform in pip_platforms:
-    temp_dir = tempfile.mkdtemp()
-    return_code, _ = execute(
-        f'{_pip()} download --no-deps --only-binary=:all: '
-        f'--platform={pip_platform} --abi={pip_abi} -r {requirements_path} -d '
-        f'{temp_dir}',
-        exit_on_error=False)
-
-    if return_code != 0:
-      print(f'Did not find package for platform: {pip_platform}')
-      continue
-
-    execute(f'unzip -o -d {target_path} "{temp_dir}/*.whl"')
-    shutil.rmtree(temp_dir, ignore_errors=True)
-    break
-
-  if return_code != 0:
-    raise RuntimeError(
-        f'Failed to find package in supported platforms: {pip_platforms}')
+  with open(requirements_path, 'r') as f:
+    requirements = [
+        line.strip() for line in f if line.strip() and not line.startswith('#')
+    ]
+
+  # Different packages on PyPI publish binary wheels targeting different macOS
+  # deployment targets (e.g. macosx_11_0 vs macosx_12_0 vs universal2). We
+  # iterate over requirements individually so each package can match its own
+  # highest compatible platform tag.
+  for req in requirements:
+    downloaded = False
+    for pip_platform_entry in pip_platforms:
+      with tempfile.TemporaryDirectory() as temp_dir:
+        return_code, _ = execute(
+            f'{_pip()} download --no-deps --only-binary=:all: '
+            f'--platform={pip_platform_entry} --abi={pip_abi} "{req}" -d '
+            f'{temp_dir}',
+            exit_on_error=False)
+
+        if return_code != 0:
+          print(f'Did not find {req} for platform: {pip_platform_entry}')
+          continue
+
+        execute(f'unzip -o -d {target_path} "{temp_dir}/*.whl"')
+        downloaded = True
+        break
+
+    if not downloaded:
+      raise RuntimeError(
+          f'Failed to find {req} in supported platforms: {pip_platforms}')
 
 
 def _remove_invalid_files():