Reorganize cross builders

* Have one giant array for everything.
* Make each entry a dictionary describing where it's run
* Add a new bot for running 32-bit linux cross targets

Author: alexcrichton

master/master.cfg

@@ -114,7 +114,7 @@ auto_platforms = [
     "mac-64-nopt-t",
     #"mac-64-opt-vg",
     #"mac-all-opt",
-    "mac-ios-opt",
+    "mac-cross-ios-opt",
     "mac-64-opt-rustbuild",
 
     "linux-32-opt",
@@ -126,17 +126,18 @@ auto_platforms = [
     "linux-64-debug-opt",
     "linux-musl-64-opt",
     "linux-cross-opt",
+    "linux-32cross-opt",
     "linux-64-opt-rustbuild",
     "linux-64-opt-mir",
 
     #"linux-64-opt-vg",
     #"linux-all-opt",
 
     "linux-64-x-android-t",
-    "linux-64-x-netbsd",
-    "linux-64-x-freebsd",
-    "linux-64-x-armsf",
-    "linux-64-x-armhf",
+    "linux-64-cross-netbsd",
+    "linux-64-cross-freebsd",
+    "linux-64-cross-armsf",
+    "linux-64-cross-armhf",
 
     "win-gnu-32-opt",
     #"win-gnu-32-nopt-c",
@@ -170,6 +171,7 @@ snap_platforms = ["linux", "win-gnu-32", "win-gnu-64", "mac", "bitrig-64",
                   "openbsd-64-opt"]
 dist_platforms = ["linux", "mac", "arm-android", "musl-linux",
                   "cross-linux",
+                  "cross32-linux",
                   "cross-host-linux",
                   "mac-ios",
                   "win-gnu-32", "win-gnu-64",
@@ -212,17 +214,6 @@ dist_nogate_platforms = [
     "cross-win",
 ]
 
-nightly_cross_host_targets = []
-beta_cross_host_targets = [
-  {'target': 'arm-unknown-linux-gnueabi', 'builder': 'armsf'},
-  {'target': 'arm-unknown-linux-gnueabihf', 'builder': 'armhf'},
-  {'target': 'aarch64-unknown-linux-gnu', 'builder': None},
-  {'target': 'armv7-unknown-linux-gnueabihf', 'builder': None},
-  {'target': 'x86_64-unknown-freebsd', 'builder': 'freebsd'},
-  {'target': 'x86_64-unknown-netbsd', 'builder': 'netbsd'},
-]
-stable_cross_host_targets = []
-
 cargo_cross_targets = [
   'arm-unknown-linux-gnueabi',
   'arm-unknown-linux-gnueabihf',
@@ -232,47 +223,48 @@ cargo_cross_targets = [
   'x86_64-unknown-netbsd',
 ]
 
-nightly_lincross_targets = [
-  #'i586-unknown-linux-gnu',
-]
-beta_lincross_targets = [
-  'mips-unknown-linux-musl',
-  'mipsel-unknown-linux-musl',
-]
-stable_lincross_targets = [
-  'armv7-unknown-linux-gnueabihf',
-  'powerpc-unknown-linux-gnu',
-  'powerpc64-unknown-linux-gnu',
-  'powerpc64le-unknown-linux-gnu',
-  'x86_64-rumprun-netbsd',
-  'arm-unknown-linux-gnueabi',
-  'arm-unknown-linux-gnueabihf',
-  'mips-unknown-linux-gnu',
-  'mipsel-unknown-linux-gnu',
-  'aarch64-unknown-linux-gnu',
-]
-nightly_wincross_targets = [
+ios = {'auto': 'cross-ios-opt', 'dist': 'cross-ios-opt'}
+lincross = {'auto': 'linux-cross', 'dist': 'cross-linux'}
+lincross32 = {'auto': 'linux-32cross', 'dist': 'cross32-linux'}
+msvc32 = {'auto': 'msvc-32-cross', 'dist': 'win-msvc-32-cross'}
+
+def xhost(name):
+    return {'auto': 'linux-64-cross-' + name, 'dist': 'cross-host-linux'}
+
+stable_cross_targets = [
+  {'t': 'aarch64-apple-ios', 'b': ios},
+  {'t': 'aarch64-unknown-linux-gnu', 'b': lincross},
+  {'t': 'arm-unknown-linux-gnueabi', 'b': lincross},
+  {'t': 'arm-unknown-linux-gnueabihf', 'b': lincross},
+  {'t': 'armv7-apple-ios', 'b': ios},
+  {'t': 'armv7-unknown-linux-gnueabihf', 'b': lincross},
+  {'t': 'armv7s-apple-ios', 'b': ios},
+  {'t': 'i386-apple-ios', 'b': ios},
+  {'t': 'mips-unknown-linux-gnu', 'b': lincross},
+  {'t': 'mipsel-unknown-linux-gnu', 'b': lincross},
+  {'t': 'powerpc-unknown-linux-gnu', 'b': lincross},
+  {'t': 'powerpc64-unknown-linux-gnu', 'b': lincross},
+  {'t': 'powerpc64le-unknown-linux-gnu', 'b': lincross},
+  {'t': 'x86_64-apple-ios', 'b': ios},
+  {'t': 'x86_64-rumprun-netbsd', 'b': lincross},
 ]
-beta_wincross_targets = [
-  'i586-pc-windows-msvc',
+beta_cross_targets = stable_cross_targets + [
+  {'t': 'aarch64-unknown-linux-gnu', 'b': xhost('NOAUTO'), 'host': True},
+  {'t': 'arm-unknown-linux-gnueabi', 'b': xhost('armsf'), 'host': True},
+  {'t': 'arm-unknown-linux-gnueabihf', 'b': xhost('armhf'), 'host': True},
+  {'t': 'armv7-unknown-linux-gnueabihf', 'b': xhost('NOAUTO'), 'host': True},
+  {'t': 'i586-pc-windows-msvc', 'b': msvc32},
+  {'t': 'i686-unknown-freebsd', 'b': xhost('NOAUTO'), 'host': True},
+  {'t': 'mips-unknown-linux-musl', 'b': lincross},
+  {'t': 'mipsel-unknown-linux-musl', 'b': lincross},
+  {'t': 'x86_64-unknown-freebsd', 'b': xhost('freebsd'), 'host': True},
+  {'t': 'x86_64-unknown-netbsd', 'b': xhost('netbsd'), 'host': True},
 ]
-stable_wincross_targets = []
-
-ios_targets = [
-  'aarch64-apple-ios',
-  'armv7-apple-ios',
-  'armv7s-apple-ios',
-  'i386-apple-ios',
-  'x86_64-apple-ios',
+nightly_cross_targets = beta_cross_targets + [
+  {'t': 'i586-unknown-linux-gnu', 'b': lincross32},
+  {'t': 'i686-unknown-linux-musl', 'b': lincross32},
 ]
 
-all_lincross_targets = nightly_lincross_targets + beta_lincross_targets + \
-  stable_lincross_targets
-all_wincross_targets = nightly_wincross_targets + beta_wincross_targets + \
-  stable_wincross_targets
-all_cross_host_targets = nightly_cross_host_targets + beta_cross_host_targets + \
-  stable_cross_host_targets
-
 ####### BUILDSLAVES
 
 # Configuration of --host and --target triples based on the above platform names
@@ -1270,7 +1262,7 @@ def distsnap_buildfactory(platform, channel_label):
         command.append("check-stage2-T-x86_64-unknown-linux-musl-" + \
                        "H-x86_64-unknown-linux-gnu")
         command.append('dist')
-    elif 'cross' in platform or 'ios' in platform:
+    elif 'cross' in platform:
         command.append('dist')
     else:
         command.append('distcheck')
@@ -1520,17 +1512,14 @@ def packaging_dist_buildfactory(platform, channel_label):
     hosts = all_platform_hosts(platform)
     if platform == 'linux':
         if channel_label == 'stable':
-            hosts += [h['target'] for h in stable_cross_host_targets]
+            extra_hosts = stable_cross_targets
         elif channel_label == 'beta':
-            hosts += [h['target'] for h in stable_cross_host_targets]
-            hosts += [h['target'] for h in beta_cross_host_targets]
+            extra_hosts = beta_cross_targets
         else:
-            hosts += [h['target'] for h in stable_cross_host_targets]
-            hosts += [h['target'] for h in beta_cross_host_targets]
-            hosts += [h['target'] for h in nightly_cross_host_targets]
+            extra_hosts = nightly_cross_targets
+        hosts += [t['t'] for t in extra_hosts if 'host' in t]
 
     for target in hosts:
-
         f.addStep(Compile(env=CommandEnv(),
                           name="fetch inputs",
                           description="fetch input",
@@ -1607,12 +1596,12 @@ def platform_slaves(p):
     if "-x-android" in p:
         return [p]
 
-    if "musl" in p:
+    if "musl" in p or "32cross" in p or "cross32" in p:
         p = "linux"
-    elif "linux-cross" in p or "linux-64-x-" in p:
-        p = "lincross"
     elif "ios" in p:
         return [slave.slavename for slave in ios_slaves]
+    elif "linux" in p and "cross" in p:
+        p = "lincross"
     else:
         p = p.split("-")[0]
     return [slave.slavename
@@ -1628,7 +1617,7 @@ def platform_snap_slaves(p):
 # FIXME: The linux AMI instances are using valgrind 3.7 and we need 3.8+
 # This rule limits which bots we run the valgrinding dist snapshot on.
 def platform_dist_slaves(p):
-    if 'musl' in p or 'cross-linux' in p or 'ios' in p:
+    if 'musl' in p or 'cross' in p or 'ios' in p:
         return platform_slaves(p)
 
     # p is exactly the platform name, ie arm-android
@@ -1696,15 +1685,9 @@ for p in auto_platforms:
         chk = "check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu"
         musl = "/musl-x86_64"
         targets.append('x86_64-unknown-linux-musl')
-    if "linux-cross" in p:
-        chk = False
-        targets += all_lincross_targets
-    if "win-msvc-32-cross" in p:
+    if "cross" in p:
         chk = False
-        targets += all_wincross_targets
-    if "ios" in p:
-        chk = False
-        targets += ios_targets
+        musl = "/musl-i686"
     if not opt_compiler:
         chk = False
     if "rustbuild" in p:
@@ -1715,6 +1698,14 @@ for p in auto_platforms:
         chk = "check-cargotest"
         rustbuild = True
 
+    for t in nightly_cross_targets:
+        if t['b']['auto'] in p:
+            if 'host' in t:
+                hosts.append(t['t'])
+                rustbuild = True
+            else:
+                targets.append(t['t'])
+
     android = False
     if "-x-android" in p:
         targets.append('arm-linux-androideabi')
@@ -1725,13 +1716,6 @@ for p in auto_platforms:
             # Only test android, not the host
             chk = "check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu"
 
-    elif "linux-64-x-" in p:
-        chk = False
-        for target in all_cross_host_targets:
-            if target['builder'] and target['builder'] in p:
-                hosts.append(target['target'])
-        rustbuild = True
-
     c['builders'].append(BuilderConfig(
             mergeRequests=True,
             name="auto-" + p,
@@ -1780,69 +1764,51 @@ for p in dist_platforms:
     if "android" in p:
         android = True
         # Not checking android for now
-        hosts = ""
+        hosts = []
         targets.append('arm-linux-androideabi')
     elif "musl" in p:
         musl = "/musl-x86_64"
-        hosts = ""
+        hosts = []
         targets.append('x86_64-unknown-linux-musl')
-    elif "cross-linux" in p:
-        hosts = ""
-    if "ios" in p:
-        hosts = ""
-        targets += ios_targets
+    elif "cross32-linux" in p:
+        musl = "/musl-i686"
+    if "ios" in p or "cross" in p:
+        hosts = []
 
     for channel in ['nightly', 'beta', 'stable']:
         my_targets = targets[:]
         my_hosts = hosts
         rustbuild = None
 
-        my_lincross_targets = []
-        my_wincross_targets = []
-        my_cross_host_targets = []
         if channel == 'stable':
-            my_lincross_targets += stable_lincross_targets
-            my_wincross_targets += stable_wincross_targets
-            my_cross_host_targets += [h['target'] for h in stable_cross_host_targets]
+            my_cross_targets = stable_cross_targets
         elif channel == 'beta':
-            my_lincross_targets += stable_lincross_targets
-            my_lincross_targets += beta_lincross_targets
-            my_wincross_targets += stable_wincross_targets
-            my_wincross_targets += beta_wincross_targets
-            my_cross_host_targets += [h['target'] for h in stable_cross_host_targets]
-            my_cross_host_targets += [h['target'] for h in beta_cross_host_targets]
+            my_cross_targets = beta_cross_targets
         elif channel == 'nightly':
-            my_lincross_targets += stable_lincross_targets
-            my_lincross_targets += beta_lincross_targets
-            my_lincross_targets += nightly_lincross_targets
-            my_wincross_targets += stable_wincross_targets
-            my_wincross_targets += beta_wincross_targets
-            my_wincross_targets += nightly_wincross_targets
-            my_cross_host_targets += [h['target'] for h in stable_cross_host_targets]
-            my_cross_host_targets += [h['target'] for h in beta_cross_host_targets]
-            my_cross_host_targets += [h['target'] for h in nightly_cross_host_targets]
-
-        # The `cross-linux` builder below is intended for just producing
-        # standard libraries, whereas the `cross-host-linux` is producing entire
+            my_cross_targets = nightly_cross_targets
+
+        for t in my_cross_targets:
+            if not t['b']['dist'] in p:
+                continue
+            if 'host' in p:
+                rustbuild = True
+                my_hosts.append(t['t'])
+            else:
+                my_targets.append(t['t'])
+
+        # The `cross-linux` builder is intended for just producing standard
+        # libraries, whereas the `cross-host-linux` is producing entire
         # compilers. Ideally these would use the same bot but unfortunately they
         # use different build systems, so they need to be separate for now.
         #
-        # In any case, if we're producing an entire compiler for a target, or
-        # rather something in `my_cross_host_targets`, then we don't want to
-        # *also* produce a standard library for that target on another bot as
-        # it'll cause upload failures later. Consequently we just filter our the
-        # list of `my_lincross_targets` to exclude everything we're building a
-        # compiler for.
-        my_lincross_targets = [t for t in my_lincross_targets \
-                               if t not in my_cross_host_targets]
-
-        if "cross-linux" in p:
-            my_targets += my_lincross_targets
-        if "win-msvc-32-cross" in p:
-            my_targets += my_wincross_targets
-        if "cross-host-linux" in p:
-            rustbuild = True
-            my_hosts = my_cross_host_targets
+        # In any case, if we're producing an entire compiler for a target then
+        # we don't want to *also* produce a standard library for that target on
+        # another bot as it'll cause upload failures later. Consequently we just
+        # filter our the list of `my_targets` to exclude everything we're
+        # building a compiler for.
+        for t in my_cross_targets:
+            if 'host' in t and t['t'] in my_targets:
+                my_targets.remove(t['t'])
 
         branch = 'master' if channel == 'nightly' else channel
         c['builders'].append(BuilderConfig(