improve sharing across RPMs (#86)

motivation: dedup more across RPM spec and scripts

changes:
* organize shared content in the a "Shared" directory
* create several more ".inc" files that extract duplicate information across RPM spec, including the description and header
* add symlinks to shared metadata and script so that the spec can also run easily outide docker
* update docker compose scripts to use new directory structure
* update Dockerfile optimization to be more resiliant by rewrting an bare-bone optimnization spec

Author: tomerd

platforms/Linux/RPM/Amazonlinux/2/Dockerfile

@@ -17,10 +17,14 @@ RUN yum install -y rpmdevtools yum-utils createrepo
 # Configure epel
 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 
-# Optimization: Install all the dependencies needed to build Swift from the spec file itself
+# Optimization: Install Swift build requirements listed in the spec file
 ADD swiftlang.spec /tmp/swiftlang.spec
-RUN touch /tmp/metadata.inc # fake metadata is okay for this optimization
-RUN cd /tmp && yum-builddep -y swiftlang.spec
+# rewrite a minimal spec with the build requirements
+RUN echo -e "Name: optimization\nVersion: optimization\nRelease: optimization\nSummary: optimization\nLicense: optimization\n" > /tmp/optimization.spec
+RUN cat /tmp/swiftlang.spec | grep BuildRequires >> /tmp/optimization.spec
+RUN echo -e "\n%description" >> /tmp/optimization.spec
+# install the build requirements
+RUN cd /tmp && yum-builddep -y optimization.spec
 
 # fix python six
 RUN mkdir -p /usr/local/lib/python3.7/site-packages/

platforms/Linux/RPM/Amazonlinux/2/build_rpm.sh

@@ -0,0 +1 @@
+../../Shared/scripts/build_rpm.sh

platforms/Linux/RPM/Amazonlinux/2/build_rpm.sh

@@ -0,0 +1 @@
+../../Shared/scripts/createrepo_rpm.sh

platforms/Linux/RPM/Amazonlinux/2/createrepo_rpm.sh

@@ -0,0 +1 @@
+../../Shared/metadata/description.inc

platforms/Linux/RPM/Amazonlinux/2/description.inc

@@ -26,10 +26,10 @@ services:
     image: amazonlinux-2-rpm-builder
     depends_on: [docker-setup]
     volumes:
-      - .:/code:z
-      - ../../../RPM:/shared:ro
+      - ../../Amazonlinux/2:/code/Amazonlinux/2:z
+      - ../../Shared:/code/Shared:z
       - ./.output:/output:z
-    working_dir: /code
+    working_dir: /code/Amazonlinux/2
     cap_drop:
       - CAP_NET_RAW
       - CAP_NET_BIND_SERVICE
@@ -40,7 +40,7 @@ services:
 
   createrepo:
     <<: *common
-    command: /bin/bash -cl "../shared/createrepo_rpm.sh"
+    command: /bin/bash -cl "./createrepo_rpm.sh"
 
   shell:
     <<: *common

platforms/Linux/RPM/Amazonlinux/2/docker-compose.yaml

@@ -0,0 +1 @@
+../../Shared/metadata/global.inc

platforms/Linux/RPM/Amazonlinux/2/global.inc

@@ -0,0 +1 @@
+../../Shared/metadata/metadata.inc

platforms/Linux/RPM/Amazonlinux/2/metadata.inc

@@ -1,14 +1,6 @@
+%include global.inc
 %include metadata.inc
 
-%global debug_package %{nil}
-
-Name:           %{package_name}
-Version:        %{package_version}
-Release:        1%{?dist}
-Summary:        %{package_summary}
-License:        %{package_license}
-URL:            %{package_url}
-
 Source0:        https://github.com/apple/swift/archive/swift-%{swift_version}.tar.gz#/swift.tar.gz
 Source1:        https://github.com/apple/swift-corelibs-libdispatch/archive/swift-%{swift_version}.tar.gz#/corelibs-libdispatch.tar.gz
 Source2:        https://github.com/apple/swift-corelibs-foundation/archive/swift-%{swift_version}.tar.gz#/corelibs-foundation.tar.gz
@@ -44,7 +36,6 @@ Source31:       https://github.com/apple/swift-format/archive/swift-%{swift_vers
 Source32:       https://github.com/apple/swift-lmdb/archive/swift-%{swift_version}.tar.gz#/swift-lmdb.tar.gz
 Source33:       https://github.com/apple/swift-markdown/archive/swift-%{swift_version}.tar.gz#/swift-markdown.tar.gz
 
-
 Patch0:         patches/hwasan_symbolize.patch
 
 BuildRequires:  clang
@@ -94,15 +85,7 @@ Requires:       tzdata
 ExclusiveArch:  x86_64 aarch64
 
 %description
-Swift is a general-purpose programming language built using
-a modern approach to safety, performance, and software design
-patterns.
-
-The goal of the Swift project is to create the best available
-language for uses ranging from systems programming, to mobile
-and desktop apps, scaling up to cloud services. Most
-importantly, Swift is designed to make writing and maintaining
-correct programs easier for the developer.
+%include description.inc
 
 %prep
 %setup -q -c -n %{swift_source_location} -a 0 -a 1 -a 2 -a 3 -a 4 -a 5 -a 6 -a 7 -a 8 -a 9 -a 10 -a 11 -a 12 -a 13 -a 14 -a 15 -a 16 -a 17 -a 18 -a 19 -a 20 -a 21 -a 22 -a 23 -a 24 -a 25 -a 26 -a 27 -a 28 -a 29 -a 30 -a 31 -a 32 -a 33

platforms/Linux/RPM/Amazonlinux/2/swiftlang.spec

@@ -17,10 +17,14 @@ RUN yum install -y rpmdevtools yum-utils createrepo
 # Configure epel
 RUN yum install -y epel-release centos-release-scl
 
-# Optimization: Install all the dependencies needed to build Swift from the spec file itself
+# Optimization: Install Swift build requirements listed in the spec file
 ADD swiftlang.spec /tmp/swiftlang.spec
-RUN touch /tmp/metadata.inc # fake metadata is okay for this optimization
-RUN cd /tmp && yum-builddep -y swiftlang.spec
+# rewrite a minimal spec with the build requirements
+RUN echo -e "Name: optimization\nVersion: optimization\nRelease: optimization\nSummary: optimization\nLicense: optimization\n" > /tmp/optimization.spec
+RUN cat /tmp/swiftlang.spec | grep BuildRequires >> /tmp/optimization.spec
+RUN echo -e "\n%description" >> /tmp/optimization.spec
+# install the build requirements
+RUN cd /tmp && yum-builddep -y optimization.spec
 
 # Workaround to support clang-3.5 or a later version
 RUN echo -e ". /opt/rh/sclo-git25/enable\n. /opt/rh/llvm-toolset-7/enable\n. /opt/rh/devtoolset-8/enable\n" >> $HOME/.bashrc

platforms/Linux/RPM/Centos/7/Dockerfile

@@ -0,0 +1 @@
+../../Shared/scripts/build_rpm.sh

platforms/Linux/RPM/Centos/7/build_rpm.sh

@@ -0,0 +1 @@
+../../Shared/scripts/createrepo_rpm.sh

platforms/Linux/RPM/Centos/7/build_rpm.sh

@@ -0,0 +1 @@
+../../Shared/metadata/description.inc

platforms/Linux/RPM/Centos/7/createrepo_rpm.sh

@@ -29,10 +29,10 @@ services:
     security_opt:
       - seccomp:unconfined
     volumes:
-      - .:/code:z
-      - ../../../RPM:/shared:ro
+      - ../../Centos/7:/code/Centos/7:z
+      - ../../Shared:/code/Shared:z
       - ./.output:/output:z
-    working_dir: /code
+    working_dir: /code/Centos/7
     cap_drop:
       - CAP_NET_RAW
       - CAP_NET_BIND_SERVICE
@@ -43,7 +43,7 @@ services:
 
   createrepo:
     <<: *common
-    command: /bin/bash -cl "../shared/createrepo_rpm.sh"
+    command: /bin/bash -cl "./createrepo_rpm.sh"
 
   shell:
     <<: *common

platforms/Linux/RPM/Centos/7/description.inc

@@ -0,0 +1 @@
+../../Shared/metadata/global.inc

platforms/Linux/RPM/Centos/7/docker-compose.yaml

@@ -0,0 +1 @@
+../../Shared/metadata/metadata.inc

platforms/Linux/RPM/Centos/7/global.inc

@@ -1,14 +1,6 @@
+%include global.inc
 %include metadata.inc
 
-%global debug_package %{nil}
-
-Name:           %{package_name}
-Version:        %{package_version}
-Release:        1%{?dist}
-Summary:        %{package_summary}
-License:        %{package_license}
-URL:            %{package_url}
-
 Source0:        https://github.com/apple/swift/archive/swift-%{swift_version}.tar.gz#/swift.tar.gz
 Source1:        https://github.com/apple/swift-corelibs-libdispatch/archive/swift-%{swift_version}.tar.gz#/corelibs-libdispatch.tar.gz
 Source2:        https://github.com/apple/swift-corelibs-foundation/archive/swift-%{swift_version}.tar.gz#/corelibs-foundation.tar.gz
@@ -95,15 +87,7 @@ Requires:       zlib-devel
 ExclusiveArch:  x86_64 aarch64
 
 %description
-Swift is a general-purpose programming language built using
-a modern approach to safety, performance, and software design
-patterns.
-
-The goal of the Swift project is to create the best available
-language for uses ranging from systems programming, to mobile
-and desktop apps, scaling up to cloud services. Most
-importantly, Swift is designed to make writing and maintaining
-correct programs easier for the developer.
+%include description.inc
 
 %prep
 %setup -q -c -n %{swift_source_location} -a 0 -a 1 -a 2 -a 3 -a 4 -a 5 -a 6 -a 7 -a 8 -a 9 -a 10 -a 11 -a 12 -a 13 -a 14 -a 15 -a 16 -a 17 -a 18 -a 19 -a 20 -a 21 -a 22 -a 23 -a 24 -a 25 -a 26 -a 27 -a 28 -a 29 -a 30 -a 31 -a 32 -a 33

platforms/Linux/RPM/Centos/7/metadata.inc

@@ -18,7 +18,11 @@ RUN yum install -y rpmdevtools yum-utils createrepo
 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
 RUN yum config-manager --set-enabled powertools
 
-# Optimization: Install all the dependencies needed to build Swift from the spec file itself
+# Optimization: Install Swift build requirements listed in the spec file
 ADD swiftlang.spec /tmp/swiftlang.spec
-RUN touch /tmp/metadata.inc # fake metadata is okay for this optimization
-RUN cd /tmp && yum-builddep -y swiftlang.spec
+# rewrite a minimal spec with the build requirements
+RUN echo -e "Name: optimization\nVersion: optimization\nRelease: optimization\nSummary: optimization\nLicense: optimization\n" > /tmp/optimization.spec
+RUN cat /tmp/swiftlang.spec | grep BuildRequires >> /tmp/optimization.spec
+RUN echo -e "\n%description" >> /tmp/optimization.spec
+# install the build requirements
+RUN cd /tmp && yum-builddep -y optimization.spec

platforms/Linux/RPM/Centos/7/swiftlang.spec

@@ -0,0 +1 @@
+../../Shared/scripts/build_rpm.sh

platforms/Linux/RPM/Centos/8/Dockerfile

@@ -0,0 +1 @@
+../../Shared/scripts/createrepo_rpm.sh

platforms/Linux/RPM/Centos/8/build_rpm.sh

@@ -0,0 +1 @@
+../../Shared/metadata/description.inc

platforms/Linux/RPM/Centos/8/build_rpm.sh

@@ -29,10 +29,10 @@ services:
     security_opt:
       - seccomp:unconfined
     volumes:
-      - .:/code:z
-      - ../../../RPM:/shared:ro
+      - ../../Centos/8:/code/Centos/8:z
+      - ../../Shared:/code/Shared:z
       - ./.output:/output:z
-    working_dir: /code
+    working_dir: /code/Centos/8
     cap_drop:
       - CAP_NET_RAW
       - CAP_NET_BIND_SERVICE
@@ -43,7 +43,7 @@ services:
 
   createrepo:
     <<: *common
-    command: /bin/bash -cl "../shared/createrepo_rpm.sh"
+    command: /bin/bash -cl "./createrepo_rpm.sh"
 
   shell:
     <<: *common

platforms/Linux/RPM/Centos/8/createrepo_rpm.sh

@@ -0,0 +1 @@
+../../Shared/metadata/global.inc

platforms/Linux/RPM/Centos/8/description.inc

@@ -0,0 +1 @@
+../../Shared/metadata/metadata.inc