Extract the rust builder

Author: mattgodbolt

Dockerfile.misc

@@ -69,8 +69,6 @@ RUN cpanm Modern::Perl
 
 RUN cpanm App::Prove CPU::Z80::Assembler Data::Dump File::Path List::Uniq Object::Tiny::RW Regexp::Common Text::Diff YAML::Tiny
 
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-
 RUN mkdir /cmake && \
     cd /cmake && \
     wget https://github.com/Kitware/CMake/releases/download/v3.24.0-rc5/cmake-3.24.0-rc5-linux-x86_64.sh && \

Dockerfile.rust-cg-gcc

@@ -0,0 +1,24 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN dpkg --add-architecture i386
+RUN apt update -y -q && apt upgrade -y -q && apt update -y -q && \
+    apt install -y -q \
+    build-essential \
+    bison \
+    curl \
+    flex \
+    git \
+    libc6-dev-i386 \
+    libc6-dev:i386 \
+    make \
+    patchelf \
+    xz-utils
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+RUN mkdir -p /root
+COPY rust-cg-gcc /root/
+COPY common.sh /root/
+
+WORKDIR /root

common.sh

@@ -2,7 +2,7 @@
 
 # Common utilities and setup for builds
 
-set -euo pipefail
+set -euxo pipefail
 
 # get_remote_revision GITURL BRANCH
 get_remote_revision() {
@@ -14,23 +14,19 @@ get_remote_revision() {
     echo "$REVISION"
 }
 
-# initialise REVISION OUTPUT_FILENAME
+# initialise REVISION OUTPUT_FILENAME [optional previously built revision]
 initialise() {
     local REVISION="$1"
     local OUTPUT="$2"
+    local LAST_REVISION="${3-previously-unbuilt}"
     echo "ce-build-revision:${REVISION}"
     echo "ce-build-output:${OUTPUT}"   
-}
-
-# skip_if_built REVISION PREV_REVISION
-skip_if_built() {
     if [[ "${REVISION}" == "${LAST_REVISION}" ]]; then
         echo "ce-build-status:SKIPPED"
         exit
     fi
 }
 
-
 # complete <source folder> <name to be extracted as> <dest tar.xz file>
 complete() {
     local SOURCE="$1"

compcert/build.sh

@@ -25,9 +25,7 @@ FULLNAME=${BASENAME}.tar.xz
 OUTPUT=$3/${FULLNAME}
 
 REVISION=$(get_remote_revision "${URL}" "${BRANCH}")
-initialise "${REVISION}" "${OUTPUT}"
-
-skip_if_built "${REVISION}" "${LAST_REVISION}"
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
 
 # The path to compcert will be hardcoded in a config file (share/compcert.ini).
 # This makes the compiler not relocatable without editing this file beforehand.

misc/build-rustc-cg-gcc.sh renamed to rust-cg-gcc/build.sh

@@ -4,9 +4,9 @@
 ## $2 : destination: a directory or S3 path (eg. s3://...)
 ## $3 : last revision (as mangled below) successfully build (optional)
 
-set -e
+set -eu
+source common.sh
 
-ROOT=$PWD
 VERSION="${1}"
 LAST_REVISION="${3-}"
 
@@ -21,34 +21,15 @@ GCC_BRANCH="master"
 CG_GCC_BRANCH="master"
 CG_GCC_URL="https://github.com/rust-lang/rustc_codegen_gcc.git"
 
-GCC_REVISION=$(git ls-remote --heads "${GCC_URL}" "refs/heads/${GCC_BRANCH}" | cut -f 1)
-CG_GCC_REVISION=$(git ls-remote --heads "${CG_GCC_URL}" "refs/heads/${CG_GCC_BRANCH}" | cut -f 1)
+GCC_REVISION=$(get_remote_revision "${GCC_URL}" "heads/${GCC_BRANCH}")
+CG_GCC_REVISION=$(get_remote_revision "${CG_GCC_URL}" "heads/${CG_GCC_BRANCH}")
 
 BASENAME=rustc-cg-gcc-${VERSION}-$(date +%Y%m%d)
 FULLNAME=${BASENAME}.tar.xz
-OUTPUT=${ROOT}/${FULLNAME}
-S3OUTPUT=
-if [[ $2 =~ ^s3:// ]]; then
-    S3OUTPUT=$2
-else
-    if [[ -d "${2}" ]]; then
-        OUTPUT=$2/${FULLNAME}
-    else
-        OUTPUT=${2-$OUTPUT}
-    fi
-fi
+OUTPUT=$2/${FULLNAME}
 
 REVISION="cggcc-${CG_GCC_REVISION}-gcc-${GCC_REVISION}"
-echo "ce-build-revision:${REVISION}"
-echo "ce-build-output:${OUTPUT}"
-
-if [[ "${REVISION}" == "${LAST_REVISION}" ]]; then
-    echo "ce-build-status:SKIPPED"
-    exit
-fi
-
-## From now, no unset variable
-set -u
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
 
 OUTPUT=$(realpath "${OUTPUT}")
 
@@ -156,15 +137,15 @@ pushd rustc_codegen_gcc
 
 ## Temporary fix, see
 ## https://github.com/compiler-explorer/compiler-explorer/issues/4715. This
-## causes all the crates to be build with panic=abort, so any unwinding will be
+## causes all the crates to be built with panic=abort, so any unwinding will be
 ## doomed :(
 CG_RUSTFLAGS=-Cpanic=abort \
     ./build.sh --release
 
 popd
 
 ##
-## Everything should be correctly build
+## Everything should be correctly built
 ##
 
 ##
@@ -217,13 +198,4 @@ test test.s
 ./test
 
 # Don't try to compress the binaries as they don't like it
-pushd toolroot
-
-export XZ_DEFAULTS="-T 0"
-tar Jcf "${OUTPUT}" --transform "s,^./,./${BASENAME}/," ./
-
-if [[ -n "${S3OUTPUT}" ]]; then
-    aws s3 cp --storage-class REDUCED_REDUNDANCY "${OUTPUT}" "${S3OUTPUT}"
-fi
-
-echo "ce-build-status:OK"
+complete toolroot "${BASENAME}" "${OUTPUT}"