Tidy up and test some misc stuff in common. Fixes some builds

Author: mattgodbolt

Dockerfile.misc

@@ -52,8 +52,6 @@ RUN cd /tmp && \
     ./aws/install && \
     rm -rf aws*
 
-RUN pip3 install conan
-
 RUN mkdir -p /opt/compiler-explorer
 
 # Add github public key to known_hosts, to enable interaction-less clone
@@ -79,5 +77,6 @@ ENV PATH=/cmake/bin:${PATH}
 
 RUN mkdir -p /root
 COPY misc /root/
+COPY common.sh /root/
 
 WORKDIR /root

common.sh

@@ -10,7 +10,10 @@ get_remote_revision() {
     local BRANCH="$2"
     local REVISION
     REVISION=$(git ls-remote --tags --heads "${URL}" "refs/${BRANCH}" | cut -f 1)
-    [[ -z "$REVISION" ]] && exit 255
+    if [[ -z "$REVISION" ]]; then
+        >&2 echo "Unable to get remote revision for ${URL} refs/${BRANCH}"
+        exit 255
+    fi
     echo "$REVISION"
 }
 

misc/build-6502-c++.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 set -ex
+source common.sh
 
-ROOT=$(pwd)
 VERSION=$1
-if echo ${VERSION} | grep 'trunk'; then
+if [[ "${VERSION}" = "trunk" ]]; then
     VERSION=trunk-$(date +%Y%m%d)
     BRANCH=master
 else
@@ -14,55 +14,31 @@ fi
 URL=https://github.com/lefticus/6502-cpp.git
 
 FULLNAME=6502-c++-${VERSION}.tar.xz
-OUTPUT=${ROOT}/${FULLNAME}
-S3OUTPUT=
-if [[ $2 =~ ^s3:// ]]; then
-    S3OUTPUT=$2
-else
-    if [[ -d "${2}" ]]; then
-        OUTPUT=$2/${FULLNAME}
-    else
-        OUTPUT=${2-$OUTPUT}
-    fi
-fi
+OUTPUT=$2/${FULLNAME}
 
-GIT_REVISION=$(git ls-remote --heads ${URL} refs/heads/${BRANCH} | cut -f 1)
-REVISION="6502-c++-${GIT_REVISION}"
-LAST_REVISION="${3}"
+REVISION="6502-c++-$(get_remote_revision "${URL}" "heads/${BRANCH}")"
+LAST_REVISION="${3:-}"
 
-echo "ce-build-revision:${REVISION}"
-echo "ce-build-output:${OUTPUT}"
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
 
-if [[ "${REVISION}" == "${LAST_REVISION}" ]]; then
-    echo "ce-build-status:SKIPPED"
-    exit
-fi
+pip3 install conan==1.59.0
 
 PREFIX=$(pwd)/prefix
 DIR=$(pwd)/6502-c++
 BUILDDIR=${DIR}/build
 
-OPT=/opt/compiler-explorer
-GXXPATH=/opt/compiler-explorer/gcc-12.1.0
-
-git clone ${URL} ${DIR}
+git clone "${URL}" "${DIR}"
 
-mkdir ${BUILDDIR}
-cd ${BUILDDIR}
+mkdir "${BUILDDIR}"
+cd "${BUILDDIR}"
 
+GXXPATH=/opt/compiler-explorer/gcc-12.1.0
 CXX=${GXXPATH}/bin/g++ cmake ..
-make 6502-c++
-
-patchelf --set-rpath /opt/compiler-explorer/gcc-12.1.0/lib64 bin/6502-c++
+make "-j$(nproc)" 6502-c++
 
-mkdir -p ${PREFIX}
-mv bin ${PREFIX}/bin
+patchelf --set-rpath "${GXXPATH}/lib64" bin/6502-c++
 
-export XZ_DEFAULTS="-T 0"
-tar Jcf ${OUTPUT} --transform "s,^./,./6502-c++-${VERSION}/," -C ${PREFIX} .
-
-if [[ -n "${S3OUTPUT}" ]]; then
-    aws s3 cp --storage-class REDUCED_REDUNDANCY "${OUTPUT}" "${S3OUTPUT}"
-fi
+mkdir -p "${PREFIX}"
+mv bin "${PREFIX}/bin"
 
-echo "ce-build-status:OK"
+complete "${PREFIX}" "6502-c++-${VERSION}" "${OUTPUT}"

misc/build-beebasm.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 set -ex
+source common.sh
 
-ROOT=$(pwd)
 VERSION=$1
-if echo ${VERSION} | grep 'trunk'; then
+if [[ "${VERSION}" = "trunk" ]]; then
     VERSION=trunk-$(date +%Y%m%d)
     BRANCH=master
 else
@@ -14,45 +14,21 @@ fi
 URL=https://github.com/stardot/beebasm
 
 FULLNAME=beebasm-${VERSION}.tar.xz
-OUTPUT=${ROOT}/${FULLNAME}
-S3OUTPUT=
-if [[ $2 =~ ^s3:// ]]; then
-    S3OUTPUT=$2
-else
-    if [[ -d "${2}" ]]; then
-        OUTPUT=$2/${FULLNAME}
-    else
-        OUTPUT=${2-$OUTPUT}
-    fi
-fi
+OUTPUT=$2/${FULLNAME}
 
-BEEBASM_REVISION=$(git ls-remote --heads ${URL} refs/heads/${BRANCH} | cut -f 1)
-REVISION="beebasm-${BEEBASM_REVISION}"
-LAST_REVISION="${3}"
+REVISION="beebasm-$(get_remote_revision "${URL}" "heads/${BRANCH}")"
+LAST_REVISION="${3:-}"
 
-echo "ce-build-revision:${REVISION}"
-echo "ce-build-output:${OUTPUT}"
-
-if [[ "${REVISION}" == "${LAST_REVISION}" ]]; then
-    echo "ce-build-status:SKIPPED"
-    exit
-fi
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
 
 PREFIX=$(pwd)/prefix
 DIR=$(pwd)/beebasm
-git clone --depth 1 -b ${BRANCH} ${URL} ${DIR}
+BUILD=$(pwd)/build
+git clone --depth 1 -b "${BRANCH}" "${URL}" "${DIR}"
 
-cd ${DIR}/src
-make all
-mkdir -p ${PREFIX}
-cd ..
-cp beebasm ${PREFIX}
-
-export XZ_DEFAULTS="-T 0"
-tar Jcf ${OUTPUT} --transform "s,^./,./beebasm-${VERSION}/," -C ${PREFIX} .
-
-if [[ -n "${S3OUTPUT}" ]]; then
-    aws s3 cp --storage-class REDUCED_REDUNDANCY "${OUTPUT}" "${S3OUTPUT}"
-fi
+cmake -S "${DIR}" -B "${BUILD}" -DCMAKE_BUILD_TYPE=Release -GNinja
+ninja -C "${BUILD}"
+mkdir -p "${PREFIX}"
+cp "${BUILD}/beebasm" "${PREFIX}"
 
-echo "ce-build-status:OK"
+complete "${PREFIX}" "beebasm-${VERSION}" "${OUTPUT}"

misc/build-chibicc.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 
 ## $1 : version, chibicc does not have any and only uses main branch.
-## $2 : destination: a directory or S3 path (eg. s3://...)
+## $2 : destination: a directory
 ## $3 : last revision successfully build
 
 set -ex
+source common.sh
 
-ROOT=$PWD
 VERSION="${1}"
-LAST_REVISION="${3}"
+LAST_REVISION="${3:-}"
 
 if [[ "${VERSION}" != "main" ]]; then
     echo "Only support building main"
@@ -17,50 +17,15 @@ fi
 
 URL="https://github.com/rui314/chibicc.git"
 BRANCH="main"
-
-REVISION=$(git ls-remote --heads "${URL}" "refs/heads/${BRANCH}" | cut -f 1)
-echo "ce-build-revision:${REVISION}"
-
-if [[ "${REVISION}" == "${LAST_REVISION}" ]]; then
-    echo "ce-build-status:SKIPPED"
-    exit
-fi
+REVISION=$(get_remote_revision "${URL}" "heads/${BRANCH}")
 
 FULLNAME=chibicc-${VERSION}-$(date +%Y%m%d)
+OUTPUT=$(realpath "$2/${FULLNAME}.tar.xz")
 
-OUTPUT=${ROOT}/${FULLNAME}.tar.xz
-S3OUTPUT=
-if [[ $2 =~ ^s3:// ]]; then
-    S3OUTPUT=$2
-else
-    if [[ -d "${2}" ]]; then
-        OUTPUT=$2/${FULLNAME}.tar.xz
-    else
-        OUTPUT=${2-$OUTPUT}
-    fi
-fi
-
-## From now, no unset variable
-set -u
-
-OUTPUT=$(realpath "${OUTPUT}")
-
-rm -rf  build-chibicc
-mkdir -p build-chibicc
-
-pushd build-chibicc
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
 
 git clone --depth 1 "${URL}" --branch "${BRANCH}"
-pushd chibicc
-
-make chibicc -j"$(nproc)"
 
-export XZ_DEFAULTS="-T 0"
-tar Jcf "${OUTPUT}" --transform "s,^./,./${FULLNAME}/," ./
-
-if [[ -n "${S3OUTPUT}" ]]; then
-    aws s3 cp --storage-class REDUCED_REDUNDANCY "${OUTPUT}" "${S3OUTPUT}"
-fi
+make -C chibicc chibicc -j"$(nproc)"
 
-popd
-popd
+complete chibicc "${FULLNAME}" "${OUTPUT}" 

misc/build-z88dk.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 set -ex
+source common.sh
 
-ROOT=$(pwd)
 VERSION=$1
 if echo ${VERSION} | grep 'trunk'; then
     VERSION=trunk-$(date +%Y%m%d)
@@ -14,29 +14,15 @@ fi
 URL=https://github.com/z88dk/z88dk.git
 
 FULLNAME=z88dk-${VERSION}.tar.xz
-OUTPUT=${ROOT}/${FULLNAME}
-S3OUTPUT=
-if [[ $2 =~ ^s3:// ]]; then
-    S3OUTPUT=$2
-else
-    if [[ -d "${2}" ]]; then
-        OUTPUT=$2/${FULLNAME}
-    else
-        OUTPUT=${2-$OUTPUT}
-    fi
-fi
+OUTPUT=$2/${FULLNAME}
 
-GIT_REVISION=$(git ls-remote --heads ${URL} refs/heads/${BRANCH} | cut -f 1)
+GIT_REVISION=$(get_remote_revision "${URL}" "heads/${BRANCH}")
 REVISION="z88dk-${GIT_REVISION}"
-LAST_REVISION="${3}"
+LAST_REVISION="${3:-}"
 
-echo "ce-build-revision:${REVISION}"
-echo "ce-build-output:${OUTPUT}"
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
 
-if [[ "${REVISION}" == "${LAST_REVISION}" ]]; then
-    echo "ce-build-status:SKIPPED"
-    exit
-fi
+pip3 install conan==1.59.0
 
 PREFIX=$(pwd)/prefix
 DIR=$(pwd)/z88dk
@@ -58,11 +44,4 @@ export CXXFLAGS=$(pkg-config boost --cflags)
 ./build.sh -i ${PREFIX}
 make install DESTDIR=${PREFIX}
 
-export XZ_DEFAULTS="-T 0"
-tar Jcf ${OUTPUT} --transform "s,^./,./z88dk-${VERSION}/," -C ${PREFIX} .
-
-if [[ -n "${S3OUTPUT}" ]]; then
-    aws s3 cp --storage-class REDUCED_REDUNDANCY "${OUTPUT}" "${S3OUTPUT}"
-fi
-
-echo "ce-build-status:OK"
+complete "${PREFIX}" "${FULLNAME}" "${OUTPUT}"

misc/custom-clean-fetch.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ## $1 : version, currently rustc_cg_gcc does not have any and only uses master branch.
-## $2 : destination: a directory or S3 path (eg. s3://...)
+## $2 : destination: a directory
 ## $3 : last revision (as mangled below) successfully build (optional)
 
 set -eu