Merge pull request #796 from heroku/platform-repo-snapshots

Platform package repository snapshots (and support for build metadata in package versions)

Author: dzuelke

.github/workflows/ci.yml

@@ -57,8 +57,17 @@ jobs:
           echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH"
       - name: Hatchet setup
         run: bundle exec hatchet ci:setup
-      - name: Export HEROKU_PHP_PLATFORM_REPOSITORIES to …${{env.src_path_suffix}} (since we are not building main or a tag)
-        if: github.ref_type != 'tag' && github.ref_name != 'main'
+      - name: Calculate formulae state Hash
+        run: |
+          echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
+          support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
+      - name: Export HEROKU_PHP_PLATFORM_REPOSITORIES to …${{env.src_path_suffix}}packages-${snapshot}.json (since we are not building main or a tag, but it's a PR)
+        if: github.ref_type != 'tag' && github.ref_name != 'main' && github.event_name == 'pull_request'
+        run: |
+          if [[ $STACK != heroku-22 ]]; then STACK="${STACK}-amd64"; fi
+          echo "HEROKU_PHP_PLATFORM_REPOSITORIES=- https://lang-php.s3.us-east-1.amazonaws.com/dist-${STACK}${{env.src_path_suffix}}packages-${PLATFORM_REPO_SNAPSHOT_SHA256}.json" >> "$GITHUB_ENV"
+      - name: Export HEROKU_PHP_PLATFORM_REPOSITORIES to …${{env.src_path_suffix}} (since we are not building main or a tag, and it's not a PR, so no snapshot)
+        if: github.ref_type != 'tag' && github.ref_name != 'main' && github.event_name != 'pull_request'
         run: |
           if [[ $STACK != heroku-22 ]]; then STACK="${STACK}-amd64"; fi
           echo "HEROKU_PHP_PLATFORM_REPOSITORIES=- https://lang-php.s3.us-east-1.amazonaws.com/dist-${STACK}${{env.src_path_suffix}}" >> "$GITHUB_ENV"

.github/workflows/platform-build.yml

@@ -129,12 +129,16 @@ jobs:
           path: /tmp/docker-cache.tar.gz
       - name: Load cached Docker image
         run: docker load -i /tmp/docker-cache.tar.gz
+      - name: Calculate formulae state Hash
+        run: |
+          echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
+          support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
       - name: Re-generate platform package repository
-        run: docker run --rm --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} mkrepo.sh --upload
+        run: docker run --rm --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} mkrepo.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" --upload
       - name: Dry-run sync.sh to show package changes available for syncing to production bucket
         run: |
           set -o pipefail
-          (yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} sync.sh lang-php dist-${{inputs.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync.out
+          (yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} sync.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" lang-php dist-${{inputs.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync.out
       - name: Output job summary
         run: |
           echo '## Package changes available for syncing to production bucket' >> "$GITHUB_STEP_SUMMARY"

.github/workflows/platform-remove.yml

@@ -64,18 +64,22 @@ jobs:
         with:
           key: ${{ steps.restore-docker.outputs.cache-primary-key }}
           path: /tmp/docker-cache.tar.gz
+      - name: Calculate formulae state Hash
+        run: |
+          echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
+          support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
       - name: List packages for removal using given input list
         if: ${{ inputs.dry-run == true }}
         run: |
           set -f
           set -o pipefail
-          (yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh ${{inputs.manifests}} 2>&1 | tee remove.out
+          (yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" ${{inputs.manifests}} 2>&1 | tee remove.out
       - name: Remove packages from repository
         if: ${{ inputs.dry-run == false }}
         run: |
           set -f
           set -o pipefail
-          (yes 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh ${{inputs.manifests}} 2>&1 | tee remove.out
+          (yes 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" ${{inputs.manifests}} 2>&1 | tee remove.out
       - name: Output dry-run summary
         if: ${{ inputs.dry-run == true }}
         run: |

.github/workflows/platform-sync.yml

@@ -82,6 +82,7 @@ jobs:
   sync:
     needs: [stack-list, docker-build]
     strategy:
+      fail-fast: false
       matrix:
         stack: ${{ fromJSON(needs.stack-list.outputs.stacks) }}
     runs-on: ${{ endsWith(matrix.stack, '-arm64') && 'pub-hk-ubuntu-24.04-arm-small' || 'ubuntu-24.04' }}
@@ -98,14 +99,18 @@ jobs:
           path: /tmp/docker-cache.tar.gz
       - name: Load cached Docker image
         run: docker load -i /tmp/docker-cache.tar.gz
+      - name: Calculate formulae state Hash
+        run: |
+          echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
+          support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
       - name: ${{ inputs.dry-run == true && 'Dry-run sync of' || 'Sync' }} changed packages to production bucket
         run: |
           # we want to fail if 'docker run' fails; without this, 'tee' would "eat" the failure status
           set -o pipefail
           # yes gets "n" to print for dry-runs so the sync aborts
           # errors are redirected to /dev/null, and we || true, to suppress SIGPIPE errors from 'docker run' exiting eventually
           # we need -i for Docker to accept input on stdin, but must not use -t for the pipeline to work
-          (yes "${{ inputs.dry-run == true && 'n' || 'y' }}" 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{matrix.stack}}:${{github.sha}} sync.sh lang-php dist-${{matrix.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync-${{matrix.stack}}.log
+          (yes "${{ inputs.dry-run == true && 'n' || 'y' }}" 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{matrix.stack}}:${{github.sha}} sync.sh --no-remove -c "$PLATFORM_REPO_SNAPSHOT_SHA256" lang-php dist-${{matrix.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync-${{matrix.stack}}.log
       - name: Upload sync log as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -119,7 +124,7 @@ jobs:
           echo '> **This is output from a dry-run**, no changes have been synced to production!' >> "$GITHUB_STEP_SUMMARY"
           echo >> "$GITHUB_STEP_SUMMARY"
           echo '```' >> "$GITHUB_STEP_SUMMARY"
-          sed -n '/^The following packages will/,/POTENTIALLY DESTRUCTIVE ACTION/{/POTENTIALLY DESTRUCTIVE ACTION/!p}' sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY"
+          sed -En '/^(The following packages will|Nothing to do except)/,/POTENTIALLY DESTRUCTIVE ACTION/{/POTENTIALLY DESTRUCTIVE ACTION/!p}' sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY"
           echo '```' >> "$GITHUB_STEP_SUMMARY"
       - name: Output sync summary
         if: ${{ inputs.dry-run == false }}

.github/workflows/prepare-release.yml

@@ -7,6 +7,20 @@ on:
 permissions: {}
 
 jobs:
+  check-platform-repo-snapshot:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Calculate formulae state Hash
+        run: |
+          echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
+          support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
+      - name: Check that repo snapshot URLs exist
+        run: |
+          curl --silent --head --fail --fail-early \
+            "https://lang-php.s3.us-east-1.amazonaws.com/dist-heroku-{22,24-amd64,24-arm64}-stable/packages-${PLATFORM_REPO_SNAPSHOT_SHA256}.json"
   prepare-release:
+    needs: check-platform-repo-snapshot
     uses: heroku/languages-github-actions/.github/workflows/_classic-buildpack-prepare-release.yml@latest
     secrets: inherit

bin/compile

@@ -232,11 +232,24 @@ else
 	stack_locator="${STACK}-$(dpkg --print-architecture)"
 fi
 s3_url="https://lang-php.s3.us-east-1.amazonaws.com/dist-${stack_locator}-stable/"
+# hash the list of "declared" build formulae, this is the "snapshot" of a fixed repo state
+platform_repo_url="${s3_url}packages-$("$bp_dir"/support/build/_util/formulae-hash.sh).json"
+if ! curl_retry_on_18  --retry-connrefused --retry 3 --connect-timeout 10 --fail --silent -I "$platform_repo_url" > /dev/null; then
+	notice_inline "Default platform repository snapshot not available."
+	export_env_dir "$env_dir" '^HEROKU_PHP_PLATFORM_REPOSITORY_SNAPSHOT_FALLBACK$'
+	# let's check if a fallback to the non-snapshot URL is allowed
+	if [[ "${HEROKU_PHP_PLATFORM_REPOSITORY_SNAPSHOT_FALLBACK:-0}" != "0" ]]; then
+		notice_inline 'Falling back to latest repository version as allowed by $HEROKU_PHP_PLATFORM_REPOSITORY_SNAPSHOT_FALLBACK.'
+		platform_repo_url=$s3_url
+	fi
+	# if no fallback is allowed, we do nothing - it is possible that $HEROKU_PHP_PLATFORM_REPOSITORIES removes the default repo
+fi
+
 # prepend the default repo to the list configured by the user
 # list of repositories to use is in ascening order of precedence
 export_env_dir "$env_dir" '^HEROKU_PHP_PLATFORM_REPOSITORIES$'
 have_custom_platform_repos="${HEROKU_PHP_PLATFORM_REPOSITORIES:+1}"
-HEROKU_PHP_PLATFORM_REPOSITORIES="${s3_url} ${HEROKU_PHP_PLATFORM_REPOSITORIES:-}"
+HEROKU_PHP_PLATFORM_REPOSITORIES="${platform_repo_url} ${HEROKU_PHP_PLATFORM_REPOSITORIES:-}"
 if [[ "${HEROKU_PHP_PLATFORM_REPOSITORIES}" == *" - "* ]]; then
 	# a single "-" in the user supplied string removes everything to the left of it; can be used to delete the default repo
 	notice_inline "Default platform repository disabled."

buildpack.toml

@@ -6,8 +6,9 @@ name = "PHP"
 		".github/",
 		".gitignore",
 		".rspec_parallel",
-		"support/build/",
+		"support/build/_*/",
 		"support/devcenter/",
+		"support/util/",
 		"test/",
 		"Gemfile",
 		"Gemfile.lock",

support/build/README.md

@@ -371,6 +371,14 @@ Additional dependencies can be expressed as well; for example, if an extension r
 
 The name of a package must begin with "`heroku-sys/`", and the part after this suffix must be the name Composer expects for the corresponding platform package. A PHP runtime package must thus be named "`heroku-sys/php`", and a "foobar" extension, known to Composer as "`ext-foobar`", must be named "`heroku-sys/ext-foobar`".
 
+#### Package version
+
+The version of a package must conform to [Composer's version specification](https://getcomposer.org/doc/articles/versions.md) and [Semantic Versioning](https://semver.org).
+
+When updating the build for an existing, published version of a package (e.g. with updated compile options), it is recommended to use *build metadata* in the version string of the updated package in order to distinguish it from the existing, published version. For example, `php-8.4.6` would become `php-8.4.6+build2`.
+
+Per Semver specifications, any build metadata in a version string is ignored during version comparison. This means it could not be guaranteed that `php-8.4.6+build2` is picked, if it exists in the same repository as `php-8.4.6`. For this reason, when [(Re-)generating Repositories](re-generating-repositories), only the package/version combination with the "highest" (compared using lenient version string comparison) build metadata will be included, meaning that in the example above, `php-8.4.6` would not be included in the generated repository, only `php-8.4.6+build2`.
+
 #### Package Type
 
 The `type` of a package must be one of the following:
@@ -566,6 +574,20 @@ Or by targeting several possible values for `heroku-sys/heroku`:
 
 However, as many of Heroku's other packages are stack-specific in their library usage, separate repositories have to be kept anyway, so the `newrelic` extension is treated the same way as all other packages.
 
+### Repository Snapshots
+
+It can be desirable to have immutable "frozen" repository states for the buildpack, or apps, to refer to. The buildpack itself uses this for the default repository to avoid older releases picking up newly published platform packages.
+
+To achieve this, a `packages-${snapshot}.json` can be [generated](#re-generating-repositories) and [synced](#syncing-repositories) in addition to the "bleeding edge" `packages.json`. The buildpack uses a hash of the list of platform package formulae as the value for `${snapshot}`. This way, the hash changes every time a formula file name is added or changed.
+
+The hash is computed and printed by the `formulae-hash.sh` helper in the `_util` directory; this hash can then be passed to `mkrepo.sh`, `sync.sh` and `remove.sh` using the `-c` option. The buildpack also uses it in `bin/compile` to construct the expected URL to `platform-${snapshot}.json` for the default platform repository.
+
+**When updating build formulae without changing the version (e.g. when changing compile options), to ensure that existing package builds are not updated, build metadata should be used in the [version](#package-version) of the updated package (so e.g. `php-8.4.6` becomes `php-8.4.6+build2`).** This way, the updated build can co-exist with the older builds, and existing repository snapshots will not pick up the changed build, because the hash of the list of formulae has changed due to the updated formula filename.
+
+When [(re)-generating repositories](#re-generating-repositories), existing snapshots can be overwritten. This is intentional, as it allows iteration during the development in a dev/staging bucket before [syncing](#syncing-repositories).
+
+However, the tooling for [syncing repositories](#syncing-repositories) will not allow updating of existing snapshots in the destination bucket to ensure their integrity.
+
 ### (Re-)generating Repositories
 
 The normal flow is to run `deploy.sh` first to deploy one or more packages, and then to use `mkrepo.sh` to re-generate the repo:
@@ -574,11 +596,17 @@ The normal flow is to run `deploy.sh` first to deploy one or more packages, and
 
 This will generate `packages.json` and upload it right away, or, if the `--upload` is not given, print upload instructions for `s5cmd`.
 
-Alternatively, `deploy.sh` can be called with `--publish` as the first argument, in which case `mkrepo.sh --upload` will be called after the package deploy and manifest upload was successful:
+If option `-c` is given to `mkrepo.sh`, the option value will be treated as a "snapshot" identifier; the result will be a `packages-${snapshot}.json` in addition to `packages.json`:
+
+    ~ $ mkrepo.sh -c "$(formulae-hash.sh)" --upload
+
+**Also see [Repository Snapshots](#repository-snapshots) above for snapshot usage considerations.**
+
+As an alternative to manually running `mkrepo.sh`, `deploy.sh` can be called with `--publish` as the first argument, in which case `mkrepo.sh --upload` will be called after the package deploy and manifest upload was successful:
 
     ~ $ deploy.sh --publish php-6.0.0
 
-**This should be used with caution, as several parallel `deploy.sh` invocations could result in a race condition when re-generating the repository.**
+**This should be used with caution, as several parallel `deploy.sh` invocations could result in a race condition when re-generating the repository. It is mostly useful for speeding up debugging or development.**
 
 ### Syncing Repositories
 
@@ -592,6 +620,12 @@ After testing builds, the contents of that "develop" repository can then be sync
 
 The `sync.sh` script automatically detects additions, updates and removals based on manifests. It will also warn if the source `packages.json` is not up to date with its manifests, and prompt for confirmation before syncing.
 
+If option `-c` is given to `mkrepo.sh`, the option value will be treated as a "snapshot" identifier. In this case, the snapshot must exist in the source bucket, but is not allowed to already exist in the destination bucket. This prevents the modification of existing buckets:
+
+    ~ $ sync.sh -c "$(formulae-hash.sh)" my-bucket dist-heroku-24-amd64-stable/ us-east-1 my-bucket dist-heroku-24-amd64-develop/ us-east-1
+
+**Also see [Repository Snapshots](#repository-snapshots) above for snapshot usage considerations.**
+
 #### Syncing from Upstream
 
 You will usually use an [Upstream Bucket](#understanding-upstream-buckets) to ensure that Bob will pull dependencies from Heroku's official bucket without having to worry about maintaining packages up the dependency tree, such as library or PHP prerequsites for an extension.
@@ -606,7 +640,13 @@ The `remove.sh` helper removes a package manifest and its tarball from a bucket,
 
     ~ $ remove.sh ext-imagick-3.4.4_php-7.3.composer.json ext-imagick-3.4.4_php-7.4.composer.json
 
-Unless the `--no-publish` option is given, the repository will be re-generated immediately after removal. Otherwise, the manifests and tarballs would be removed, but the main repository would remain in place, pointing to non-existing packages, so usage of this flag is only recommended for debugging purposes or similar.
+This will always [re-generate the repository](#re-generating-repositories). **Packages should typically only be removed from staging/development buckets during development,** especially when [repository snapshots](#repository-snapshots) are used.
+
+If option `-c` is given to `remove.sh`, the option value will be treated as a "snapshot" identifier; the result will be a `packages-${snapshot}.json` in addition to `packages.json`:
+
+    ~ $ remove.sh -c "$(formulae-hash.sh)" ext-imagick-3.4.4_php-7.3.composer.json ext-imagick-3.4.4_php-7.4.composer.json
+
+**Also see [Repository Snapshots](#repository-snapshots) above for snapshot usage considerations.**
 
 ## Examples
 

support/build/patches/httpd-ignoresigterm.patch renamed to support/build/_patches/httpd-ignoresigterm.patch

@@ -5,14 +5,22 @@ set -o pipefail
 # fail harder
 set -eu
 
+help=false
+
 publish=false
 
 # process flags
-optstring=":-:"
+optstring=":-:h"
 while getopts "$optstring" opt; do
 	case $opt in
+		h)
+			help=true
+			;;
 		-)
 			case "$OPTARG" in
+				help)
+					help=true
+					;;
 				publish)
 					publish=true
 					break
@@ -27,7 +35,7 @@ done
 # clear processed "publish" argument
 shift $((OPTIND-1))
 
-if [[ $# -lt 1 ]]; then
+if $help || [[ $# -lt 1 ]]; then
 	cat >&2 <<-EOF
 		Usage: $(basename "$0") [--publish] FORMULA-VERSION [--overwrite]
 		  If --publish is given, mkrepo.sh will be invoked after a successful deploy to

support/build/patches/nginx-ignoresigterm.patch renamed to support/build/_patches/nginx-ignoresigterm.patch

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+find_cmd=$(command -v gfind find | head -n1) # prefer gfind, since we use a GNU extension (-printf)
+
+# default for $1
+scan_dir=$(dirname "$BASH_SOURCE")/../
+scan_dir=${1:-$scan_dir}
+
+# Find using $find_cmd
+# - in $scan_dir
+# - first expression group:
+#   - all directories
+#   - that have a name starting with an underscore
+#   - get pruned (removed)
+# - second expression group:
+#   - all files
+#   - that match a name-with-version-number looking pattern
+#   - get printed using their "nested path", meaning with the $scan_dir prefix removed
+# Sort (with forced C collation for sorting) for stable hashing
+# Calculate SHA256 hash
+# Extract just the hash from sha256sum output
+"$find_cmd" "$scan_dir" -type d -name '_*' -prune -or -type f -name '*-[0-9]*.[0-9]*' -printf '%P\n' \
+  | LC_ALL=C sort --version-sort | sha256sum | cut -d" " -f1