Skip to content

Commit 5f00b09

Browse files
authored
Merge pull request #796 from heroku/platform-repo-snapshots
Platform package repository snapshots (and support for build metadata in package versions)
2 parents f11fce9 + 0f0a507 commit 5f00b09

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+921
-130
lines changed

.github/workflows/ci.yml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,17 @@ jobs:
5757
echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH"
5858
- name: Hatchet setup
5959
run: bundle exec hatchet ci:setup
60-
- name: Export HEROKU_PHP_PLATFORM_REPOSITORIES to …${{env.src_path_suffix}} (since we are not building main or a tag)
61-
if: github.ref_type != 'tag' && github.ref_name != 'main'
60+
- name: Calculate formulae state Hash
61+
run: |
62+
echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
63+
support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
64+
- name: Export HEROKU_PHP_PLATFORM_REPOSITORIES to …${{env.src_path_suffix}}packages-${snapshot}.json (since we are not building main or a tag, but it's a PR)
65+
if: github.ref_type != 'tag' && github.ref_name != 'main' && github.event_name == 'pull_request'
66+
run: |
67+
if [[ $STACK != heroku-22 ]]; then STACK="${STACK}-amd64"; fi
68+
echo "HEROKU_PHP_PLATFORM_REPOSITORIES=- https://lang-php.s3.us-east-1.amazonaws.com/dist-${STACK}${{env.src_path_suffix}}packages-${PLATFORM_REPO_SNAPSHOT_SHA256}.json" >> "$GITHUB_ENV"
69+
- name: Export HEROKU_PHP_PLATFORM_REPOSITORIES to …${{env.src_path_suffix}} (since we are not building main or a tag, and it's not a PR, so no snapshot)
70+
if: github.ref_type != 'tag' && github.ref_name != 'main' && github.event_name != 'pull_request'
6271
run: |
6372
if [[ $STACK != heroku-22 ]]; then STACK="${STACK}-amd64"; fi
6473
echo "HEROKU_PHP_PLATFORM_REPOSITORIES=- https://lang-php.s3.us-east-1.amazonaws.com/dist-${STACK}${{env.src_path_suffix}}" >> "$GITHUB_ENV"

.github/workflows/platform-build.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,12 +129,16 @@ jobs:
129129
path: /tmp/docker-cache.tar.gz
130130
- name: Load cached Docker image
131131
run: docker load -i /tmp/docker-cache.tar.gz
132+
- name: Calculate formulae state Hash
133+
run: |
134+
echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
135+
support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
132136
- name: Re-generate platform package repository
133-
run: docker run --rm --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} mkrepo.sh --upload
137+
run: docker run --rm --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} mkrepo.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" --upload
134138
- name: Dry-run sync.sh to show package changes available for syncing to production bucket
135139
run: |
136140
set -o pipefail
137-
(yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} sync.sh lang-php dist-${{inputs.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync.out
141+
(yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} sync.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" lang-php dist-${{inputs.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync.out
138142
- name: Output job summary
139143
run: |
140144
echo '## Package changes available for syncing to production bucket' >> "$GITHUB_STEP_SUMMARY"

.github/workflows/platform-remove.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,18 +64,22 @@ jobs:
6464
with:
6565
key: ${{ steps.restore-docker.outputs.cache-primary-key }}
6666
path: /tmp/docker-cache.tar.gz
67+
- name: Calculate formulae state Hash
68+
run: |
69+
echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
70+
support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
6771
- name: List packages for removal using given input list
6872
if: ${{ inputs.dry-run == true }}
6973
run: |
7074
set -f
7175
set -o pipefail
72-
(yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh ${{inputs.manifests}} 2>&1 | tee remove.out
76+
(yes n 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" ${{inputs.manifests}} 2>&1 | tee remove.out
7377
- name: Remove packages from repository
7478
if: ${{ inputs.dry-run == false }}
7579
run: |
7680
set -f
7781
set -o pipefail
78-
(yes 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh ${{inputs.manifests}} 2>&1 | tee remove.out
82+
(yes 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{inputs.stack}}:${{github.sha}} remove.sh -c "$PLATFORM_REPO_SNAPSHOT_SHA256" ${{inputs.manifests}} 2>&1 | tee remove.out
7983
- name: Output dry-run summary
8084
if: ${{ inputs.dry-run == true }}
8185
run: |

.github/workflows/platform-sync.yml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ jobs:
8282
sync:
8383
needs: [stack-list, docker-build]
8484
strategy:
85+
fail-fast: false
8586
matrix:
8687
stack: ${{ fromJSON(needs.stack-list.outputs.stacks) }}
8788
runs-on: ${{ endsWith(matrix.stack, '-arm64') && 'pub-hk-ubuntu-24.04-arm-small' || 'ubuntu-24.04' }}
@@ -98,14 +99,18 @@ jobs:
9899
path: /tmp/docker-cache.tar.gz
99100
- name: Load cached Docker image
100101
run: docker load -i /tmp/docker-cache.tar.gz
102+
- name: Calculate formulae state Hash
103+
run: |
104+
echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
105+
support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
101106
- name: ${{ inputs.dry-run == true && 'Dry-run sync of' || 'Sync' }} changed packages to production bucket
102107
run: |
103108
# we want to fail if 'docker run' fails; without this, 'tee' would "eat" the failure status
104109
set -o pipefail
105110
# yes gets "n" to print for dry-runs so the sync aborts
106111
# errors are redirected to /dev/null, and we || true, to suppress SIGPIPE errors from 'docker run' exiting eventually
107112
# we need -i for Docker to accept input on stdin, but must not use -t for the pipeline to work
108-
(yes "${{ inputs.dry-run == true && 'n' || 'y' }}" 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{matrix.stack}}:${{github.sha}} sync.sh lang-php dist-${{matrix.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync-${{matrix.stack}}.log
113+
(yes "${{ inputs.dry-run == true && 'n' || 'y' }}" 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{matrix.stack}}:${{github.sha}} sync.sh --no-remove -c "$PLATFORM_REPO_SNAPSHOT_SHA256" lang-php dist-${{matrix.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync-${{matrix.stack}}.log
109114
- name: Upload sync log as artifact
110115
uses: actions/upload-artifact@v4
111116
with:
@@ -119,7 +124,7 @@ jobs:
119124
echo '> **This is output from a dry-run**, no changes have been synced to production!' >> "$GITHUB_STEP_SUMMARY"
120125
echo >> "$GITHUB_STEP_SUMMARY"
121126
echo '```' >> "$GITHUB_STEP_SUMMARY"
122-
sed -n '/^The following packages will/,/POTENTIALLY DESTRUCTIVE ACTION/{/POTENTIALLY DESTRUCTIVE ACTION/!p}' sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY"
127+
sed -En '/^(The following packages will|Nothing to do except)/,/POTENTIALLY DESTRUCTIVE ACTION/{/POTENTIALLY DESTRUCTIVE ACTION/!p}' sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY"
123128
echo '```' >> "$GITHUB_STEP_SUMMARY"
124129
- name: Output sync summary
125130
if: ${{ inputs.dry-run == false }}

.github/workflows/prepare-release.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,20 @@ on:
77
permissions: {}
88

99
jobs:
10+
check-platform-repo-snapshot:
11+
runs-on: ubuntu-24.04
12+
steps:
13+
- name: Checkout
14+
uses: actions/checkout@v4
15+
- name: Calculate formulae state Hash
16+
run: |
17+
echo -n "PLATFORM_REPO_SNAPSHOT_SHA256=" >> "$GITHUB_ENV"
18+
support/build/_util/formulae-hash.sh >> "$GITHUB_ENV"
19+
- name: Check that repo snapshot URLs exist
20+
run: |
21+
curl --silent --head --fail --fail-early \
22+
"https://lang-php.s3.us-east-1.amazonaws.com/dist-heroku-{22,24-amd64,24-arm64}-stable/packages-${PLATFORM_REPO_SNAPSHOT_SHA256}.json"
1023
prepare-release:
24+
needs: check-platform-repo-snapshot
1125
uses: heroku/languages-github-actions/.github/workflows/_classic-buildpack-prepare-release.yml@latest
1226
secrets: inherit

bin/compile

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,11 +232,24 @@ else
232232
stack_locator="${STACK}-$(dpkg --print-architecture)"
233233
fi
234234
s3_url="https://lang-php.s3.us-east-1.amazonaws.com/dist-${stack_locator}-stable/"
235+
# hash the list of "declared" build formulae, this is the "snapshot" of a fixed repo state
236+
platform_repo_url="${s3_url}packages-$("$bp_dir"/support/build/_util/formulae-hash.sh).json"
237+
if ! curl_retry_on_18 --retry-connrefused --retry 3 --connect-timeout 10 --fail --silent -I "$platform_repo_url" > /dev/null; then
238+
notice_inline "Default platform repository snapshot not available."
239+
export_env_dir "$env_dir" '^HEROKU_PHP_PLATFORM_REPOSITORY_SNAPSHOT_FALLBACK$'
240+
# let's check if a fallback to the non-snapshot URL is allowed
241+
if [[ "${HEROKU_PHP_PLATFORM_REPOSITORY_SNAPSHOT_FALLBACK:-0}" != "0" ]]; then
242+
notice_inline 'Falling back to latest repository version as allowed by $HEROKU_PHP_PLATFORM_REPOSITORY_SNAPSHOT_FALLBACK.'
243+
platform_repo_url=$s3_url
244+
fi
245+
# if no fallback is allowed, we do nothing - it is possible that $HEROKU_PHP_PLATFORM_REPOSITORIES removes the default repo
246+
fi
247+
235248
# prepend the default repo to the list configured by the user
236249
# list of repositories to use is in ascening order of precedence
237250
export_env_dir "$env_dir" '^HEROKU_PHP_PLATFORM_REPOSITORIES$'
238251
have_custom_platform_repos="${HEROKU_PHP_PLATFORM_REPOSITORIES:+1}"
239-
HEROKU_PHP_PLATFORM_REPOSITORIES="${s3_url} ${HEROKU_PHP_PLATFORM_REPOSITORIES:-}"
252+
HEROKU_PHP_PLATFORM_REPOSITORIES="${platform_repo_url} ${HEROKU_PHP_PLATFORM_REPOSITORIES:-}"
240253
if [[ "${HEROKU_PHP_PLATFORM_REPOSITORIES}" == *" - "* ]]; then
241254
# a single "-" in the user supplied string removes everything to the left of it; can be used to delete the default repo
242255
notice_inline "Default platform repository disabled."

buildpack.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ name = "PHP"
66
".github/",
77
".gitignore",
88
".rspec_parallel",
9-
"support/build/",
9+
"support/build/_*/",
1010
"support/devcenter/",
11+
"support/util/",
1112
"test/",
1213
"Gemfile",
1314
"Gemfile.lock",

support/build/README.md

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,14 @@ Additional dependencies can be expressed as well; for example, if an extension r
371371

372372
The name of a package must begin with "`heroku-sys/`", and the part after this suffix must be the name Composer expects for the corresponding platform package. A PHP runtime package must thus be named "`heroku-sys/php`", and a "foobar" extension, known to Composer as "`ext-foobar`", must be named "`heroku-sys/ext-foobar`".
373373

374+
#### Package version
375+
376+
The version of a package must conform to [Composer's version specification](https://getcomposer.org/doc/articles/versions.md) and [Semantic Versioning](https://semver.org).
377+
378+
When updating the build for an existing, published version of a package (e.g. with updated compile options), it is recommended to use *build metadata* in the version string of the updated package in order to distinguish it from the existing, published version. For example, `php-8.4.6` would become `php-8.4.6+build2`.
379+
380+
Per Semver specifications, any build metadata in a version string is ignored during version comparison. This means it could not be guaranteed that `php-8.4.6+build2` is picked, if it exists in the same repository as `php-8.4.6`. For this reason, when [(Re-)generating Repositories](re-generating-repositories), only the package/version combination with the "highest" (compared using lenient version string comparison) build metadata will be included, meaning that in the example above, `php-8.4.6` would not be included in the generated repository, only `php-8.4.6+build2`.
381+
374382
#### Package Type
375383

376384
The `type` of a package must be one of the following:
@@ -566,6 +574,20 @@ Or by targeting several possible values for `heroku-sys/heroku`:
566574

567575
However, as many of Heroku's other packages are stack-specific in their library usage, separate repositories have to be kept anyway, so the `newrelic` extension is treated the same way as all other packages.
568576

577+
### Repository Snapshots
578+
579+
It can be desirable to have immutable "frozen" repository states for the buildpack, or apps, to refer to. The buildpack itself uses this for the default repository to avoid older releases picking up newly published platform packages.
580+
581+
To achieve this, a `packages-${snapshot}.json` can be [generated](#re-generating-repositories) and [synced](#syncing-repositories) in addition to the "bleeding edge" `packages.json`. The buildpack uses a hash of the list of platform package formulae as the value for `${snapshot}`. This way, the hash changes every time a formula file name is added or changed.
582+
583+
The hash is computed and printed by the `formulae-hash.sh` helper in the `_util` directory; this hash can then be passed to `mkrepo.sh`, `sync.sh` and `remove.sh` using the `-c` option. The buildpack also uses it in `bin/compile` to construct the expected URL to `platform-${snapshot}.json` for the default platform repository.
584+
585+
**When updating build formulae without changing the version (e.g. when changing compile options), to ensure that existing package builds are not updated, build metadata should be used in the [version](#package-version) of the updated package (so e.g. `php-8.4.6` becomes `php-8.4.6+build2`).** This way, the updated build can co-exist with the older builds, and existing repository snapshots will not pick up the changed build, because the hash of the list of formulae has changed due to the updated formula filename.
586+
587+
When [(re)-generating repositories](#re-generating-repositories), existing snapshots can be overwritten. This is intentional, as it allows iteration during the development in a dev/staging bucket before [syncing](#syncing-repositories).
588+
589+
However, the tooling for [syncing repositories](#syncing-repositories) will not allow updating of existing snapshots in the destination bucket to ensure their integrity.
590+
569591
### (Re-)generating Repositories
570592

571593
The normal flow is to run `deploy.sh` first to deploy one or more packages, and then to use `mkrepo.sh` to re-generate the repo:
@@ -574,11 +596,17 @@ The normal flow is to run `deploy.sh` first to deploy one or more packages, and
574596

575597
This will generate `packages.json` and upload it right away, or, if the `--upload` is not given, print upload instructions for `s5cmd`.
576598

577-
Alternatively, `deploy.sh` can be called with `--publish` as the first argument, in which case `mkrepo.sh --upload` will be called after the package deploy and manifest upload was successful:
599+
If option `-c` is given to `mkrepo.sh`, the option value will be treated as a "snapshot" identifier; the result will be a `packages-${snapshot}.json` in addition to `packages.json`:
600+
601+
~ $ mkrepo.sh -c "$(formulae-hash.sh)" --upload
602+
603+
**Also see [Repository Snapshots](#repository-snapshots) above for snapshot usage considerations.**
604+
605+
As an alternative to manually running `mkrepo.sh`, `deploy.sh` can be called with `--publish` as the first argument, in which case `mkrepo.sh --upload` will be called after the package deploy and manifest upload was successful:
578606

579607
~ $ deploy.sh --publish php-6.0.0
580608

581-
**This should be used with caution, as several parallel `deploy.sh` invocations could result in a race condition when re-generating the repository.**
609+
**This should be used with caution, as several parallel `deploy.sh` invocations could result in a race condition when re-generating the repository. It is mostly useful for speeding up debugging or development.**
582610

583611
### Syncing Repositories
584612

@@ -592,6 +620,12 @@ After testing builds, the contents of that "develop" repository can then be sync
592620

593621
The `sync.sh` script automatically detects additions, updates and removals based on manifests. It will also warn if the source `packages.json` is not up to date with its manifests, and prompt for confirmation before syncing.
594622

623+
If option `-c` is given to `mkrepo.sh`, the option value will be treated as a "snapshot" identifier. In this case, the snapshot must exist in the source bucket, but is not allowed to already exist in the destination bucket. This prevents the modification of existing buckets:
624+
625+
~ $ sync.sh -c "$(formulae-hash.sh)" my-bucket dist-heroku-24-amd64-stable/ us-east-1 my-bucket dist-heroku-24-amd64-develop/ us-east-1
626+
627+
**Also see [Repository Snapshots](#repository-snapshots) above for snapshot usage considerations.**
628+
595629
#### Syncing from Upstream
596630

597631
You will usually use an [Upstream Bucket](#understanding-upstream-buckets) to ensure that Bob will pull dependencies from Heroku's official bucket without having to worry about maintaining packages up the dependency tree, such as library or PHP prerequsites for an extension.
@@ -606,7 +640,13 @@ The `remove.sh` helper removes a package manifest and its tarball from a bucket,
606640

607641
~ $ remove.sh ext-imagick-3.4.4_php-7.3.composer.json ext-imagick-3.4.4_php-7.4.composer.json
608642

609-
Unless the `--no-publish` option is given, the repository will be re-generated immediately after removal. Otherwise, the manifests and tarballs would be removed, but the main repository would remain in place, pointing to non-existing packages, so usage of this flag is only recommended for debugging purposes or similar.
643+
This will always [re-generate the repository](#re-generating-repositories). **Packages should typically only be removed from staging/development buckets during development,** especially when [repository snapshots](#repository-snapshots) are used.
644+
645+
If option `-c` is given to `remove.sh`, the option value will be treated as a "snapshot" identifier; the result will be a `packages-${snapshot}.json` in addition to `packages.json`:
646+
647+
~ $ remove.sh -c "$(formulae-hash.sh)" ext-imagick-3.4.4_php-7.3.composer.json ext-imagick-3.4.4_php-7.4.composer.json
648+
649+
**Also see [Repository Snapshots](#repository-snapshots) above for snapshot usage considerations.**
610650

611651
## Examples
612652

support/build/_util/deploy.sh

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,22 @@ set -o pipefail
55
# fail harder
66
set -eu
77

8+
help=false
9+
810
publish=false
911

1012
# process flags
11-
optstring=":-:"
13+
optstring=":-:h"
1214
while getopts "$optstring" opt; do
1315
case $opt in
16+
h)
17+
help=true
18+
;;
1419
-)
1520
case "$OPTARG" in
21+
help)
22+
help=true
23+
;;
1624
publish)
1725
publish=true
1826
break
@@ -27,7 +35,7 @@ done
2735
# clear processed "publish" argument
2836
shift $((OPTIND-1))
2937

30-
if [[ $# -lt 1 ]]; then
38+
if $help || [[ $# -lt 1 ]]; then
3139
cat >&2 <<-EOF
3240
Usage: $(basename "$0") [--publish] FORMULA-VERSION [--overwrite]
3341
If --publish is given, mkrepo.sh will be invoked after a successful deploy to

support/build/_util/formulae-hash.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
# fail hard
4+
set -o pipefail
5+
# fail harder
6+
set -eu
7+
8+
find_cmd=$(command -v gfind find | head -n1) # prefer gfind, since we use a GNU extension (-printf)
9+
10+
# default for $1
11+
scan_dir=$(dirname "$BASH_SOURCE")/../
12+
scan_dir=${1:-$scan_dir}
13+
14+
# Find using $find_cmd
15+
# - in $scan_dir
16+
# - first expression group:
17+
# - all directories
18+
# - that have a name starting with an underscore
19+
# - get pruned (removed)
20+
# - second expression group:
21+
# - all files
22+
# - that match a name-with-version-number looking pattern
23+
# - get printed using their "nested path", meaning with the $scan_dir prefix removed
24+
# Sort (with forced C collation for sorting) for stable hashing
25+
# Calculate SHA256 hash
26+
# Extract just the hash from sha256sum output
27+
"$find_cmd" "$scan_dir" -type d -name '_*' -prune -or -type f -name '*-[0-9]*.[0-9]*' -printf '%P\n' \
28+
| LC_ALL=C sort --version-sort | sha256sum | cut -d" " -f1

0 commit comments

Comments
 (0)