Always try "stanza-upgrade" after a failed "stanza-create"

Error messages could be on either stderr or stdout depending on logging
options. Error messages and exit codes could change unexpectedly, so
use a shell list to run "stanza-upgrade" any time "stanza-create" exits
non-zero.

Issue: PGO-1558

Author: cbandy

internal/controller/postgrescluster/pgbackrest.go

@@ -2678,8 +2678,7 @@ func (r *Reconciler) reconcileStanzaCreate(ctx context.Context,
 	}
 
 	// Always attempt to create pgBackRest stanza first
-	configHashMismatch, err := pgbackrest.Executor(exec).StanzaCreateOrUpgrade(ctx, configHash,
-		false, postgresCluster)
+	configHashMismatch, err := pgbackrest.Executor(exec).StanzaCreateOrUpgrade(ctx, configHash, postgresCluster)
 	if err != nil {
 		// record and log any errors resulting from running the stanza-create command
 		r.Recorder.Event(postgresCluster, corev1.EventTypeWarning, EventUnableToCreateStanzas,

internal/pgbackrest/pgbackrest.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"strings"
 
 	"github.com/pkg/errors"
 
@@ -24,10 +23,6 @@ const (
 	// errMsgStaleReposWithVolumesConfig is the error message displayed when a volume-backed repo has been
 	// configured, but the configuration has not yet propagated into the container.
 	errMsgStaleReposWithVolumesConfig = "postgres operator error: pgBackRest stale volume-backed repo configuration"
-
-	// errMsgBackupDbMismatch is the error message returned from pgBackRest when PG versions
-	// or PG system identifiers do not match between the PG instance and the existing stanza
-	errMsgBackupDbMismatch = "backup and archive info files exist but do not match the database"
 )
 
 // Executor calls "pgbackrest" commands
@@ -46,15 +41,10 @@ type Executor func(
 // from running (with a config mismatch indicating that the pgBackRest configuration as stored in
 // the cluster's pgBackRest ConfigMap has not yet propagated to the Pod).
 func (exec Executor) StanzaCreateOrUpgrade(ctx context.Context, configHash string,
-	upgrade bool, postgresCluster *v1beta1.PostgresCluster) (bool, error) {
+	postgresCluster *v1beta1.PostgresCluster) (bool, error) {
 
 	var stdout, stderr bytes.Buffer
 
-	stanzaCmd := "create"
-	if upgrade {
-		stanzaCmd = "upgrade"
-	}
-
 	var reposWithVolumes []v1beta1.PGBackRestRepo
 	for _, repo := range postgresCluster.Spec.Backups.PGBackRest.Repos {
 		if repo.Volume != nil {
@@ -83,18 +73,18 @@ func (exec Executor) StanzaCreateOrUpgrade(ctx context.Context, configHash strin
 	// Otherwise, it runs the pgbackrest command, which will either be "stanza-create" or
 	// "stanza-upgrade", depending on the value of the boolean "upgrade" parameter.
 	const script = `
-declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" cmd="$5" check_repo_cmd="$6"
+declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" check_repo_cmd="$5"
 if [[ "$(< /etc/pgbackrest/conf.d/config-hash)" != "${hash}" ]]; then
     printf >&2 "%s" "${hash_msg}"; exit 1;
 elif ! bash -c "${check_repo_cmd}"; then
  	 printf >&2 "%s" "${vol_msg}"; exit 1;
 else
-    pgbackrest "${cmd}" --stanza="${stanza}"
+    pgbackrest stanza-create --stanza="${stanza}" || pgbackrest stanza-upgrade --stanza="${stanza}"
 fi
 `
 	if err := exec(ctx, nil, &stdout, &stderr, "bash", "-ceu", "--",
 		script, "-", configHash, DefaultStanzaName, errMsgConfigHashMismatch, errMsgStaleReposWithVolumesConfig,
-		fmt.Sprintf("stanza-%s", stanzaCmd), checkRepoCmd); err != nil {
+		checkRepoCmd); err != nil {
 
 		errReturn := stderr.String()
 
@@ -111,12 +101,6 @@ fi
 			return true, nil
 		}
 
-		// if the err returned from pgbackrest command is about a version mismatch
-		// then we should run upgrade rather than create
-		if strings.Contains(errReturn, errMsgBackupDbMismatch) {
-			return exec.StanzaCreateOrUpgrade(ctx, configHash, true, postgresCluster)
-		}
-
 		// if none of the above errors, return the err
 		return false, errors.WithStack(fmt.Errorf("%w: %v", err, errReturn))
 	}

internal/pgbackrest/pgbackrest_test.go

@@ -28,18 +28,17 @@ func TestStanzaCreateOrUpgrade(t *testing.T) {
 	ctx := context.Background()
 	configHash := "7f5d4d5bdc"
 	expectedCommand := []string{"bash", "-ceu", "--", `
-declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" cmd="$5" check_repo_cmd="$6"
+declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" check_repo_cmd="$5"
 if [[ "$(< /etc/pgbackrest/conf.d/config-hash)" != "${hash}" ]]; then
     printf >&2 "%s" "${hash_msg}"; exit 1;
 elif ! bash -c "${check_repo_cmd}"; then
  	 printf >&2 "%s" "${vol_msg}"; exit 1;
 else
-    pgbackrest "${cmd}" --stanza="${stanza}"
+    pgbackrest stanza-create --stanza="${stanza}" || pgbackrest stanza-upgrade --stanza="${stanza}"
 fi
 `,
 		"-", "7f5d4d5bdc", "db", "postgres operator error: pgBackRest config hash mismatch",
 		"postgres operator error: pgBackRest stale volume-backed repo configuration",
-		"stanza-create",
 		"grep repo1-path /etc/pgbackrest/conf.d/pgbackrest_instance.conf",
 	}
 
@@ -84,7 +83,7 @@ fi
 		},
 	}
 
-	configHashMismatch, err := Executor(stanzaExec).StanzaCreateOrUpgrade(ctx, configHash, false, postgresCluster)
+	configHashMismatch, err := Executor(stanzaExec).StanzaCreateOrUpgrade(ctx, configHash, postgresCluster)
 	assert.NilError(t, err)
 	assert.Assert(t, !configHashMismatch)