From 3c3c0acc816cc333227174fdc952a480b52f6403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 6 Nov 2025 18:09:20 +0100 Subject: [PATCH 1/4] feat: test pg_upgrade compatibility with older extension versions Add test to verify that all extension versions from PostgreSQL 15 can successfully upgrade to PostgreSQL 17 using pg_upgrade. The test now validates: - Each PG 15 extension version can be upgraded to PG 17 - Extension update scripts are properly generated during upgrade - Version handling works correctly with and without update scripts - Final extension versions match expected values after upgrade --- nix/ext/tests/default.nix | 37 +++++++-- nix/ext/tests/lib.py | 2 +- nix/ext/tests/pgmq.nix | 34 ++++++-- nix/ext/tests/pgroonga.nix | 34 +++++++- nix/ext/tests/pgsodium.nix | 120 +++++++++++++++++++++------- nix/ext/tests/postgis.nix | 157 ------------------------------------- nix/ext/tests/vault.nix | 35 ++++++++- 7 files changed, 213 insertions(+), 206 deletions(-) delete mode 100644 nix/ext/tests/postgis.nix diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index 57644ee7f..788abfedc 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -137,9 +137,6 @@ let }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -148,7 +145,9 @@ let } extension_name = "${pname}" support_upgrade = True - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -204,6 +203,33 @@ let with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; }; in @@ -227,10 +253,11 @@ builtins.listToAttrs ( "pg_hashids" "pg_jsonschema" "pg_net" + "pg_partman" "pg_stat_monitor" "pg_tle" "pgaudit" - "pg_partman" + "postgis" "vector" "wal2json" "wrappers" diff --git a/nix/ext/tests/lib.py b/nix/ext/tests/lib.py index d04f830fd..4c33bcca6 100644 --- a/nix/ext/tests/lib.py +++ b/nix/ext/tests/lib.py @@ -52,7 +52,7 @@ def run_sql_file(self, file: str) -> str: ).strip() def drop_extension(self): - self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};") + self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name} CASCADE;") def install_extension(self, version: str): if self.schema != "public": diff --git a/nix/ext/tests/pgmq.nix b/nix/ext/tests/pgmq.nix index b0c432921..28c2aa4cb 100644 --- a/nix/ext/tests/pgmq.nix +++ b/nix/ext/tests/pgmq.nix @@ -40,7 +40,7 @@ let psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; in self.inputs.nixpkgs.lib.nixos.runTest { - name = "timescaledb"; + name = "pgmq"; hostPkgs = pkgs; nodes.server = { config, ... }: @@ -106,9 +106,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -117,7 +114,9 @@ self.inputs.nixpkgs.lib.nixos.runTest { } extension_name = "${pname}" support_upgrade = True - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -174,5 +173,30 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) ''; } diff --git a/nix/ext/tests/pgroonga.nix b/nix/ext/tests/pgroonga.nix index abfef8804..bd097740f 100644 --- a/nix/ext/tests/pgroonga.nix +++ b/nix/ext/tests/pgroonga.nix @@ -126,9 +126,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -136,7 +133,9 @@ self.inputs.nixpkgs.lib.nixos.runTest { "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -188,5 +187,32 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } diff --git a/nix/ext/tests/pgsodium.nix b/nix/ext/tests/pgsodium.nix index 0eae3c0a0..90a9551cb 100644 --- a/nix/ext/tests/pgsodium.nix +++ b/nix/ext/tests/pgsodium.nix @@ -42,6 +42,8 @@ let echo 0000000000000000000000000000000000000000000000000000000000000000 '' ); + psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; in self.inputs.nixpkgs.lib.nixos.runTest { name = pname; @@ -61,7 +63,21 @@ self.inputs.nixpkgs.lib.nixos.runTest { services.postgresql = { enable = true; - package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + package = psql_15; + authentication = '' + local all postgres peer map=postgres + local all all peer map=root + ''; + identMap = '' + root root supabase_admin + postgres postgres postgres + ''; + ensureUsers = [ + { + name = "supabase_admin"; + ensureClauses.superuser = true; + } + ]; settings = { "shared_preload_libraries" = pname; "pgsodium.getkey_script" = pgsodiumGetKey; @@ -70,7 +86,7 @@ self.inputs.nixpkgs.lib.nixos.runTest { specialisation.postgresql17.configuration = { services.postgresql = { - package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + package = lib.mkForce psql_17; }; systemd.services.postgresql-migrate = { @@ -84,8 +100,8 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; script = let - oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldPostgresql = psql_15; + newPostgresql = psql_17; oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; in @@ -111,49 +127,93 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' + from pathlib import Path versions = { "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } + extension_name = "${pname}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" + ext_has_background_worker = ${ + if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" + } + sql_test_directory = Path("${../../tests}") + pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}" - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + ${builtins.readFile ./lib.py} start_all() server.wait_for_unit("multi-user.target") server.wait_for_unit("postgresql.service") - check_upgrade_path("15") + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory) + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["15"][-1] - assert f"${pname},{latestVersion}" in installed_extensions + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + if ext_has_background_worker: + with subtest("Test switch_${pname}_version"): + test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15") + + with subtest("Check pg_regress with postgresql 15 after installing the last version"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) with subtest("switch to postgresql 17"): server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" + f"{pg17_configuration}/bin/switch-to-configuration test >&2" ) - check_upgrade_path("17") + with subtest("Check last version of the extension after postgresql upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Check the install of the last version of the extension"): + test.check_install_last_version("17") + + with subtest("Check pg_regress with postgresql 17 after installing the last version"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } diff --git a/nix/ext/tests/postgis.nix b/nix/ext/tests/postgis.nix deleted file mode 100644 index f20e2de24..000000000 --- a/nix/ext/tests/postgis.nix +++ /dev/null @@ -1,157 +0,0 @@ -{ self, pkgs }: -let - pname = "postgis"; - inherit (pkgs) lib; - installedExtension = - postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; - versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; - postgresqlWithExtension = - postgresql: - let - majorVersion = lib.versions.major postgresql.version; - pkg = pkgs.buildEnv { - name = "postgresql-${majorVersion}-${pname}"; - paths = [ - postgresql - postgresql.lib - (installedExtension majorVersion) - ]; - passthru = { - inherit (postgresql) version psqlSchema; - lib = pkg; - withPackages = _: pkg; - }; - nativeBuildInputs = [ pkgs.makeWrapper ]; - pathsToLink = [ - "/" - "/bin" - "/lib" - ]; - postBuild = '' - wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib - ''; - }; - in - pkg; -in -self.inputs.nixpkgs.lib.nixos.runTest { - name = pname; - hostPkgs = pkgs; - nodes.server = - { config, ... }: - { - virtualisation = { - forwardPorts = [ - { - from = "host"; - host.port = 13022; - guest.port = 22; - } - ]; - }; - services.openssh = { - enable = true; - }; - - services.postgresql = { - enable = true; - package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - }; - - specialisation.postgresql17.configuration = { - services.postgresql = { - package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); - }; - - systemd.services.postgresql-migrate = { - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - User = "postgres"; - Group = "postgres"; - StateDirectory = "postgresql"; - WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; - }; - script = - let - oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; - oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; - newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; - in - '' - if [[ ! -d ${newDataDir} ]]; then - install -d -m 0700 -o postgres -g postgres "${newDataDir}" - ${newPostgresql}/bin/initdb -D "${newDataDir}" - ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ - --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" - else - echo "${newDataDir} already exists" - fi - ''; - }; - - systemd.services.postgresql = { - after = [ "postgresql-migrate.service" ]; - requires = [ "postgresql-migrate.service" ]; - }; - }; - }; - testScript = - { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in - '' - versions = { - "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], - "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], - } - - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" - - start_all() - - server.wait_for_unit("multi-user.target") - server.wait_for_unit("postgresql.service") - - check_upgrade_path("15") - - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension where extname = '${pname}';""") - latestVersion = versions["15"][-1] - majMinVersion = ".".join(latestVersion.split('.')[:1]) - assert f"${pname},{majMinVersion}" in installed_extensions, f"Expected ${pname} version {latestVersion}, but found {installed_extensions}" - - with subtest("switch to postgresql 17"): - server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" - ) - - with subtest("Check ${pname} latest extension version after upgrade"): - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["17"][-1] - majMinVersion = ".".join(latestVersion.split('.')[:1]) - assert f"${pname},{majMinVersion}" in installed_extensions - - check_upgrade_path("17") - ''; -} diff --git a/nix/ext/tests/vault.nix b/nix/ext/tests/vault.nix index a3f9cb24f..bea8d9d9e 100644 --- a/nix/ext/tests/vault.nix +++ b/nix/ext/tests/vault.nix @@ -134,9 +134,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -144,8 +141,10 @@ self.inputs.nixpkgs.lib.nixos.runTest { "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" support_upgrade = True - pg17_configuration = "${pg17-configuration}" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -198,5 +197,33 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.run_sql_file("${../../../ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql}") test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.run_sql_file("${../../../ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql}") + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } From 441718de393359ff13b4f1ecc60663bc7cf28312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 6 Nov 2025 19:17:02 +0100 Subject: [PATCH 2/4] fix(pg_graphql): include unsupported versions as SQL-only for pg_upgrade compatibility When upgrading PostgreSQL versions, pg_upgrade needs access to old extension versions sql to migrate from. This adds unsupported pg_graphql versions (those not compilable with current PostgreSQL) as SQL-only packages, ensuring migration paths exist when upgrading from older PostgreSQL versions. --- nix/ext/pg_graphql/default.nix | 51 ++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index a7f6d1065..898765d93 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -79,7 +79,7 @@ let preCheck = '' export PGRX_HOME="$(mktemp -d)" - export PG_VERSION="${lib.versions.major postgresql.version}" + export PG_VERSION="${pgVersion}" export NIX_PGLIBDIR="$PGRX_HOME/$PG_VERSION/lib" export PATH="$PGRX_HOME/$PG_VERSION/bin:$PATH" ${lib.getExe rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ "$PGRX_HOME/$PG_VERSION/" @@ -119,8 +119,9 @@ let } ); allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_graphql; + pgVersion = lib.versions.major postgresql.version; supportedVersions = lib.filterAttrs ( - _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + _: value: builtins.elem pgVersion value.postgresql ) allVersions; versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; @@ -128,10 +129,54 @@ let packages = builtins.attrValues ( lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); + buildUnsupported = + # Build SQL-only packages for unsupported versions needed by pg_upgrade. + # When upgrading PostgreSQL, pg_upgrade requires old extension versions to exist + # even if they can't compile against the new PostgreSQL version. + version: hash: _rustVersion: _pgrxVersion: + stdenv.mkDerivation { + inherit pname version; + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "v${version}"; + inherit hash; + }; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/share/postgresql/extension + for file in $src/sql/*.sql; do + filename=$(basename "$file") + if [[ "$filename" != "load_sql_config.sql" && "$filename" != "load_sql_context.sql" ]]; then + cat "$file" + echo ";" + fi + done > $out/share/postgresql/extension/${pname}--${version}.sql + ''; + meta = with lib; { + description = "GraphQL support for PostreSQL"; + homepage = "https://github.com/supabase/${pname}"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; + unsupportedVersions = lib.filterAttrs ( + _: value: !builtins.elem pgVersion value.postgresql + ) allVersions; + unsupportedPackages = + if pgVersion == 15 then + [ ] + else + # Include SQL-only packages for PG15 extension versions incompatible with current PG + builtins.attrValues ( + lib.mapAttrs ( + name: value: buildUnsupported name value.hash value.rust value.pgrx + ) unsupportedVersions + ); in (buildEnv { name = pname; - paths = packages; + paths = packages ++ unsupportedPackages; pathsToLink = [ "/lib" "/share/postgresql/extension" From af32c947710fc68a5598e91c358ec4724595e438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 6 Nov 2025 23:03:52 +0100 Subject: [PATCH 3/4] fix(pg_jsonschema): include unsupported versions as SQL-only for pg_upgrade compatibility When upgrading PostgreSQL versions, pg_upgrade needs access to old extension versions sql to migrate from. This adds unsupported pg_graphql versions (those not compilable with current PostgreSQL) as SQL-only packages, ensuring migration paths exist when upgrading from older PostgreSQL versions. --- nix/ext/pg_jsonschema/default.nix | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 1a2e8ee58..be34a86a5 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -49,16 +49,6 @@ let # update the following array when the pg_jsonschema version is updated # required to ensure that extensions update scripts from previous versions are generated - previousVersions = [ - "0.3.1" - "0.3.0" - "0.2.0" - "0.1.4" - "0.1.3" - "0.1.2" - "0.1.1" - "0.1.0" - ]; CARGO = "${cargo}/bin/cargo"; #darwin env needs PGPORT to be unique for build to not clash with other pgrx extensions env = lib.optionalAttrs stdenv.isDarwin { @@ -79,19 +69,15 @@ let preCheck = '' export PGRX_HOME=$(mktemp -d) - export NIX_PGLIBDIR=$PGRX_HOME/${lib.versions.major postgresql.version}/lib - ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${lib.versions.major postgresql.version}/ - cargo pgrx init --pg${lib.versions.major postgresql.version} $PGRX_HOME/${lib.versions.major postgresql.version}/bin/pg_config + export NIX_PGLIBDIR=$PGRX_HOME/${pgVersion}/lib + ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${pgVersion}/ + cargo pgrx init --pg${pgVersion} $PGRX_HOME/${pgVersion}/bin/pg_config ''; doCheck = true; - preBuild = '' - echo "Processing git tags..." - echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt - ''; - postInstall = '' + find $out mv $out/lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix} create_control_files() { @@ -120,8 +106,9 @@ let }; }; allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_jsonschema; + pgVersion = lib.versions.major postgresql.version; supportedVersions = lib.filterAttrs ( - _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + _: value: builtins.elem pgVersion value.postgresql ) allVersions; versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; @@ -129,6 +116,8 @@ let packages = builtins.attrValues ( lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); + unsupportedVersionsItems = lib.filterAttrs (_: value: value.postgresql == [ "15" ]) allVersions; + unsupportedVersions = if pgVersion == "17" then lib.attrNames unsupportedVersionsItems else [ ]; in (pkgs.buildEnv { name = pname; @@ -145,6 +134,10 @@ in }" ) + for v in ${lib.concatStringsSep " " unsupportedVersions}; do + cp $out/share/postgresql/extension/${pname}--${lib.head versions}.sql $out/share/postgresql/extension/${pname}--$v.sql + done + create_sql_files() { PREVIOUS_VERSION="" while IFS= read -r i; do From d36e133e4da2d797cf0a07117eef4b09c0016e3d Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Fri, 21 Nov 2025 15:41:19 +0100 Subject: [PATCH 4/4] fix(pg_stat_monitor): exclude v1.0 from pg_upgrade compatibility tests When upgrading from PostgreSQL 15 to 17, pg_stat_monitor version 1.0 (PG 15-only) cannot be migrated as it uses .sql.in template files that reference MODULE_PATHNAME without proper processing for the target version. This marks version 1.0 as not pg_upgrade compatible and filters it from the version test list, allowing the test to use version 2.1 (which supports both PG 15 and 17) for pg_upgrade validation instead. Version 1.0 remains available for PG 15 installations. Version 2.1 has different schemas on PostgreSQL 15 vs 17 despite sharing the same version number. On PG 15 it uses the older schema with blk_read_time and blk_write_time columns, while on PG 17 it uses a newer schema with shared_blk_read_time, shared_blk_write_time, local_blk_read_time, local_blk_write_time and additional JIT statistics columns. During pg_upgrade from PG 15 to 17, the extension version remains 2.1 without schema migration since no update script is generated. Fresh installations on PG 17 receive the new schema while pg_upgrade retains the old schema, creating a test conflict as both scenarios share the same expected output file. A custom test implementation skips pg_regress validation after pg_upgrade when no update script is generated, since the schema mismatch is expected behavior. This maintains full test coverage for fresh installations through the regular psql_17 check while validating extension version compatibility for pg_upgrade scenarios. --- nix/ext/pg_stat_monitor.nix | 11 +- nix/ext/tests/default.nix | 1 - nix/ext/tests/pg_stat_monitor.nix | 262 ++++++++++++++++++++++++++++++ nix/ext/versions.json | 3 +- 4 files changed, 272 insertions(+), 5 deletions(-) create mode 100644 nix/ext/tests/pg_stat_monitor.nix diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index ddf46de30..eaedcb991 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -17,11 +17,16 @@ let ) allVersions; # Derived version information - versions = lib.naturalSort (lib.attrNames supportedVersions); - latestVersion = lib.last versions; + allVersionsList = lib.naturalSort (lib.attrNames supportedVersions); + versions = builtins.filter (v: (allVersions.${v}.pgUpgradeCompatible or true)) allVersionsList; + latestVersion = lib.last allVersionsList; numberOfVersions = builtins.length versions; + # Filter to only build pg_upgrade compatible versions + pgUpgradeCompatibleVersions = lib.filterAttrs ( + name: _: allVersions.${name}.pgUpgradeCompatible or true + ) supportedVersions; packages = builtins.attrValues ( - lib.mapAttrs (name: value: build name value.hash value.revision) supportedVersions + lib.mapAttrs (name: value: build name value.hash value.revision) pgUpgradeCompatibleVersions ); # Build function for individual versions diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index 788abfedc..7f32612b0 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -254,7 +254,6 @@ builtins.listToAttrs ( "pg_jsonschema" "pg_net" "pg_partman" - "pg_stat_monitor" "pg_tle" "pgaudit" "postgis" diff --git a/nix/ext/tests/pg_stat_monitor.nix b/nix/ext/tests/pg_stat_monitor.nix new file mode 100644 index 000000000..71e3f731a --- /dev/null +++ b/nix/ext/tests/pg_stat_monitor.nix @@ -0,0 +1,262 @@ +{ self, pkgs }: +# +# Custom test for pg_stat_monitor extension +# +# IMPORTANT: This extension requires special handling because version 2.1 has different +# schemas on PostgreSQL 15 vs 17, even though they share the same version number: +# +# - PG 15 (version 2.1): Uses older schema with `blk_read_time`, `blk_write_time` +# - PG 17 (version 2.1): Uses newer schema with `shared_blk_read_time`, +# `shared_blk_write_time`, `local_blk_read_time`, `local_blk_write_time`, plus +# additional JIT columns (`jit_deform_count`, `jit_deform_time`, `stats_since`, +# `minmax_stats_since`) +# +# During pg_upgrade from PG 15 to PG 17: +# - The extension remains at version 2.1 +# - No update script is generated (same version number) +# - The schema stays as PG 15 (old schema) +# - Fresh installs on PG 17 get the new schema +# +# This creates a conflict: the same expected output file (`z_17_pg_stat_monitor.out`) +# is used for both scenarios, but they produce different schemas. +# +# Solution: Skip pg_regress after pg_upgrade when no update script is generated, +# since the schema won't match the expected output for fresh PG 17 installs. +# This still validates that: +# - The extension version is correct after upgrade +# - Fresh installs work correctly (tested by regular psql_17 check) +# - Upgrade paths work correctly +# +let + pname = "pg_stat_monitor"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; + psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = psql_15; + enableTCPIP = true; + authentication = '' + local all postgres peer map=postgres + local all all peer map=root + ''; + identMap = '' + root root supabase_admin + postgres postgres postgres + ''; + ensureUsers = [ + { + name = "supabase_admin"; + ensureClauses.superuser = true; + } + ]; + settings = (installedExtension "15").defaultSettings or { }; + }; + + networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ]; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce psql_17; + settings = (installedExtension "17").defaultSettings or { }; + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = psql_15; + newPostgresql = psql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" \ + ${ + if config.services.postgresql.settings.shared_preload_libraries != null then + " --old-options='-c shared_preload_libraries=${config.services.postgresql.settings.shared_preload_libraries}' --new-options='-c shared_preload_libraries=${config.services.postgresql.settings.shared_preload_libraries}'" + else + "" + } + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + '' + from pathlib import Path + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + extension_name = "${pname}" + support_upgrade = True + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" + ext_has_background_worker = ${ + if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" + } + sql_test_directory = Path("${../../tests}") + pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}" + ext_schema = "${(installedExtension "15").defaultSchema or "public"}" + lib_name = "${(installedExtension "15").libName or pname}" + print(f"Running tests for extension: {lib_name}") + + ${builtins.readFile ./lib.py} + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade, ext_schema) + test.create_schema() + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") + + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + if ext_has_background_worker: + with subtest("Test switch_${pname}_version"): + test.check_switch_extension_with_background_worker(Path(f"${psql_15}/lib/{lib_name}.so"), "15") + + with subtest("Check pg_regress with postgresql 15 after installing the last version"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + with subtest("switch to postgresql 17"): + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check last version of the extension after postgresql upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Check the install of the last version of the extension"): + test.check_install_last_version("17") + + with subtest("Check pg_regress with postgresql 17 after installing the last version"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + # With update script, the schema should match PG 17 expectations + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + # Skip pg_regress when no update script is generated because: + # - The extension retains the PG 15 schema (old column names) + # - The expected output file expects PG 17 schema (new column names) + # - This mismatch is expected behavior - pg_upgrade doesn't change schemas + # when version numbers don't change + # - The extension is still functional, just with the older schema + print(f"Skipping pg_regress for {extension_name} after pg_upgrade without update script") + print(f"Version {version} retains PG 15 schema, which differs from PG 17 fresh install schema") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index b35c6c40b..e83366a0c 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -775,7 +775,8 @@ "15" ], "revision": "1.0.1", - "hash": "sha256-sQEpIknAFOmvNTX2G23X4BvMdy3Ms7sXx7hLZt8jyUk=" + "hash": "sha256-sQEpIknAFOmvNTX2G23X4BvMdy3Ms7sXx7hLZt8jyUk=", + "pgUpgradeCompatible": false }, "2.1": { "postgresql": [