ci: Test against further MySQL and Postgres versions

ci/nightly/pipeline.template.yml

@@ -435,7 +435,7 @@ steps:
 
       - id: testdrive-alloydb
         label: "testdrive with AlloyDB"
-        depends_on: build-aarch64
+        depends_on: build-x86_64
         timeout_in_minutes: 60
         parallelism: 2
         inputs: [test/testdrive]

ci/release-qualification/pipeline.template.yml

@@ -279,35 +279,152 @@ steps:
   - group: "MySQL: other versions"
     key: mysql-versions
     steps:
-      - id: mysql-cdc-5_7
-        label: "MySQL CDC w/ 5.7"
-        depends_on: build-x86_64
+      - id: mysql-cdc-8_0
+        label: "MySQL CDC w/ 8.0"
+        depends_on: build-aarch64
         timeout_in_minutes: 60
         agents:
-          # no matching manifest of MySQL 5.7.x for linux/arm64/v8 in the manifest list entries
-          # Increased memory usage following new source syntax
-          queue: hetzner-x86-64-12cpu-24gb
+          queue: hetzner-aarch64-4cpu-8gb
         plugins:
           - ./ci/plugins/mzcompose:
               composition: mysql-cdc
-              args: [ "--mysql-version=5.7.44" ]
-        skip: "Fails when restarted"
+              args: [ "--mysql-version=8.0.42" ]
 
-      - id: mysql-cdc-8_0
-        label: "MySQL CDC w/ 8.0"
+      - id: mysql-cdc-8_1
+        label: "MySQL CDC w/ 8.1"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=8.1.0" ]
+
+      - id: mysql-cdc-8_2
+        label: "MySQL CDC w/ 8.2"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=8.2.0" ]
+
+      - id: mysql-cdc-8_3
+        label: "MySQL CDC w/ 8.3"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=8.3.0" ]
+
+      - id: mysql-cdc-8_4
+        label: "MySQL CDC w/ 8.4"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=8.4.9" ]
+
+      - id: mysql-cdc-9_0
+        label: "MySQL CDC w/ 9.0"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=9.0.1" ]
+
+      - id: mysql-cdc-9_1
+        label: "MySQL CDC w/ 9.1"
         depends_on: build-aarch64
         timeout_in_minutes: 60
         agents:
           queue: hetzner-aarch64-4cpu-8gb
         plugins:
           - ./ci/plugins/mzcompose:
               composition: mysql-cdc
-              args: [ "--mysql-version=8.0.40" ]
-        skip: "Doesn't support new RESET BINARY LOGS AND GTIDS syntax"
+              args: [ "--mysql-version=9.1.0" ]
+
+      - id: mysql-cdc-9_2
+        label: "MySQL CDC w/ 9.2"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=9.2.0" ]
+
+      - id: mysql-cdc-9_3
+        label: "MySQL CDC w/ 9.3"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=9.3.0" ]
+
+      - id: mysql-cdc-9_4
+        label: "MySQL CDC w/ 9.4"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=9.4.0" ]
+
+      - id: mysql-cdc-9_5
+        label: "MySQL CDC w/ 9.5"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=9.5.0" ]
+
+      - id: mysql-cdc-9_6
+        label: "MySQL CDC w/ 9.6"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: mysql-cdc
+              args: [ "--mysql-version=9.6.0" ]
 
   - group: "Postgres: other versions"
     key: postgres-versions
     steps:
+      - id: pg-cdc-17
+        label: "Postgres CDC w/ 17"
+        depends_on: build-aarch64
+        timeout_in_minutes: 60
+        inputs: [test/pg-cdc]
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: pg-cdc
+              args: [ "--pg-version=17.10" ]
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
       - id: pg-cdc-16
         label: "Postgres CDC w/ 16"
         depends_on: build-aarch64
@@ -316,7 +433,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: pg-cdc
-              args: [ "--pg-version=16.6" ]
+              args: [ "--pg-version=16.14" ]
         agents:
           queue: hetzner-aarch64-4cpu-8gb
       - id: pg-cdc-15
@@ -327,7 +444,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: pg-cdc
-              args: [ "--pg-version=15.10" ]
+              args: [ "--pg-version=15.18" ]
         agents:
           queue: hetzner-aarch64-4cpu-8gb
       - id: pg-cdc-14
@@ -338,7 +455,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: pg-cdc
-              args: [ "--pg-version=14.15" ]
+              args: [ "--pg-version=14.23" ]
         agents:
           queue: hetzner-aarch64-4cpu-8gb
       - id: pg-cdc-13

misc/python/materialize/cloudtest/k8s/mysql.py

@@ -59,7 +59,7 @@ def __init__(self, namespace: str, apply_node_selectors: bool) -> None:
         ports = [V1ContainerPort(container_port=3306, name="sql")]
         container = V1Container(
             name="mysql",
-            image=self.image("mysql", tag="9.4.0", release_mode=True, org=None),
+            image=self.image("mysql", tag="9.7.0", release_mode=True, org=None),
             args=[
                 "--log-bin=mysql-bin",
                 "--gtid_mode=ON",

misc/python/materialize/mzcompose/services/mysql.py

@@ -37,7 +37,7 @@ def create_mysql_server_args(
 
 class MySql(Service):
     DEFAULT_ROOT_PASSWORD = "p@ssw0rd"
-    DEFAULT_VERSION = "9.5.0"
+    DEFAULT_VERSION = "9.7.0"
 
     DEFAULT_ADDITIONAL_ARGS = create_mysql_server_args(server_id="1", is_master=True)
 

test/mysql-cdc-old-syntax/mzcompose.py

@@ -89,6 +89,13 @@ def get_targeted_mysql_version(parser: WorkflowArgumentParser) -> str:
     return args.mysql_version
 
 
+def reset_binlog_stmt(mysql_version: str) -> str:
+    # RESET BINARY LOGS AND GTIDS was introduced in MySQL 8.2;
+    # RESET MASTER was removed in MySQL 8.4.
+    major_minor = tuple(int(p) for p in mysql_version.split(".")[:2])
+    return "RESET MASTER" if major_minor < (8, 2) else "RESET BINARY LOGS AND GTIDS"
+
+
 def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     def process(name: str) -> None:
         if name in ("default", "migration"):
@@ -160,6 +167,7 @@ def workflow_replica_connection(c: Composition, parser: WorkflowArgumentParser)
         c.up("materialized", "mysql", "mysql-replica")
         c.run_testdrive_files(
             f"--var=mysql-root-password={MySql.DEFAULT_ROOT_PASSWORD}",
+            f"--var=reset-binlog={reset_binlog_stmt(mysql_version)}",
             "override/10-replica-connection.td",
         )
 

test/mysql-cdc-old-syntax/override/10-replica-connection.td

@@ -22,8 +22,7 @@ $ mysql-connect name=mysql-replica url=mysql://root@mysql-replica password=${arg
 # state when we create our source.
 #
 $ mysql-execute name=mysql
-DROP DATABASE IF EXISTS public;
-RESET BINARY LOGS AND GTIDS;
+DROP DATABASE IF EXISTS public; ${arg.reset-binlog};
 CREATE DATABASE public;
 USE public;
 CREATE TABLE t1 (f1 INTEGER);

test/mysql-cdc-old-syntax/toast-columns.td

@@ -23,8 +23,9 @@ $ set-sql-timeout duration=1s
 
 $ mysql-connect name=mysql url=mysql://root@mysql password=${arg.mysql-root-password}
 
-# Insert data pre-snapshot by generating 16kB of uncompressible data to force
-# TOASTed storage by concatenating 1024 random MD5 hashes, each being 128bit
+# Insert data pre-snapshot by generating 32kB of uncompressible data to force
+# TOASTed storage by concatenating 512 random SHA-256 hashes, each being 256 bit.
+# Avoid md5()/sha1() which were moved to the classic_hashing component in MySQL 9.6.
 $ mysql-execute name=mysql
 DROP DATABASE IF EXISTS public;
 CREATE DATABASE public;
@@ -36,13 +37,13 @@ SET SESSION group_concat_max_len = 32768;
 
 # necessary because the limit is not respected by group_concat
 CREATE TABLE temp_rand (rand_value TEXT);
-INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024;
+INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512;
 
-INSERT INTO t1 SELECT 1, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand;
+INSERT INTO t1 SELECT 1, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand;
 
 DELETE FROM temp_rand;
-INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024;
-INSERT INTO t1 SELECT 2, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand;
+INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512;
+INSERT INTO t1 SELECT 2, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand;
 
 DROP TABLE temp_rand;
 

test/mysql-cdc/mzcompose.py

@@ -93,6 +93,13 @@ def get_targeted_mysql_version(parser: WorkflowArgumentParser) -> str:
     return args.mysql_version
 
 
+def reset_binlog_stmt(mysql_version: str) -> str:
+    # RESET BINARY LOGS AND GTIDS was introduced in MySQL 8.2;
+    # RESET MASTER was removed in MySQL 8.4.
+    major_minor = tuple(int(p) for p in mysql_version.split(".")[:2])
+    return "RESET MASTER" if major_minor < (8, 2) else "RESET BINARY LOGS AND GTIDS"
+
+
 def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     def process(name: str) -> None:
         if name in ("default", "large-scale"):
@@ -164,6 +171,7 @@ def workflow_replica_connection(c: Composition, parser: WorkflowArgumentParser)
         c.up("materialized", "mysql", "mysql-replica")
         c.run_testdrive_files(
             f"--var=mysql-root-password={MySql.DEFAULT_ROOT_PASSWORD}",
+            f"--var=reset-binlog={reset_binlog_stmt(mysql_version)}",
             "override/10-replica-connection.td",
         )
 

test/mysql-cdc/override/10-replica-connection.td

@@ -22,8 +22,7 @@ $ mysql-connect name=mysql-replica url=mysql://root@mysql-replica password=${arg
 # state when we create our source.
 #
 $ mysql-execute name=mysql
-DROP DATABASE IF EXISTS public;
-RESET BINARY LOGS AND GTIDS;
+DROP DATABASE IF EXISTS public; ${arg.reset-binlog};
 CREATE DATABASE public;
 USE public;
 CREATE TABLE t1 (f1 INTEGER);

test/mysql-cdc/toast-columns.td

@@ -23,8 +23,9 @@ $ set-sql-timeout duration=1s
 
 $ mysql-connect name=mysql url=mysql://root@mysql password=${arg.mysql-root-password}
 
-# Insert data pre-snapshot by generating 16kB of uncompressible data to force
-# TOASTed storage by concatenating 1024 random MD5 hashes, each being 128bit
+# Insert data pre-snapshot by generating 32kB of uncompressible data to force
+# TOASTed storage by concatenating 512 random SHA-256 hashes, each being 256 bit.
+# Avoid md5()/sha1() which were moved to the classic_hashing component in MySQL 9.6.
 $ mysql-execute name=mysql
 DROP DATABASE IF EXISTS public;
 CREATE DATABASE public;
@@ -36,13 +37,13 @@ SET SESSION group_concat_max_len = 32768;
 
 # necessary because the limit is not respected by group_concat
 CREATE TABLE temp_rand (rand_value TEXT);
-INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024;
+INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512;
 
-INSERT INTO t1 SELECT 1, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand;
+INSERT INTO t1 SELECT 1, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand;
 
 DELETE FROM temp_rand;
-INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024;
-INSERT INTO t1 SELECT 2, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand;
+INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512;
+INSERT INTO t1 SELECT 2, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand;
 
 DROP TABLE temp_rand;
 

test/mysql/Dockerfile

@@ -9,7 +9,7 @@
 
 MZFROM test-certs AS certs
 
-FROM mysql:9.5.0 AS seed
+FROM mysql:9.7.0 AS seed
 
 ENV MYSQL_ROOT_HOST=%
 
@@ -20,7 +20,7 @@ RUN chmod +x /usr/local/bin/seed-mysql-datadir.sh \
     && chown -R mysql:mysql /var/lib/mysql-seed /var/lib/mysql-files \
     && gosu mysql /usr/local/bin/seed-mysql-datadir.sh
 
-FROM mysql:9.5.0
+FROM mysql:9.7.0
 
 COPY --from=seed /var/lib/mysql-seed /var/lib/mysql-seed
 COPY mysql-entrypoint.sh /usr/local/bin/mysql-entrypoint.sh

test/pg-cdc/pg-cdc.td

@@ -38,6 +38,7 @@ CREATE SCHEMA public;
 
 DROP PUBLICATION IF EXISTS mz_source;
 CREATE PUBLICATION mz_source FOR ALL TABLES;
+DO $do$ BEGIN IF current_setting('server_version_num')::int >= 180000 THEN EXECUTE 'ALTER PUBLICATION mz_source SET (publish_generated_columns = ''stored'')'; END IF; END $do$;
 
 CREATE TABLE pk_table (pk INTEGER PRIMARY KEY, f2 TEXT);
 INSERT INTO pk_table VALUES (1, 'one');

test/postgres/Dockerfile

@@ -9,11 +9,12 @@
 
 MZFROM test-certs as certs
 
-FROM postgres:17.7
+FROM postgres:18.4
 
 ENV POSTGRES_PASSWORD=postgres
+ENV PGDATA=/var/lib/postgresql/data
 
-RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql-17-cron eatmydata \
+RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql-18-cron eatmydata \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* /usr/share/locale/* /var/cache/* /var/log/*