Try again to add test coverage for pg_combinebackup w/tablespaces.
authorRobert Haas <rhaas@postgresql.org>
Tue, 23 Apr 2024 20:33:19 +0000 (16:33 -0400)
committerRobert Haas <rhaas@postgresql.org>
Tue, 23 Apr 2024 20:33:19 +0000 (16:33 -0400)
My previous attempt to add this had to be reverted in commit
82023d47de9e262730b1f9b4ea77fae201a89d0a. I've revised the problematic
code a bit; hopefully it is OK now.

Discussion: http://postgr.es/m/CA+Tgmobiv1QJR5PEJoDKeZDrJHZFRmi4XmWOqufN49DJj-3e2g@mail.gmail.com

src/bin/pg_basebackup/t/010_pg_basebackup.pl
src/bin/pg_combinebackup/t/002_compare_backups.pl
src/test/perl/PostgreSQL/Test/Cluster.pm

index 63f7bd2735ac6d09210101dfaaa1f6e09bc699c4..31c369688e03f6d5c91361e1157b10328983b024 100644 (file)
@@ -407,25 +407,12 @@ SKIP:
 
        my $node2 = PostgreSQL::Test::Cluster->new('replica');
 
-       # Recover main data directory
-       $node2->init_from_backup($node, 'tarbackup2', tar_program => $tar);
-
-       # Recover tablespace into a new directory (not where it was!)
-       my $repTsDir = "$tempdir/tblspc1replica";
-       my $realRepTsDir = "$real_sys_tempdir/tblspc1replica";
-       mkdir $repTsDir;
-       PostgreSQL::Test::Utils::system_or_bail($tar, 'xf', $tblspc_tars[0],
-               '-C', $repTsDir);
-
-       # Update tablespace map to point to new directory.
-       # XXX Ideally pg_basebackup would handle this.
+       # Recover the backup
        $tblspc_tars[0] =~ m|/([0-9]*)\.tar$|;
        my $tblspcoid = $1;
-       my $escapedRepTsDir = $realRepTsDir;
-       $escapedRepTsDir =~ s/\\/\\\\/g;
-       open my $mapfile, '>', $node2->data_dir . '/tablespace_map' or die $!;
-       print $mapfile "$tblspcoid $escapedRepTsDir\n";
-       close $mapfile;
+       my $realRepTsDir = "$real_sys_tempdir/tblspc1replica";
+       $node2->init_from_backup($node, 'tarbackup2', tar_program => $tar,
+               'tablespace_map' => { $tblspcoid => $realRepTsDir });
 
        $node2->start;
        my $result = $node2->safe_psql('postgres', 'SELECT * FROM test1');
index 71e9a90d22609c61898b15289d439e7054148103..31342579cf0c710b7e10fc5acc09f673804bb945 100644 (file)
@@ -7,11 +7,15 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+
 # Set up a new database instance.
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(has_archiving => 1, allows_streaming => 1);
 $primary->append_conf('postgresql.conf', 'summarize_wal = on');
 $primary->start;
+my $tsprimary = $tempdir . '/ts';
+mkdir($tsprimary) || die "mkdir $tsprimary: $!";
 
 # Create some test tables, each containing one row of data, plus a whole
 # extra database.
@@ -29,24 +33,42 @@ INSERT INTO will_get_dropped VALUES (1, 'initial test row');
 CREATE TABLE will_get_rewritten (a int, b text);
 INSERT INTO will_get_rewritten VALUES (1, 'initial test row');
 CREATE DATABASE db_will_get_dropped;
+CREATE TABLESPACE ts1 LOCATION '$tsprimary';
+CREATE TABLE will_not_change_in_ts (a int, b text) TABLESPACE ts1;
+INSERT INTO will_not_change_in_ts VALUES (1, 'initial test row');
+CREATE TABLE will_change_in_ts (a int, b text) TABLESPACE ts1;
+INSERT INTO will_change_in_ts VALUES (1, 'initial test row');
+CREATE TABLE will_get_dropped_in_ts (a int, b text);
+INSERT INTO will_get_dropped_in_ts VALUES (1, 'initial test row');
 EOM
 
+# Read list of tablespace OIDs. There should be just one.
+my @tsoids = grep { /^\d+/ } slurp_dir($primary->data_dir . '/pg_tblspc');
+is(0+@tsoids, 1, "exactly one user-defined tablespace");
+my $tsoid = $tsoids[0];
+
 # Take a full backup.
 my $backup1path = $primary->backup_dir . '/backup1';
+my $tsbackup1path = $tempdir . '/ts1backup';
+mkdir($tsbackup1path) || die "mkdir $tsbackup1path: $!";
 $primary->command_ok(
-       [ 'pg_basebackup', '-D', $backup1path, '--no-sync', '-cfast' ],
-       "full backup");
+       [ 'pg_basebackup', '-D', $backup1path, '--no-sync', '-cfast',
+      "-T${tsprimary}=${tsbackup1path}" ], "full backup");
 
 # Now make some database changes.
 $primary->safe_psql('postgres', <<EOM);
 UPDATE will_change SET b = 'modified value' WHERE a = 1;
+UPDATE will_change_in_ts SET b = 'modified value' WHERE a = 1;
 INSERT INTO will_grow
        SELECT g, 'additional row' FROM generate_series(2, 5000) g;
 TRUNCATE will_shrink;
 VACUUM will_get_vacuumed;
 DROP TABLE will_get_dropped;
+DROP TABLE will_get_dropped_in_ts;
 CREATE TABLE newly_created (a int, b text);
 INSERT INTO newly_created VALUES (1, 'row for new table');
+CREATE TABLE newly_created_in_ts (a int, b text) TABLESPACE ts1;
+INSERT INTO newly_created_in_ts VALUES (1, 'row for new table');
 VACUUM FULL will_get_rewritten;
 DROP DATABASE db_will_get_dropped;
 CREATE DATABASE db_newly_created;
@@ -54,8 +76,11 @@ EOM
 
 # Take an incremental backup.
 my $backup2path = $primary->backup_dir . '/backup2';
+my $tsbackup2path = $tempdir . '/tsbackup2';
+mkdir($tsbackup2path) || die "mkdir $tsbackup2path: $!";
 $primary->command_ok(
        [ 'pg_basebackup', '-D', $backup2path, '--no-sync', '-cfast',
+      "-T${tsprimary}=${tsbackup2path}",
          '--incremental', $backup1path . '/backup_manifest' ],
        "incremental backup");
 
@@ -78,9 +103,11 @@ $primary->poll_query_until('postgres', $archive_wait_query)
 # Perform PITR from the full backup. Disable archive_mode so that the archive
 # doesn't find out about the new timeline; that way, the later PITR below will
 # choose the same timeline.
+my $tspitr1path = $tempdir . '/tspitr1';
 my $pitr1 = PostgreSQL::Test::Cluster->new('pitr1');
 $pitr1->init_from_backup($primary, 'backup1',
-                                                standby => 1, has_restoring => 1);
+                                                standby => 1, has_restoring => 1,
+                                                tablespace_map => { $tsoid => $tspitr1path });
 $pitr1->append_conf('postgresql.conf', qq{
 recovery_target_lsn = '$lsn'
 recovery_target_action = 'promote'
@@ -90,10 +117,12 @@ $pitr1->start();
 
 # Perform PITR to the same LSN from the incremental backup. Use the same
 # basic configuration as before.
+my $tspitr2path = $tempdir . '/tspitr2';
 my $pitr2 = PostgreSQL::Test::Cluster->new('pitr2');
 $pitr2->init_from_backup($primary, 'backup2',
                                                 standby => 1, has_restoring => 1,
-                                                combine_with_prior => [ 'backup1' ]);
+                                                combine_with_prior => [ 'backup1' ],
+                                                tablespace_map => { $tsbackup2path => $tspitr2path });
 $pitr2->append_conf('postgresql.conf', qq{
 recovery_target_lsn = '$lsn'
 recovery_target_action = 'promote'
index 9b2879c145c29426b80bb5bcca8ef1ad756d3bd6..aa9646329fb4c4370b46c32c6281bea39cd120f5 100644 (file)
@@ -777,7 +777,7 @@ sub backup_fs_cold
 
 =pod
 
-=item $node->init_from_backup(root_node, backup_name)
+=item $node->init_from_backup(root_node, backup_name, %params)
 
 Initialize a node from a backup, which may come from this node or a different
 node. root_node must be a PostgreSQL::Test::Cluster reference, backup_name the string name
@@ -787,8 +787,13 @@ Does not start the node after initializing it.
 
 By default, the backup is assumed to be plain format.  To restore from
 a tar-format backup, pass the name of the tar program to use in the
-keyword parameter tar_program.  Note that tablespace tar files aren't
-handled here.
+keyword parameter tar_program.
+
+If there are tablespace present in the backup, include tablespace_map as
+a keyword parameter whose values is a hash. When combine_with_prior is used,
+the hash keys are the tablespace pathnames used in the backup; otherwise,
+they are tablespace OIDs.  In either case, the values are the tablespace
+pathnames that should be used for the target cluster.
 
 To restore from an incremental backup, pass the parameter combine_with_prior
 as a reference to an array of prior backup names with which this backup
@@ -843,12 +848,20 @@ sub init_from_backup
                }
 
                local %ENV = $self->_get_env();
-               PostgreSQL::Test::Utils::system_or_bail('pg_combinebackup', '-d',
-                       @prior_backup_path, $backup_path, '-o', $data_path);
+               my @combineargs = ('pg_combinebackup', '-d');
+               if (exists $params{tablespace_map})
+               {
+                       while (my ($olddir, $newdir) = each %{$params{tablespace_map}})
+                       {
+                               push @combineargs, "-T$olddir=$newdir";
+                       }
+               }
+               push @combineargs, @prior_backup_path, $backup_path, '-o', $data_path;
+               PostgreSQL::Test::Utils::system_or_bail(@combineargs);
        }
        elsif (defined $params{tar_program})
        {
-               mkdir($data_path);
+               mkdir($data_path) || die "mkdir $data_path: $!";
                PostgreSQL::Test::Utils::system_or_bail($params{tar_program}, 'xf',
                        $backup_path . '/base.tar',
                        '-C', $data_path);
@@ -856,11 +869,80 @@ sub init_from_backup
                        $params{tar_program}, 'xf',
                        $backup_path . '/pg_wal.tar', '-C',
                        $data_path . '/pg_wal');
+
+               # We need to generate a tablespace_map file.
+               open(my $tsmap, ">", "$data_path/tablespace_map")
+                       || die "$data_path/tablespace_map: $!";
+
+               # Extract tarfiles and add tablespace_map entries
+               my @tstars = grep { /^\d+.tar/ }
+                       PostgreSQL::Test::Utils::slurp_dir($backup_path);
+               for my $tstar (@tstars)
+               {
+                       my $tsoid = $tstar;
+                       $tsoid =~ s/\.tar$//;
+
+                       die "no tablespace mapping for $tstar"
+                               if !exists $params{tablespace_map} ||
+                                  !exists $params{tablespace_map}{$tsoid};
+                       my $newdir = $params{tablespace_map}{$tsoid};
+
+                       mkdir($newdir) || die "mkdir $newdir: $!";
+                       PostgreSQL::Test::Utils::system_or_bail($params{tar_program}, 'xf',
+                               $backup_path . '/' . $tstar, '-C', $newdir);
+
+                       my $escaped_newdir = $newdir;
+                       $escaped_newdir =~ s/\\/\\\\/g;
+                       print $tsmap "$tsoid $escaped_newdir\n";
+               }
+
+               # Close tablespace_map.
+               close($tsmap);
        }
        else
        {
+               my @tsoids;
                rmdir($data_path);
-               PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+
+               # Copy the main backup. If we see a tablespace directory for which we
+               # have a tablespace mapping, skip it, but remember that we saw it.
+               PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path,
+                       'filterfn' => sub {
+                               my ($path) = @_;
+                               if ($path =~ /^pg_tblspc\/(\d+)$/ &&
+                                       exists $params{tablespace_map}{$1})
+                               {
+                                       push @tsoids, $1;
+                                       return 0;
+                               }
+                               return 1;
+                       });
+
+               if (@tsoids > 0)
+               {
+                       # We need to generate a tablespace_map file.
+                       open(my $tsmap, ">", "$data_path/tablespace_map")
+                               || die "$data_path/tablespace_map: $!";
+
+                       # Now use the list of tablespace links to copy each tablespace.
+                       for my $tsoid (@tsoids)
+                       {
+                               die "no tablespace mapping for $tsoid"
+                                       if !exists $params{tablespace_map} ||
+                                          !exists $params{tablespace_map}{$tsoid};
+
+                               my $olddir = $backup_path . '/pg_tblspc/' . $tsoid;
+                               my $newdir = $params{tablespace_map}{$tsoid};
+                               PostgreSQL::Test::RecursiveCopy::copypath($olddir, $newdir);
+
+                               my $escaped_newdir = $newdir;
+                               $escaped_newdir =~ s/\\/\\\\/g;
+                               print $tsmap "$tsoid $escaped_newdir\n";
+                       }
+
+                       # Close tablespace_map.
+                       close($tsmap);
+               }
        }
        chmod(0700, $data_path) or die $!;