<term><literal>PostgreSQL-Backup-Manifest-Version</literal></term>
<listitem>
<para>
- The associated value is always the integer 1.
+ The associated value is an integer. Beginning in
+ <productname>PostgreSQL</productname> <literal>17</literal>,
+ it is <literal>2</literal>; in older versions, it is <literal>1</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><literal>System-Identifier</literal></term>
+ <listitem>
+ <para>
+ The database system identifier of the
+ <productname>PostgreSQL</productname> instance where the backup was
+ taken. This field is present only when
+ <literal>PostgreSQL-Backup-Manifest-Version</literal> is
+ <literal>2</literal>.
</para>
</listitem>
</varlistentry>
Backup verification proceeds in four stages. First,
<literal>pg_verifybackup</literal> reads the
<literal>backup_manifest</literal> file. If that file
- does not exist, cannot be read, is malformed, or fails verification
- against its own internal checksum, <literal>pg_verifybackup</literal>
- will terminate with a fatal error.
+ does not exist, cannot be read, is malformed, fails to match the system
+ identifier with <filename>pg_control</filename> of the backup directory or
+ fails verification against its own internal checksum,
+ <literal>pg_verifybackup</literal> will terminate with a fatal error.
</para>
<para>
#include "postgres.h"
#include "access/timeline.h"
+#include "access/xlog.h"
#include "backup/backup_manifest.h"
#include "backup/basebackup_sink.h"
#include "mb/pg_wchar.h"
if (want_manifest != MANIFEST_OPTION_NO)
AppendToManifest(manifest,
- "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
- "\"Files\": [");
+ "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
+ "\"System-Identifier\": " UINT64_FORMAT ",\n"
+ "\"Files\": [",
+ GetSystemIdentifier());
}
/*
#include "postgres.h"
#include "access/timeline.h"
+#include "access/xlog.h"
#include "backup/basebackup_incremental.h"
#include "backup/walsummary.h"
#include "common/blkreftable.h"
BlockRefTable *brtab;
};
+static void manifest_process_version(JsonManifestParseContext *context,
+ int manifest_version);
+static void manifest_process_system_identifier(JsonManifestParseContext *context,
+ uint64 manifest_system_identifier);
static void manifest_process_file(JsonManifestParseContext *context,
char *pathname,
size_t size,
/* Parse the manifest. */
context.private_data = ib;
+ context.version_cb = manifest_process_version;
+ context.system_identifier_cb = manifest_process_system_identifier;
context.per_file_cb = manifest_process_file;
context.per_wal_range_cb = manifest_process_wal_range;
context.error_cb = manifest_report_error;
return hash_bytes(ss, strlen(s));
}
+/*
+ * This callback to validate the manifest version for incremental backup.
+ */
+static void
+manifest_process_version(JsonManifestParseContext *context,
+ int manifest_version)
+{
+ /* Incremental backups don't work with manifest version 1 */
+ if (manifest_version == 1)
+ context->error_cb(context,
+ "backup manifest version 1 does not support incremental backup");
+}
+
+/*
+ * This callback to validate the manifest system identifier against the current
+ * database server.
+ */
+static void
+manifest_process_system_identifier(JsonManifestParseContext *context,
+ uint64 manifest_system_identifier)
+{
+ uint64 system_identifier;
+
+ /* Get system identifier of current system */
+ system_identifier = GetSystemIdentifier();
+
+ if (manifest_system_identifier != system_identifier)
+ context->error_cb(context,
+ "manifest system identifier is %llu, but database system identifier is %llu",
+ (unsigned long long) manifest_system_identifier,
+ (unsigned long long) system_identifier);
+}
+
/*
* This callback is invoked for each file mentioned in the backup manifest.
*
my @dst_tblspc = glob "$backupdir/pg_tblspc/$tblspc_oid/PG_*";
is(@dst_tblspc, 1, 'tblspc directory copied');
+# Can't take backup with referring manifest of different cluster
+#
+# Set up another new database instance with force initdb option. We don't want
+# to initializing database system by copying initdb template for this, because
+# we want it to be a separate cluster with a different system ID.
+my $node2 = PostgreSQL::Test::Cluster->new('node2');
+$node2->init(force_initdb => 1, has_archiving => 1, allows_streaming => 1);
+$node2->append_conf('postgresql.conf', 'summarize_wal = on');
+$node2->start;
+
+$node2->command_fails_like(
+ [ @pg_basebackup_defs, '-D', "$tempdir" . '/diff_sysid',
+ '--incremental', "$backupdir" . '/backup_manifest' ],
+ qr/manifest system identifier is .*, but database system identifier is/,
+ "pg_basebackup fails with different database system manifest");
+
done_testing();
#define SH_DEFINE
#include "lib/simplehash.h"
+static void combinebackup_version_cb(JsonManifestParseContext *context,
+ int manifest_version);
+static void combinebackup_system_identifier_cb(JsonManifestParseContext *context,
+ uint64 manifest_system_identifier);
static void combinebackup_per_file_cb(JsonManifestParseContext *context,
char *pathname, size_t size,
pg_checksum_type checksum_type,
result = pg_malloc0(sizeof(manifest_data));
result->files = ht;
context.private_data = result;
+ context.version_cb = combinebackup_version_cb;
+ context.system_identifier_cb = combinebackup_system_identifier_cb;
context.per_file_cb = combinebackup_per_file_cb;
context.per_wal_range_cb = combinebackup_per_wal_range_cb;
context.error_cb = report_manifest_error;
exit(1);
}
+/*
+ * This callback to validate the manifest version number for incremental backup.
+ */
+static void
+combinebackup_version_cb(JsonManifestParseContext *context,
+ int manifest_version)
+{
+ /* Incremental backups supported on manifest version 2 or later */
+ if (manifest_version == 1)
+ pg_fatal("backup manifest version 1 does not support incremental backup");
+}
+
+/*
+ * Record system identifier extracted from the backup manifest.
+ */
+static void
+combinebackup_system_identifier_cb(JsonManifestParseContext *context,
+ uint64 manifest_system_identifier)
+{
+ manifest_data *manifest = context->private_data;
+
+ /* Validation will be at the later stage */
+ manifest->system_identifier = manifest_system_identifier;
+}
+
/*
* Record details extracted from the backup manifest for one file.
*/
*/
typedef struct manifest_data
{
+ uint64 system_identifier;
manifest_files_hash *files;
manifest_wal_range *first_wal_range;
manifest_wal_range *last_wal_range;
static void add_tablespace_mapping(cb_options *opt, char *arg);
static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
-static void check_control_files(int n_backups, char **backup_dirs);
+static uint64 check_control_files(int n_backups, char **backup_dirs);
static void check_input_dir_permissions(char *dir);
static void cleanup_directories_atexit(void);
static void create_output_directory(char *dirname, cb_options *opt);
const char *progname;
char *last_input_dir;
+ int i;
int optindex;
int c;
int n_backups;
int n_prior_backups;
int version;
+ uint64 system_identifier;
char **prior_backup_dirs;
cb_options opt;
cb_tablespace *tablespaces;
/* Sanity-check control files. */
n_backups = argc - optind;
- check_control_files(n_backups, argv + optind);
+ system_identifier = check_control_files(n_backups, argv + optind);
/* Sanity-check backup_label files, and get the contents of the last one. */
last_backup_label = check_backup_label_files(n_backups, argv + optind);
/* Load backup manifests. */
manifests = load_backup_manifests(n_backups, prior_backup_dirs);
+ /*
+ * Validate the manifest system identifier against the backup system
+ * identifier.
+ */
+ for (i = 0; i < n_backups; i++)
+ {
+ if (manifests[i] &&
+ manifests[i]->system_identifier != system_identifier)
+ {
+ char *controlpath;
+
+ controlpath = psprintf("%s/%s", prior_backup_dirs[i], "global/pg_control");
+
+ pg_fatal("%s: manifest system identifier is %llu, but control file has %llu",
+ controlpath,
+ (unsigned long long) manifests[i]->system_identifier,
+ (unsigned long long) system_identifier);
+ }
+ }
+
/* Figure out which tablespaces are going to be included in the output. */
last_input_dir = argv[argc - 1];
check_input_dir_permissions(last_input_dir);
/* If we need to write a backup_manifest, prepare to do so. */
if (!opt.dry_run && !opt.no_manifest)
{
- mwriter = create_manifest_writer(opt.output);
+ mwriter = create_manifest_writer(opt.output, system_identifier);
/*
* Verify that we have a backup manifest for the final backup; else we
}
/*
- * Sanity check control files.
+ * Sanity check control files and return system_identifier.
*/
-static void
+static uint64
check_control_files(int n_backups, char **backup_dirs)
{
int i;
*/
pg_log_debug("system identifier is %llu",
(unsigned long long) system_identifier);
+
+ return system_identifier;
}
/*
use warnings FATAL => 'all';
use File::Compare;
use File::Path qw(rmtree);
+use File::Copy;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
qr/expected system identifier.*but found/,
"can't combine backups from different nodes");
+# Can't combine when different manifest system identifier
+rename("$backup2path/backup_manifest", "$backup2path/backup_manifest.orig")
+ or die "could not move $backup2path/backup_manifest";
+copy("$backupother2path/backup_manifest", "$backup2path/backup_manifest")
+ or die "could not copy $backupother2path/backup_manifest";
+$node1->command_fails_like(
+ [ 'pg_combinebackup', $backup1path, $backup2path, $backup3path, '-o', $resultpath ],
+ qr/ manifest system identifier is .*, but control file has /,
+ "can't combine backups with different manifest system identifier ");
+# Restore the backup state
+move("$backup2path/backup_manifest.orig", "$backup2path/backup_manifest")
+ or die "could not move $backup2path/backup_manifest";
+
# Can't omit a required backup.
$node1->command_fails_like(
[ 'pg_combinebackup', $backup1path, $backup3path, '-o', $resultpath ],
* in the specified directory.
*/
manifest_writer *
-create_manifest_writer(char *directory)
+create_manifest_writer(char *directory, uint64 system_identifier)
{
manifest_writer *mwriter = pg_malloc(sizeof(manifest_writer));
pg_checksum_init(&mwriter->manifest_ctx, CHECKSUM_TYPE_SHA256);
appendStringInfo(&mwriter->buf,
- "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
- "\"Files\": [");
+ "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
+ "\"System-Identifier\": " UINT64_FORMAT ",\n"
+ "\"Files\": [",
+ system_identifier);
return mwriter;
}
struct manifest_writer;
typedef struct manifest_writer manifest_writer;
-extern manifest_writer *create_manifest_writer(char *directory);
+extern manifest_writer *create_manifest_writer(char *directory,
+ uint64 system_identifier);
extern void add_file_to_manifest(manifest_writer *mwriter,
const char *manifest_path,
size_t size, time_t mtime,
#include <sys/stat.h>
#include <time.h>
+#include "common/controldata_utils.h"
#include "common/hashfn.h"
#include "common/logging.h"
#include "common/parse_manifest.h"
*/
typedef struct manifest_data
{
+ int version;
+ uint64 system_identifier;
manifest_files_hash *files;
manifest_wal_range *first_wal_range;
manifest_wal_range *last_wal_range;
} verifier_context;
static manifest_data *parse_manifest_file(char *manifest_path);
+static void verifybackup_version_cb(JsonManifestParseContext *context,
+ int manifest_version);
+static void verifybackup_system_identifier(JsonManifestParseContext *context,
+ uint64 manifest_system_identifier);
static void verifybackup_per_file_cb(JsonManifestParseContext *context,
char *pathname, size_t size,
pg_checksum_type checksum_type,
char *relpath, char *fullpath);
static void verify_backup_file(verifier_context *context,
char *relpath, char *fullpath);
+static void verify_control_file(const char *controlpath,
+ uint64 manifest_system_identifier);
static void report_extra_backup_files(verifier_context *context);
static void verify_backup_checksums(verifier_context *context);
static void verify_file_checksum(verifier_context *context,
}
/*
- * Parse a manifest file. Construct a hash table with information about
- * all the files it mentions, and a linked list of all the WAL ranges it
- * mentions.
+ * Parse a manifest file and return a data structure describing the contents.
*/
static manifest_data *
parse_manifest_file(char *manifest_path)
result = pg_malloc0(sizeof(manifest_data));
result->files = ht;
context.private_data = result;
+ context.version_cb = verifybackup_version_cb;
+ context.system_identifier_cb = verifybackup_system_identifier;
context.per_file_cb = verifybackup_per_file_cb;
context.per_wal_range_cb = verifybackup_per_wal_range_cb;
context.error_cb = report_manifest_error;
exit(1);
}
+/*
+ * Record details extracted from the backup manifest.
+ */
+static void
+verifybackup_version_cb(JsonManifestParseContext *context,
+ int manifest_version)
+{
+ manifest_data *manifest = context->private_data;
+
+ /* Validation will be at the later stage */
+ manifest->version = manifest_version;
+}
+
+/*
+ * Record details extracted from the backup manifest.
+ */
+static void
+verifybackup_system_identifier(JsonManifestParseContext *context,
+ uint64 manifest_system_identifier)
+{
+ manifest_data *manifest = context->private_data;
+
+ /* Validation will be at the later stage */
+ manifest->system_identifier = manifest_system_identifier;
+}
+
/*
* Record details extracted from the backup manifest for one file.
*/
m->bad = true;
}
+ /*
+ * Validate the manifest system identifier, not available in manifest
+ * version 1.
+ */
+ if (context->manifest->version != 1 &&
+ strcmp(relpath, "global/pg_control") == 0)
+ verify_control_file(fullpath, context->manifest->system_identifier);
+
/* Update statistics for progress report, if necessary */
if (show_progress && !skip_checksums && should_verify_checksum(m))
total_size += m->size;
*/
}
+/*
+ * Sanity check control file and validate system identifier against manifest
+ * system identifier.
+ */
+static void
+verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+{
+ ControlFileData *control_file;
+ bool crc_ok;
+
+ pg_log_debug("reading \"%s\"", controlpath);
+ control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
+
+ /* Control file contents not meaningful if CRC is bad. */
+ if (!crc_ok)
+ report_fatal_error("%s: CRC is incorrect", controlpath);
+
+ /* Can't interpret control file if not current version. */
+ if (control_file->pg_control_version != PG_CONTROL_VERSION)
+ report_fatal_error("%s: unexpected control file version",
+ controlpath);
+
+ /* System identifiers should match. */
+ if (manifest_system_identifier != control_file->system_identifier)
+ report_fatal_error("%s: manifest system identifier is %llu, but control file has %llu",
+ controlpath,
+ (unsigned long long) manifest_system_identifier,
+ (unsigned long long) control_file->system_identifier);
+
+ /* Release memory. */
+ pfree(control_file);
+}
+
/*
* Scan the hash table for entries where the 'matched' flag is not set; report
* that such files are present in the manifest but not on disk.
use strict;
use warnings FATAL => 'all';
use File::Path qw(rmtree);
+use File::Copy;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
'mutilate' => \&mutilate_replace_file,
'fails_like' => qr/checksum mismatch for file/
},
+ {
+ 'name' => 'system_identifier',
+ 'mutilate' => \&mutilate_system_identifier,
+ 'fails_like' => qr/manifest system identifier is .*, but control file has/
+ },
{
'name' => 'bad_manifest',
'mutilate' => \&mutilate_bad_manifest,
sub mutilate_truncate_file
{
my ($backup_path) = @_;
- my $pathname = "$backup_path/global/pg_control";
+ my $pathname = "$backup_path/pg_hba.conf";
open(my $fh, '>', $pathname) || die "open $pathname: $!";
close($fh);
return;
return;
}
+# Copy manifest of other backups to demonstrate the case where the wrong
+# manifest is referred
+sub mutilate_system_identifier
+{
+ my ($backup_path) = @_;
+
+ # Set up another new database instance with different system identifier and
+ # make backup
+ my $node = PostgreSQL::Test::Cluster->new('node');
+ $node->init(force_initdb => 1, allows_streaming => 1);
+ $node->start;
+ $node->backup('backup2');
+ move($node->backup_dir.'/backup2/backup_manifest', $backup_path.'/backup_manifest')
+ or BAIL_OUT "could not copy manifest to $backup_path";
+ $node->teardown_node(fail_ok => 1);
+ return;
+}
+
# Corrupt the backup manifest.
sub mutilate_bad_manifest
{
{"not-expected": 1}
EOM
-test_parse_error('unexpected manifest version', <<EOM);
+test_parse_error('manifest version not an integer', <<EOM);
{"PostgreSQL-Backup-Manifest-Version": "phooey"}
EOM
+test_parse_error('unexpected manifest version', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 9876599}
+EOM
+
test_parse_error('unexpected scalar', <<EOM);
{"PostgreSQL-Backup-Manifest-Version": 1, "Files": true}
EOM
JM_EXPECT_TOPLEVEL_END,
JM_EXPECT_TOPLEVEL_FIELD,
JM_EXPECT_VERSION_VALUE,
+ JM_EXPECT_SYSTEM_IDENTIFIER_VALUE,
JM_EXPECT_FILES_START,
JM_EXPECT_FILES_NEXT,
JM_EXPECT_THIS_FILE_FIELD,
/* Miscellaneous other stuff. */
bool saw_version_field;
+ char *manifest_version;
+ char *manifest_system_identifier;
char *manifest_checksum;
} JsonManifestParseState;
bool isnull);
static JsonParseErrorType json_manifest_scalar(void *state, char *token,
JsonTokenType tokentype);
+static void json_manifest_finalize_version(JsonManifestParseState *parse);
+static void json_manifest_finalize_system_identifier(JsonManifestParseState *parse);
static void json_manifest_finalize_file(JsonManifestParseState *parse);
static void json_manifest_finalize_wal_range(JsonManifestParseState *parse);
static void verify_manifest_checksum(JsonManifestParseState *parse,
break;
}
+ /* Is this the system identifier? */
+ if (strcmp(fname, "System-Identifier") == 0)
+ {
+ parse->state = JM_EXPECT_SYSTEM_IDENTIFIER_VALUE;
+ break;
+ }
+
/* Is this the list of files? */
if (strcmp(fname, "Files") == 0)
{
switch (parse->state)
{
case JM_EXPECT_VERSION_VALUE:
- if (strcmp(token, "1") != 0)
- json_manifest_parse_failure(parse->context,
- "unexpected manifest version");
+ parse->manifest_version = token;
+ json_manifest_finalize_version(parse);
+ parse->state = JM_EXPECT_TOPLEVEL_FIELD;
+ break;
+
+ case JM_EXPECT_SYSTEM_IDENTIFIER_VALUE:
+ parse->manifest_system_identifier = token;
+ json_manifest_finalize_system_identifier(parse);
parse->state = JM_EXPECT_TOPLEVEL_FIELD;
break;
return JSON_SUCCESS;
}
+/*
+ * Do additional parsing and sanity-checking of the manifest version, and invoke
+ * the callback so that the caller can gets that detail and take actions
+ * accordingly. This happens for each manifest when the corresponding JSON
+ * object is completely parsed.
+ */
+static void
+json_manifest_finalize_version(JsonManifestParseState *parse)
+{
+ JsonManifestParseContext *context = parse->context;
+ int version;
+ char *ep;
+
+ Assert(parse->saw_version_field);
+
+ /* Parse version. */
+ version = strtoi64(parse->manifest_version, &ep, 10);
+ if (*ep)
+ json_manifest_parse_failure(parse->context,
+ "manifest version not an integer");
+
+ if (version != 1 && version != 2)
+ json_manifest_parse_failure(parse->context,
+ "unexpected manifest version");
+
+ /* Invoke the callback for version */
+ context->version_cb(context, version);
+}
+
+/*
+ * Do additional parsing and sanity-checking of the system identifier, and
+ * invoke the callback so that the caller can gets that detail and take actions
+ * accordingly.
+ */
+static void
+json_manifest_finalize_system_identifier(JsonManifestParseState *parse)
+{
+ JsonManifestParseContext *context = parse->context;
+ uint64 system_identifier;
+ char *ep;
+
+ Assert(parse->manifest_system_identifier != NULL);
+
+ /* Parse system identifier. */
+ system_identifier = strtou64(parse->manifest_system_identifier, &ep, 10);
+ if (*ep)
+ json_manifest_parse_failure(parse->context,
+ "manifest system identifier not an integer");
+
+ /* Invoke the callback for system identifier */
+ context->system_identifier_cb(context, system_identifier);
+}
+
/*
* Do additional parsing and sanity-checking of the details gathered for one
* file, and invoke the per-file callback so that the caller gets those
struct JsonManifestParseContext;
typedef struct JsonManifestParseContext JsonManifestParseContext;
+typedef void (*json_manifest_version_callback) (JsonManifestParseContext *,
+ int manifest_version);
+typedef void (*json_manifest_system_identifier_callback) (JsonManifestParseContext *,
+ uint64 manifest_system_identifier);
typedef void (*json_manifest_per_file_callback) (JsonManifestParseContext *,
char *pathname,
size_t size, pg_checksum_type checksum_type,
struct JsonManifestParseContext
{
void *private_data;
+ json_manifest_version_callback version_cb;
+ json_manifest_system_identifier_callback system_identifier_cb;
json_manifest_per_file_callback per_file_cb;
json_manifest_per_wal_range_callback per_wal_range_cb;
json_manifest_error_callback error_cb;
projects / postgresql.git / commitdiff