Invent WAL timelines, as per recent discussion, to make point-in-time

recovery more manageable.  Also, undo recent change to add FILE_HEADER
and WASTED_SPACE records to XLOG; instead make the XLOG page header
variable-size with extra fields in the first page of an XLOG file.
This should fix the boundary-case bugs observed by Mark Kirkwood.
initdb forced due to change of XLOG representation.

5 files changed, 241 insertions, 122 deletions

diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 714518d308c..458b3012adf 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xact.h,v 1.65 2004/07/17 03:30:38 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/xact.h,v 1.66 2004/07/21 22:31:25 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -15,6 +15,7 @@
 #define XACT_H
 
 #include "access/xlog.h"
+#include "storage/relfilenode.h"
 #include "utils/nabstime.h"
 
 
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 630a62d77a9..f2542d6fc71 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.53 2004/07/19 02:47:13 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.54 2004/07/21 22:31:25 tgl Exp $
  */
 #ifndef XLOG_H
 #define XLOG_H
@@ -14,7 +14,7 @@
 #include "access/rmgr.h"
 #include "access/transam.h"
 #include "access/xlogdefs.h"
-#include "storage/bufmgr.h"
+#include "storage/buf.h"
 #include "utils/pg_crc.h"
 
 
@@ -76,107 +76,6 @@ typedef struct XLogRecord
 #define XLOG_NO_TRAN			XLR_INFO_MASK
 
 /*
- * Header info for a backup block appended to an XLOG record.
- *
- * Note that the backup block has its own CRC, and is not covered by
- * the CRC of the XLOG record proper.  Also note that we don't attempt
- * to align either the BkpBlock struct or the block's data.
- */
-typedef struct BkpBlock
-{
-	crc64		crc;
-	RelFileNode node;
-	BlockNumber block;
-} BkpBlock;
-
-/*
- * When there is not enough space on current page for whole record, we
- * continue on the next page with continuation record.	(However, the
- * XLogRecord header will never be split across pages; if there's less than
- * SizeOfXLogRecord space left at the end of a page, we just waste it.)
- *
- * Note that xl_rem_len includes backup-block data, unlike xl_len in the
- * initial header.
- */
-typedef struct XLogContRecord
-{
-	uint32		xl_rem_len;		/* total len of remaining data for record */
-
-	/* ACTUAL LOG DATA FOLLOWS AT END OF STRUCT */
-
-} XLogContRecord;
-
-#define SizeOfXLogContRecord	MAXALIGN(sizeof(XLogContRecord))
-
-/*
- * Each page of XLOG file has a header like this:
- */
-#define XLOG_PAGE_MAGIC 0xD05B	/* can be used as WAL version indicator */
-
-typedef struct XLogPageHeaderData
-{
-	uint16		xlp_magic;		/* magic value for correctness checks */
-	uint16		xlp_info;		/* flag bits, see below */
-	StartUpID	xlp_sui;		/* StartUpID of first record on page */
-	XLogRecPtr	xlp_pageaddr;	/* XLOG address of this page */
-} XLogPageHeaderData;
-
-#define SizeOfXLogPHD	MAXALIGN(sizeof(XLogPageHeaderData))
-
-typedef XLogPageHeaderData *XLogPageHeader;
-
-/* When record crosses page boundary, set this flag in new page's header */
-#define XLP_FIRST_IS_CONTRECORD		0x0001
-/* All defined flag bits in xlp_info (used for validity checking of header) */
-#define XLP_ALL_FLAGS				0x0001
-
-/*
- * We break each logical log file (xlogid value) into segment files of the
- * size indicated by XLOG_SEG_SIZE.  One possible segment at the end of each
- * log file is wasted, to ensure that we don't have problems representing
- * last-byte-position-plus-1.
- */
-#define XLogSegSize		((uint32) XLOG_SEG_SIZE)
-#define XLogSegsPerFile (((uint32) 0xffffffff) / XLogSegSize)
-#define XLogFileSize	(XLogSegsPerFile * XLogSegSize)
-
-/*
- * The first XLOG record in each segment file is always an XLOG_FILE_HEADER
- * record.  This record does nothing as far as XLOG replay is concerned,
- * but it is useful for verifying that we haven't mixed up XLOG segment files.
- * The body of an XLOG_FILE_HEADER record is a struct XLogFileHeaderData.
- * Note: the xlogid/segno fields are really redundant with xlp_pageaddr in
- * the page header, but we store them anyway as an extra check.
- */
-typedef struct XLogFileHeaderData
-{
-	uint64		xlfhd_sysid;	/* system identifier from pg_control */
-	uint32		xlfhd_xlogid;	/* logical log file # */
-	uint32		xlfhd_segno;	/* segment number within logical log file */
-	uint32		xlfhd_seg_size;	/* just as a cross-check */
-} XLogFileHeaderData;
-
-#define SizeOfXLogFHD	MAXALIGN(sizeof(XLogFileHeaderData))
-
-
-/*
- * Method table for resource managers.
- *
- * RmgrTable[] is indexed by RmgrId values (see rmgr.h).
- */
-typedef struct RmgrData
-{
-	const char *rm_name;
-	void		(*rm_redo) (XLogRecPtr lsn, XLogRecord *rptr);
-	void		(*rm_undo) (XLogRecPtr lsn, XLogRecord *rptr);
-	void		(*rm_desc) (char *buf, uint8 xl_info, char *rec);
-	void		(*rm_startup) (void);
-	void		(*rm_cleanup) (void);
-} RmgrData;
-
-extern RmgrData RmgrTable[];
-
-/*--------------------
  * List of these structs is used to pass data to XLogInsert().
  *
  * If buffer is valid then XLOG will check if buffer must be backed up
@@ -188,7 +87,6 @@ extern RmgrData RmgrTable[];
  * the XLOG record, since we assume it's present in the buffer.  Therefore,
  * rmgr redo routines MUST pay attention to XLR_BKP_BLOCK_X to know what
  * is actually stored in the XLOG record.
- *--------------------
  */
 typedef struct XLogRecData
 {
@@ -198,7 +96,7 @@ typedef struct XLogRecData
 	struct XLogRecData *next;
 } XLogRecData;
 
-extern StartUpID ThisStartUpID; /* current SUI */
+extern TimeLineID ThisTimeLineID; /* current TLI */
 extern bool InRecovery;
 extern XLogRecPtr MyLastRecPtr;
 extern bool MyXactMadeXLogEntry;
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
new file mode 100644
index 00000000000..09877bf64be
--- /dev/null
+++ b/src/include/access/xlog_internal.h
@@ -0,0 +1,224 @@
+/*
+ * xlog_internal.h
+ *
+ * PostgreSQL transaction log internal declarations
+ *
+ * NOTE: this file is intended to contain declarations useful for
+ * manipulating the XLOG files directly, but it is not supposed to be
+ * needed by rmgr routines (redo/undo support for individual record types).
+ * So the XLogRecord typedef and associated stuff appear in xlog.h.
+ *
+ * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * $PostgreSQL: pgsql/src/include/access/xlog_internal.h,v 1.1 2004/07/21 22:31:25 tgl Exp $
+ */
+#ifndef XLOG_INTERNAL_H
+#define XLOG_INTERNAL_H
+
+#include "access/xlog.h"
+#include "storage/block.h"
+#include "storage/relfilenode.h"
+
+
+/*
+ * Header info for a backup block appended to an XLOG record.
+ *
+ * Note that the backup block has its own CRC, and is not covered by
+ * the CRC of the XLOG record proper.  Also note that we don't attempt
+ * to align either the BkpBlock struct or the block's data.
+ */
+typedef struct BkpBlock
+{
+	crc64		crc;
+	RelFileNode node;
+	BlockNumber block;
+} BkpBlock;
+
+/*
+ * When there is not enough space on current page for whole record, we
+ * continue on the next page with continuation record.	(However, the
+ * XLogRecord header will never be split across pages; if there's less than
+ * SizeOfXLogRecord space left at the end of a page, we just waste it.)
+ *
+ * Note that xl_rem_len includes backup-block data, unlike xl_len in the
+ * initial header.
+ */
+typedef struct XLogContRecord
+{
+	uint32		xl_rem_len;		/* total len of remaining data for record */
+
+	/* ACTUAL LOG DATA FOLLOWS AT END OF STRUCT */
+
+} XLogContRecord;
+
+#define SizeOfXLogContRecord	MAXALIGN(sizeof(XLogContRecord))
+
+/*
+ * Each page of XLOG file has a header like this:
+ */
+#define XLOG_PAGE_MAGIC 0xD05B	/* can be used as WAL version indicator */
+
+typedef struct XLogPageHeaderData
+{
+	uint16		xlp_magic;		/* magic value for correctness checks */
+	uint16		xlp_info;		/* flag bits, see below */
+	TimeLineID	xlp_tli;		/* TimeLineID of first record on page */
+	XLogRecPtr	xlp_pageaddr;	/* XLOG address of this page */
+} XLogPageHeaderData;
+
+#define SizeOfXLogShortPHD	MAXALIGN(sizeof(XLogPageHeaderData))
+
+typedef XLogPageHeaderData *XLogPageHeader;
+
+/*
+ * When the XLP_LONG_HEADER flag is set, we store additional fields in the
+ * page header.  (This is ordinarily done just in the first page of an
+ * XLOG file.)  The additional fields serve to identify the file accurately.
+ */
+typedef struct XLogLongPageHeaderData
+{
+	XLogPageHeaderData std;		/* standard header fields */
+	uint64		xlp_sysid;		/* system identifier from pg_control */
+	uint32		xlp_seg_size;	/* just as a cross-check */
+} XLogLongPageHeaderData;
+
+#define SizeOfXLogLongPHD	MAXALIGN(sizeof(XLogLongPageHeaderData))
+
+typedef XLogLongPageHeaderData *XLogLongPageHeader;
+
+/* When record crosses page boundary, set this flag in new page's header */
+#define XLP_FIRST_IS_CONTRECORD		0x0001
+/* This flag indicates a "long" page header */
+#define XLP_LONG_HEADER				0x0002
+/* All defined flag bits in xlp_info (used for validity checking of header) */
+#define XLP_ALL_FLAGS				0x0003
+
+#define XLogPageHeaderSize(hdr)		\
+	(((hdr)->xlp_info & XLP_LONG_HEADER) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD)
+
+/*
+ * We break each logical log file (xlogid value) into segment files of the
+ * size indicated by XLOG_SEG_SIZE.  One possible segment at the end of each
+ * log file is wasted, to ensure that we don't have problems representing
+ * last-byte-position-plus-1.
+ */
+#define XLogSegSize		((uint32) XLOG_SEG_SIZE)
+#define XLogSegsPerFile (((uint32) 0xffffffff) / XLogSegSize)
+#define XLogFileSize	(XLogSegsPerFile * XLogSegSize)
+
+
+/*
+ * Macros for manipulating XLOG pointers
+ */
+
+/* Increment an xlogid/segment pair */
+#define NextLogSeg(logId, logSeg)	\
+	do { \
+		if ((logSeg) >= XLogSegsPerFile-1) \
+		{ \
+			(logId)++; \
+			(logSeg) = 0; \
+		} \
+		else \
+			(logSeg)++; \
+	} while (0)
+
+/* Decrement an xlogid/segment pair (assume it's not 0,0) */
+#define PrevLogSeg(logId, logSeg)	\
+	do { \
+		if (logSeg) \
+			(logSeg)--; \
+		else \
+		{ \
+			(logId)--; \
+			(logSeg) = XLogSegsPerFile-1; \
+		} \
+	} while (0)
+
+/*
+ * Compute ID and segment from an XLogRecPtr.
+ *
+ * For XLByteToSeg, do the computation at face value.  For XLByteToPrevSeg,
+ * a boundary byte is taken to be in the previous segment.	This is suitable
+ * for deciding which segment to write given a pointer to a record end,
+ * for example.  (We can assume xrecoff is not zero, since no valid recptr
+ * can have that.)
+ */
+#define XLByteToSeg(xlrp, logId, logSeg)	\
+	( logId = (xlrp).xlogid, \
+	  logSeg = (xlrp).xrecoff / XLogSegSize \
+	)
+#define XLByteToPrevSeg(xlrp, logId, logSeg)	\
+	( logId = (xlrp).xlogid, \
+	  logSeg = ((xlrp).xrecoff - 1) / XLogSegSize \
+	)
+
+/*
+ * Is an XLogRecPtr within a particular XLOG segment?
+ *
+ * For XLByteInSeg, do the computation at face value.  For XLByteInPrevSeg,
+ * a boundary byte is taken to be in the previous segment.
+ */
+#define XLByteInSeg(xlrp, logId, logSeg)	\
+	((xlrp).xlogid == (logId) && \
+	 (xlrp).xrecoff / XLogSegSize == (logSeg))
+
+#define XLByteInPrevSeg(xlrp, logId, logSeg)	\
+	((xlrp).xlogid == (logId) && \
+	 ((xlrp).xrecoff - 1) / XLogSegSize == (logSeg))
+
+/* Check if an xrecoff value is in a plausible range */
+#define XRecOffIsValid(xrecoff) \
+		((xrecoff) % BLCKSZ >= SizeOfXLogShortPHD && \
+		(BLCKSZ - (xrecoff) % BLCKSZ) >= SizeOfXLogRecord)
+
+/*
+ * These macros encapsulate knowledge about the exact layout of XLog file
+ * names, timeline history file names, and archive-status file names.
+ */
+#define MAXFNAMELEN		32
+
+#define XLogFileName(fname, tli, log, seg)	\
+	snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli, log, seg)
+
+#define XLogFilePath(path, tli, log, seg)	\
+	snprintf(path, MAXPGPATH, "%s/%08X%08X%08X", XLogDir, tli, log, seg)
+
+#define TLHistoryFileName(fname, tli)	\
+	snprintf(fname, MAXFNAMELEN, "%08X.history", tli)
+
+#define TLHistoryFilePath(path, tli)	\
+	snprintf(path, MAXPGPATH, "%s/%08X.history", XLogDir, tli)
+
+#define StatusFilePath(path, xlog, suffix)	\
+	snprintf(path, MAXPGPATH, "%s/archive_status/%s%s", XLogDir, xlog, suffix)
+
+extern char XLogDir[MAXPGPATH];
+
+/*
+ * _INTL_MAXLOGRECSZ: max space needed for a record including header and
+ * any backup-block data.
+ */
+#define _INTL_MAXLOGRECSZ	(SizeOfXLogRecord + MAXLOGRECSZ + \
+							 XLR_MAX_BKP_BLOCKS * (sizeof(BkpBlock) + BLCKSZ))
+
+
+/*
+ * Method table for resource managers.
+ *
+ * RmgrTable[] is indexed by RmgrId values (see rmgr.h).
+ */
+typedef struct RmgrData
+{
+	const char *rm_name;
+	void		(*rm_redo) (XLogRecPtr lsn, XLogRecord *rptr);
+	void		(*rm_undo) (XLogRecPtr lsn, XLogRecord *rptr);
+	void		(*rm_desc) (char *buf, uint8 xl_info, char *rec);
+	void		(*rm_startup) (void);
+	void		(*rm_cleanup) (void);
+} RmgrData;
+
+extern const RmgrData RmgrTable[];
+
+#endif   /* XLOG_INTERNAL_H */
diff --git a/src/include/access/xlogdefs.h b/src/include/access/xlogdefs.h
index 057236e4eda..4507723e5a8 100644
--- a/src/include/access/xlogdefs.h
+++ b/src/include/access/xlogdefs.h
@@ -2,12 +2,12 @@
  * xlogdefs.h
  *
  * Postgres transaction log manager record pointer and
- * system startup number definitions
+ * timeline number definitions
  *
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xlogdefs.h,v 1.11 2003/12/20 17:31:21 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/access/xlogdefs.h,v 1.12 2004/07/21 22:31:25 tgl Exp $
  */
 #ifndef XLOG_DEFS_H
 #define XLOG_DEFS_H
@@ -33,12 +33,6 @@ typedef struct XLogRecPtr
 	uint32		xrecoff;		/* byte offset of location in log file */
 } XLogRecPtr;
 
-typedef struct XLogwrtResult
-{
-	XLogRecPtr	Write;			/* last byte + 1 written out */
-	XLogRecPtr	Flush;			/* last byte + 1 flushed */
-} XLogwrtResult;
-
 
 /*
  * Macros for comparing XLogRecPtrs
@@ -57,10 +51,16 @@ typedef struct XLogwrtResult
 #define XLByteEQ(a, b)		\
 			((a).xlogid == (b).xlogid && (a).xrecoff == (b).xrecoff)
 
+
 /*
- * StartUpID (SUI) - system startups counter. It's to allow removing
- * pg_clog after shutdown, in future.
+ * TimeLineID (TLI) - identifies different database histories to prevent
+ * confusion after restoring a prior state of a database installation.
+ * TLI does not change in a normal stop/restart of the database (including
+ * crash-and-recover cases); but we must assign a new TLI after doing
+ * a recovery to a prior state, a/k/a point-in-time recovery.  This makes
+ * the new WAL logfile sequence we generate distinguishable from the
+ * sequence that was generated in the previous incarnation.
  */
-typedef uint32 StartUpID;
+typedef uint32 TimeLineID;
 
 #endif   /* XLOG_DEFS_H */
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 8b1dc671fa0..a5b8f30978e 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -6,19 +6,15 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xlogutils.h,v 1.14 2004/02/11 22:55:25 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/xlogutils.h,v 1.15 2004/07/21 22:31:25 tgl Exp $
  */
 #ifndef XLOG_UTILS_H
 #define XLOG_UTILS_H
 
 #include "access/rmgr.h"
 #include "storage/buf.h"
-#include "storage/itemptr.h"
 #include "utils/rel.h"
 
-extern int XLogIsOwnerOfTuple(RelFileNode hnode, ItemPointer iptr,
-				   TransactionId xid, CommandId cid);
-extern bool XLogIsValidTuple(RelFileNode hnode, ItemPointer iptr);
 
 extern void XLogInitRelationCache(void);
 extern void XLogCloseRelationCache(void);