diff options
author | Heikki Linnakangas | 2010-01-15 09:19:10 +0000 |
---|---|---|
committer | Heikki Linnakangas | 2010-01-15 09:19:10 +0000 |
commit | 40f908bdcdc726fc11912cd95dfd2f89603d1f37 (patch) | |
tree | fcbbda839e58184fc284d261cc5a0f5e8ac921f3 /src/include | |
parent | 4cbe473938779ec414d90c2063c4398e68a70838 (diff) |
Introduce Streaming Replication.
This includes two new kinds of postmaster processes, walsenders and
walreceiver. Walreceiver is responsible for connecting to the primary server
and streaming WAL to disk, while walsender runs in the primary server and
streams WAL from disk to the client.
Documentation still needs work, but the basics are there. We will probably
pull the replication section to a new chapter later on, as well as the
sections describing file-based replication. But let's do that as a separate
patch, so that it's easier to see what has been added/changed. This patch
also adds a new section to the chapter about FE/BE protocol, documenting the
protocol used by walsender/walreceivxer.
Bump catalog version because of two new functions,
pg_last_xlog_receive_location() and pg_last_xlog_replay_location(), for
monitoring the progress of replication.
Fujii Masao, with additional hacking by me
Diffstat (limited to 'src/include')
-rw-r--r-- | src/include/Makefile | 6 | ||||
-rw-r--r-- | src/include/access/xlog.h | 27 | ||||
-rw-r--r-- | src/include/access/xlog_internal.h | 17 | ||||
-rw-r--r-- | src/include/access/xlogdefs.h | 18 | ||||
-rw-r--r-- | src/include/bootstrap/bootstrap.h | 3 | ||||
-rw-r--r-- | src/include/catalog/catversion.h | 4 | ||||
-rw-r--r-- | src/include/catalog/pg_proc.h | 7 | ||||
-rw-r--r-- | src/include/libpq/libpq-be.h | 3 | ||||
-rw-r--r-- | src/include/libpq/libpq.h | 3 | ||||
-rw-r--r-- | src/include/replication/walreceiver.h | 70 | ||||
-rw-r--r-- | src/include/replication/walsender.h | 49 | ||||
-rw-r--r-- | src/include/storage/pmsignal.h | 6 | ||||
-rw-r--r-- | src/include/storage/proc.h | 8 | ||||
-rw-r--r-- | src/include/utils/guc_tables.h | 3 |
14 files changed, 206 insertions, 18 deletions
diff --git a/src/include/Makefile b/src/include/Makefile index ed88dca3942..c89960c7d62 100644 --- a/src/include/Makefile +++ b/src/include/Makefile @@ -4,7 +4,7 @@ # # 'make install' installs whole contents of src/include. # -# $PostgreSQL: pgsql/src/include/Makefile,v 1.30 2010/01/05 01:06:56 tgl Exp $ +# $PostgreSQL: pgsql/src/include/Makefile,v 1.31 2010/01/15 09:19:05 heikki Exp $ # #------------------------------------------------------------------------- @@ -18,8 +18,8 @@ all: pg_config.h pg_config_os.h # Subdirectories containing headers for server-side dev SUBDIRS = access bootstrap catalog commands executor foreign lib libpq mb \ - nodes optimizer parser postmaster regex rewrite storage tcop \ - snowball snowball/libstemmer tsearch tsearch/dicts utils \ + nodes optimizer parser postmaster regex replication rewrite storage \ + tcop snowball snowball/libstemmer tsearch tsearch/dicts utils \ port port/win32 port/win32_msvc port/win32_msvc/sys \ port/win32/arpa port/win32/netinet port/win32/sys \ portability diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 1d18cb5b1b9..20083e14c54 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -6,7 +6,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.95 2010/01/02 16:58:00 momjian Exp $ + * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.96 2010/01/15 09:19:06 heikki Exp $ */ #ifndef XLOG_H #define XLOG_H @@ -188,6 +188,18 @@ extern int MaxStandbyDelay; #define XLogArchiveCommandSet() (XLogArchiveCommand[0] != '\0') #define XLogStandbyInfoActive() (XLogRequestRecoveryConnections && XLogArchiveMode) +/* + * This is in walsender.c, but declared here so that we don't need to include + * walsender.h in all files that check XLogIsNeeded() + */ +extern int MaxWalSenders; + +/* + * Is WAL-logging necessary? We need to log an XLOG record iff either + * WAL archiving is enabled or XLOG streaming is allowed. + */ +#define XLogIsNeeded() (XLogArchivingActive() || (MaxWalSenders > 0)) + #ifdef WAL_DEBUG extern bool XLOG_DEBUG; #endif @@ -228,12 +240,19 @@ typedef struct CheckpointStatsData extern CheckpointStatsData CheckpointStats; +/* Read from recovery.conf, in startup process */ +extern char *TriggerFile; + extern XLogRecPtr XLogInsert(RmgrId rmid, uint8 info, XLogRecData *rdata); extern void XLogFlush(XLogRecPtr RecPtr); extern void XLogBackgroundFlush(void); extern void XLogAsyncCommitFlush(void); extern bool XLogNeedsFlush(XLogRecPtr RecPtr); +extern int XLogFileInit(uint32 log, uint32 seg, + bool *use_existent, bool use_lock); +extern int XLogFileOpen(uint32 log, uint32 seg); + extern void XLogSetAsyncCommitLSN(XLogRecPtr record); @@ -242,11 +261,14 @@ extern void RestoreBkpBlocks(XLogRecPtr lsn, XLogRecord *record, bool cleanup); extern void xlog_redo(XLogRecPtr lsn, XLogRecord *record); extern void xlog_desc(StringInfo buf, uint8 xl_info, char *rec); +extern void issue_xlog_fsync(int fd, uint32 log, uint32 seg); + extern bool RecoveryInProgress(void); extern bool XLogInsertAllowed(void); extern TimestampTz GetLatestXLogTime(void); extern void UpdateControlFile(void); +extern uint64 GetSystemIdentifier(void); extern Size XLOGShmemSize(void); extern void XLOGShmemInit(void); extern void BootStrapXLOG(void); @@ -258,8 +280,11 @@ extern bool CreateRestartPoint(int flags); extern void XLogPutNextOid(Oid nextOid); extern XLogRecPtr GetRedoRecPtr(void); extern XLogRecPtr GetInsertRecPtr(void); +extern XLogRecPtr GetWriteRecPtr(void); extern void GetNextXidAndEpoch(TransactionId *xid, uint32 *epoch); +extern TimeLineID GetRecoveryTargetTLI(void); +extern void HandleStartupProcInterrupts(void); extern void StartupProcessMain(void); #endif /* XLOG_H */ diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index a3614665e18..cfb7f0a4de6 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -11,7 +11,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/access/xlog_internal.h,v 1.27 2010/01/02 16:58:01 momjian Exp $ + * $PostgreSQL: pgsql/src/include/access/xlog_internal.h,v 1.28 2010/01/15 09:19:06 heikki Exp $ */ #ifndef XLOG_INTERNAL_H #define XLOG_INTERNAL_H @@ -151,6 +151,19 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader; } \ } while (0) +/* Align a record pointer to next page */ +#define NextLogPage(recptr) \ + do { \ + if (recptr.xrecoff % XLOG_BLCKSZ != 0) \ + recptr.xrecoff += \ + (XLOG_BLCKSZ - recptr.xrecoff % XLOG_BLCKSZ); \ + if (recptr.xrecoff >= XLogFileSize) \ + { \ + (recptr.xlogid)++; \ + recptr.xrecoff = 0; \ + } \ + } while (0) + /* * Compute ID and segment from an XLogRecPtr. * @@ -253,6 +266,8 @@ extern Datum pg_stop_backup(PG_FUNCTION_ARGS); extern Datum pg_switch_xlog(PG_FUNCTION_ARGS); extern Datum pg_current_xlog_location(PG_FUNCTION_ARGS); extern Datum pg_current_xlog_insert_location(PG_FUNCTION_ARGS); +extern Datum pg_last_xlog_receive_location(PG_FUNCTION_ARGS); +extern Datum pg_last_xlog_replay_location(PG_FUNCTION_ARGS); extern Datum pg_xlogfile_name_offset(PG_FUNCTION_ARGS); extern Datum pg_xlogfile_name(PG_FUNCTION_ARGS); extern Datum pg_is_in_recovery(PG_FUNCTION_ARGS); diff --git a/src/include/access/xlogdefs.h b/src/include/access/xlogdefs.h index 8f524df75e2..8ecc3a21b1c 100644 --- a/src/include/access/xlogdefs.h +++ b/src/include/access/xlogdefs.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/access/xlogdefs.h,v 1.24 2010/01/02 16:58:01 momjian Exp $ + * $PostgreSQL: pgsql/src/include/access/xlogdefs.h,v 1.25 2010/01/15 09:19:06 heikki Exp $ */ #ifndef XLOG_DEFS_H #define XLOG_DEFS_H @@ -57,6 +57,22 @@ typedef struct XLogRecPtr /* + * Macro for advancing a record pointer by the specified number of bytes. + */ +#define XLByteAdvance(recptr, nbytes) \ + do { \ + if (recptr.xrecoff + nbytes >= XLogFileSize) \ + { \ + recptr.xlogid += 1; \ + recptr.xrecoff \ + = recptr.xrecoff + nbytes - XLogFileSize; \ + } \ + else \ + recptr.xrecoff += nbytes; \ + } while (0) + + +/* * TimeLineID (TLI) - identifies different database histories to prevent * confusion after restoring a prior state of a database installation. * TLI does not change in a normal stop/restart of the database (including diff --git a/src/include/bootstrap/bootstrap.h b/src/include/bootstrap/bootstrap.h index eef42018eb4..5e989eff4ec 100644 --- a/src/include/bootstrap/bootstrap.h +++ b/src/include/bootstrap/bootstrap.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/bootstrap/bootstrap.h,v 1.54 2010/01/02 16:58:01 momjian Exp $ + * $PostgreSQL: pgsql/src/include/bootstrap/bootstrap.h,v 1.55 2010/01/15 09:19:06 heikki Exp $ * *------------------------------------------------------------------------- */ @@ -23,6 +23,7 @@ typedef enum StartupProcess, BgWriterProcess, WalWriterProcess, + WalReceiverProcess, NUM_AUXPROCTYPES /* Must be last! */ } AuxProcType; diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index e981823320c..b2c92860f07 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -37,7 +37,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.572 2010/01/14 16:31:09 teodor Exp $ + * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.573 2010/01/15 09:19:07 heikki Exp $ * *------------------------------------------------------------------------- */ @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201001141 +#define CATALOG_VERSION_NO 201001151 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 7a16e3faa73..a498cc51720 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/pg_proc.h,v 1.561 2010/01/14 16:31:09 teodor Exp $ + * $PostgreSQL: pgsql/src/include/catalog/pg_proc.h,v 1.562 2010/01/15 09:19:07 heikki Exp $ * * NOTES * The script catalog/genbki.pl reads this file and generates .bki @@ -3290,6 +3290,11 @@ DESCR("xlog filename, given an xlog location"); DATA(insert OID = 3810 ( pg_is_in_recovery PGNSP PGUID 12 1 0 0 f f f t f v 0 0 16 "" _null_ _null_ _null_ _null_ pg_is_in_recovery _null_ _null_ _null_ )); DESCR("true if server is in recovery"); +DATA(insert OID = 3820 ( pg_last_xlog_receive_location PGNSP PGUID 12 1 0 0 f f f t f v 0 0 25 "" _null_ _null_ _null_ _null_ pg_last_xlog_receive_location _null_ _null_ _null_ )); +DESCR("current xlog flush location"); +DATA(insert OID = 3821 ( pg_last_xlog_replay_location PGNSP PGUID 12 1 0 0 f f f t f v 0 0 25 "" _null_ _null_ _null_ _null_ pg_last_xlog_replay_location _null_ _null_ _null_ )); +DESCR("last xlog replay location"); + DATA(insert OID = 2621 ( pg_reload_conf PGNSP PGUID 12 1 0 0 f f f t f v 0 0 16 "" _null_ _null_ _null_ _null_ pg_reload_conf _null_ _null_ _null_ )); DESCR("reload configuration files"); DATA(insert OID = 2622 ( pg_rotate_logfile PGNSP PGUID 12 1 0 0 f f f t f v 0 0 16 "" _null_ _null_ _null_ _null_ pg_rotate_logfile _null_ _null_ _null_ )); diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index 1d358715288..6ee4714489b 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -11,7 +11,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/libpq/libpq-be.h,v 1.73 2010/01/10 14:16:08 mha Exp $ + * $PostgreSQL: pgsql/src/include/libpq/libpq-be.h,v 1.74 2010/01/15 09:19:08 heikki Exp $ * *------------------------------------------------------------------------- */ @@ -104,6 +104,7 @@ typedef struct typedef struct Port { pgsocket sock; /* File descriptor */ + bool noblock; /* is the socket in non-blocking mode? */ ProtocolVersion proto; /* FE/BE protocol version */ SockAddr laddr; /* local addr (postmaster) */ SockAddr raddr; /* remote addr (client) */ diff --git a/src/include/libpq/libpq.h b/src/include/libpq/libpq.h index c9c7e0d7640..c8fa2778824 100644 --- a/src/include/libpq/libpq.h +++ b/src/include/libpq/libpq.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/libpq/libpq.h,v 1.73 2010/01/10 14:16:08 mha Exp $ + * $PostgreSQL: pgsql/src/include/libpq/libpq.h,v 1.74 2010/01/15 09:19:08 heikki Exp $ * *------------------------------------------------------------------------- */ @@ -57,6 +57,7 @@ extern int pq_getstring(StringInfo s); extern int pq_getmessage(StringInfo s, int maxlen); extern int pq_getbyte(void); extern int pq_peekbyte(void); +extern int pq_getbyte_if_available(unsigned char *c); extern int pq_putbytes(const char *s, size_t len); extern int pq_flush(void); extern int pq_putmessage(char msgtype, const char *s, size_t len); diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h new file mode 100644 index 00000000000..7651a696a0f --- /dev/null +++ b/src/include/replication/walreceiver.h @@ -0,0 +1,70 @@ +/*------------------------------------------------------------------------- + * + * walreceiver.h + * Exports from replication/walreceiverfuncs.c. + * + * Portions Copyright (c) 2010-2010, PostgreSQL Global Development Group + * + * $PostgreSQL: pgsql/src/include/replication/walreceiver.h,v 1.1 2010/01/15 09:19:09 heikki Exp $ + * + *------------------------------------------------------------------------- + */ +#ifndef _WALRECEIVER_H +#define _WALRECEIVER_H + +#include "storage/spin.h" + +/* + * MAXCONNINFO: maximum size of a connection string. + * + * XXX: Should this move to pg_config_manual.h? + */ +#define MAXCONNINFO 1024 + +/* + * Values for WalRcv->walRcvState. + */ +typedef enum +{ + WALRCV_NOT_STARTED, + WALRCV_RUNNING, /* walreceiver has been started */ + WALRCV_STOPPING, /* requested to stop, but still running */ + WALRCV_STOPPED /* stopped and mustn't start up again */ +} WalRcvState; + +/* Shared memory area for management of walreceiver process */ +typedef struct +{ + /* + * connection string; is used for walreceiver to connect with + * the primary. + */ + char conninfo[MAXCONNINFO]; + + /* + * PID of currently active walreceiver process, and the current state. + */ + pid_t pid; + WalRcvState walRcvState; + + /* + * receivedUpto-1 is the last byte position that has been already + * received. When startup process starts the walreceiver, it sets this + * to the point where it wants the streaming to begin. After that, + * walreceiver updates this whenever it flushes the received WAL. + */ + XLogRecPtr receivedUpto; + + slock_t mutex; /* locks shared variables shown above */ +} WalRcvData; + +extern WalRcvData *WalRcv; + +extern Size WalRcvShmemSize(void); +extern void WalRcvShmemInit(void); +extern bool WalRcvInProgress(void); +extern XLogRecPtr WaitNextXLogAvailable(XLogRecPtr recptr, bool *finished); +extern void RequestXLogStreaming(XLogRecPtr recptr, const char *conninfo); +extern XLogRecPtr GetWalRcvWriteRecPtr(void); + +#endif /* _WALRECEIVER_H */ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h new file mode 100644 index 00000000000..c9bfd12e8bc --- /dev/null +++ b/src/include/replication/walsender.h @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------- + * + * walsender.h + * Exports from replication/walsender.c. + * + * Portions Copyright (c) 2010-2010, PostgreSQL Global Development Group + * + * $PostgreSQL: pgsql/src/include/replication/walsender.h,v 1.1 2010/01/15 09:19:09 heikki Exp $ + * + *------------------------------------------------------------------------- + */ +#ifndef _WALSENDER_H +#define _WALSENDER_H + +#include "access/xlog.h" +#include "storage/spin.h" + +/* + * Each walsender has a WalSnd struct in shared memory. + */ +typedef struct WalSnd +{ + pid_t pid; /* this walsender's process id, or 0 */ + XLogRecPtr sentPtr; /* WAL has been sent up to this point */ + + slock_t mutex; /* locks shared variables shown above */ +} WalSnd; + +/* There is one WalSndCtl struct for the whole database cluster */ +typedef struct +{ + WalSnd walsnds[1]; /* VARIABLE LENGTH ARRAY */ +} WalSndCtlData; + +extern WalSndCtlData *WalSndCtl; + +/* global state */ +extern bool am_walsender; + +/* user-settable parameters */ +extern int WalSndDelay; + +extern int WalSenderMain(void); +extern void WalSndSignals(void); +extern Size WalSndShmemSize(void); +extern void WalSndShmemInit(void); +extern XLogRecPtr GetOldestWALSendPointer(void); + +#endif /* _WALSENDER_H */ diff --git a/src/include/storage/pmsignal.h b/src/include/storage/pmsignal.h index 4db0f8c1db9..75ef17a5a0a 100644 --- a/src/include/storage/pmsignal.h +++ b/src/include/storage/pmsignal.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/storage/pmsignal.h,v 1.27 2010/01/02 16:58:08 momjian Exp $ + * $PostgreSQL: pgsql/src/include/storage/pmsignal.h,v 1.28 2010/01/15 09:19:09 heikki Exp $ * *------------------------------------------------------------------------- */ @@ -29,6 +29,8 @@ typedef enum PMSIGNAL_ROTATE_LOGFILE, /* send SIGUSR1 to syslogger to rotate logfile */ PMSIGNAL_START_AUTOVAC_LAUNCHER, /* start an autovacuum launcher */ PMSIGNAL_START_AUTOVAC_WORKER, /* start an autovacuum worker */ + PMSIGNAL_START_WALRECEIVER, /* start a walreceiver */ + PMSIGNAL_SHUTDOWN_WALRECEIVER, /* shut down a walreceiver */ NUM_PMSIGNALS /* Must be last value of enum! */ } PMSignalReason; @@ -45,7 +47,9 @@ extern void SendPostmasterSignal(PMSignalReason reason); extern bool CheckPostmasterSignal(PMSignalReason reason); extern int AssignPostmasterChildSlot(void); extern bool ReleasePostmasterChildSlot(int slot); +extern bool IsPostmasterChildWalSender(int slot); extern void MarkPostmasterChildActive(void); +extern void MarkPostmasterChildWalSender(void); extern void MarkPostmasterChildInactive(void); extern bool PostmasterIsAlive(bool amDirectChild); diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index ba725d8fa81..cee02c359d7 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/storage/proc.h,v 1.116 2010/01/02 16:58:08 momjian Exp $ + * $PostgreSQL: pgsql/src/include/storage/proc.h,v 1.117 2010/01/15 09:19:09 heikki Exp $ * *------------------------------------------------------------------------- */ @@ -152,10 +152,10 @@ typedef struct PROC_HDR * ie things that aren't full-fledged backends but need shmem access. * * Background writer and WAL writer run during normal operation. Startup - * process also consumes one slot, but WAL writer is launched only after - * startup has exited, so we only need 2 slots. + * process and WAL receiver also consume 2 slots, but WAL writer is + * launched only after startup has exited, so we only need 3 slots. */ -#define NUM_AUXILIARY_PROCS 2 +#define NUM_AUXILIARY_PROCS 3 /* configurable options */ diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h index 4815bc08703..bb96b0e546e 100644 --- a/src/include/utils/guc_tables.h +++ b/src/include/utils/guc_tables.h @@ -7,7 +7,7 @@ * * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * - * $PostgreSQL: pgsql/src/include/utils/guc_tables.h,v 1.47 2010/01/02 16:58:10 momjian Exp $ + * $PostgreSQL: pgsql/src/include/utils/guc_tables.h,v 1.48 2010/01/15 09:19:09 heikki Exp $ * *------------------------------------------------------------------------- */ @@ -53,6 +53,7 @@ enum config_group WAL, WAL_SETTINGS, WAL_CHECKPOINTS, + WAL_REPLICATION, QUERY_TUNING, QUERY_TUNING_METHOD, QUERY_TUNING_COST, |