Track total number of WAL records, FPIs and bytes generated in the cluster.

Commit 6b466bf5f2 allowed pg_stat_statements to track the number of
WAL records, full page images and bytes that each statement generated.
Similarly this commit allows us to track the cluster-wide WAL statistics
counters.

New columns wal_records, wal_fpi and wal_bytes are added into the
pg_stat_wal view, and reports the total number of WAL records,
full page images and bytes generated in the , respectively.

Author: Masahiro Ikeda
Reviewed-by: Amit Kapila, Movead Li, Kyotaro Horiguchi, Fujii Masao
Discussion: https://postgr.es/m/35ef960128b90bfae3b3fdf60a3a860f@oss.nttdata.com

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index cc48c30ef30cf0cdce357c6072c3f1fa7a8255e3..52a69a536604a517f0b7f186086239c91d3e8b23 100644 (file)
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -3447,6 +3447,33 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
     </thead>
 
     <tbody>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>wal_records</structfield> <type>bigint</type>
+      </para>
+      <para>
+       Total number of WAL records generated
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>wal_fpi</structfield> <type>bigint</type>
+      </para>
+      <para>
+       Total number of WAL full page images generated
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>wal_bytes</structfield> <type>numeric</type>
+      </para>
+      <para>
+       Total amount of WAL bytes generated
+      </para></entry>
+     </row>
+
      <row>
       <entry role="catalog_table_entry"><para role="column_definition">
        <structfield>wal_buffers_full</structfield> <type>bigint</type>

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2e4aa1c4b6606710728d70177c92f735de8763f5..b140c210bc7995e5b71bc805daa4e616a88cf72e 100644 (file)
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -993,6 +993,9 @@ CREATE VIEW pg_stat_bgwriter AS
 
 CREATE VIEW pg_stat_wal AS
     SELECT
+        w.wal_records,
+        w.wal_fpi,
+        w.wal_bytes,
         w.wal_buffers_full,
         w.stats_reset
     FROM pg_stat_get_wal() w;

diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index e76e627c6b2b746c7e8ef94c2e820c5c15ca5259..9bad14981b861f32f36bddf53b72fdd2761f3052 100644 (file)
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -41,6 +41,7 @@
 #include "catalog/pg_database.h"
 #include "catalog/pg_proc.h"
 #include "common/ip.h"
+#include "executor/instrument.h"
 #include "libpq/libpq.h"
 #include "libpq/pqsignal.h"
 #include "mb/pg_wchar.h"
@@ -143,6 +144,14 @@ char          *pgstat_stat_tmpname = NULL;
 PgStat_MsgBgWriter BgWriterStats;
 PgStat_MsgWal WalStats;
 
+/*
+ * WAL usage counters saved from pgWALUsage at the previous call to
+ * pgstat_send_wal(). This is used to calculate how much WAL usage
+ * happens between pgstat_send_wal() calls, by substracting
+ * the previous counters from the current ones.
+ */
+static WalUsage prevWalUsage;
+
 /*
  * List of SLRU names that we keep stats for.  There is no central registry of
  * SLRUs, so we use this fixed list instead.  The "other" entry is used for
@@ -3048,6 +3057,13 @@ pgstat_initialize(void)
                MyBEEntry = &BackendStatusArray[MaxBackends + MyAuxProcType];
        }
 
+       /*
+        * Initialize prevWalUsage with pgWalUsage so that pgstat_send_wal() can
+        * calculate how much pgWalUsage counters are increased by substracting
+        * prevWalUsage from pgWalUsage.
+        */
+       prevWalUsage = pgWalUsage;
+
        /* Set up a process-exit hook to clean up */
        on_shmem_exit(pgstat_beshutdown_hook, 0);
 }
@@ -4577,6 +4593,20 @@ pgstat_send_wal(void)
        /* We assume this initializes to zeroes */
        static const PgStat_MsgWal all_zeroes;
 
+       WalUsage        walusage;
+
+       /*
+        * Calculate how much WAL usage counters are increased by substracting the
+        * previous counters from the current ones. Fill the results in WAL stats
+        * message.
+        */
+       MemSet(&walusage, 0, sizeof(WalUsage));
+       WalUsageAccumDiff(&walusage, &pgWalUsage, &prevWalUsage);
+
+       WalStats.m_wal_records = walusage.wal_records;
+       WalStats.m_wal_fpi = walusage.wal_fpi;
+       WalStats.m_wal_bytes = walusage.wal_bytes;
+
        /*
         * This function can be called even if nothing at all has happened. In
         * this case, avoid sending a completely empty message to the stats
@@ -4591,6 +4621,11 @@ pgstat_send_wal(void)
        pgstat_setheader(&WalStats.m_hdr, PGSTAT_MTYPE_WAL);
        pgstat_send(&WalStats, sizeof(WalStats));
 
+       /*
+        * Save the current counters for the subsequent calculation of WAL usage.
+        */
+       prevWalUsage = pgWalUsage;
+
        /*
         * Clear out the statistics buffer, so it can be re-used.
         */
@@ -6759,6 +6794,9 @@ pgstat_recv_bgwriter(PgStat_MsgBgWriter *msg, int len)
 static void
 pgstat_recv_wal(PgStat_MsgWal *msg, int len)
 {
+       walStats.wal_records += msg->m_wal_records;
+       walStats.wal_fpi += msg->m_wal_fpi;
+       walStats.wal_bytes += msg->m_wal_bytes;
        walStats.wal_buffers_full += msg->m_wal_buffers_full;
 }
 

diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index a210fc93b415691abc7d068bca4589dc79fc6d8b..6afe1b6f56ebcf4d96961a5590fac0a46632fe7c 100644 (file)
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -1703,10 +1703,11 @@ pg_stat_get_buf_alloc(PG_FUNCTION_ARGS)
 Datum
 pg_stat_get_wal(PG_FUNCTION_ARGS)
 {
-#define PG_STAT_GET_WAL_COLS   2
+#define PG_STAT_GET_WAL_COLS   5
        TupleDesc       tupdesc;
        Datum           values[PG_STAT_GET_WAL_COLS];
        bool            nulls[PG_STAT_GET_WAL_COLS];
+       char            buf[256];
        PgStat_WalStats *wal_stats;
 
        /* Initialise values and NULL flags arrays */
@@ -1715,9 +1716,15 @@ pg_stat_get_wal(PG_FUNCTION_ARGS)
 
        /* Initialise attributes information in the tuple descriptor */
        tupdesc = CreateTemplateTupleDesc(PG_STAT_GET_WAL_COLS);
-       TupleDescInitEntry(tupdesc, (AttrNumber) 1, "wal_buffers_full",
+       TupleDescInitEntry(tupdesc, (AttrNumber) 1, "wal_records",
                                           INT8OID, -1, 0);
-       TupleDescInitEntry(tupdesc, (AttrNumber) 2, "stats_reset",
+       TupleDescInitEntry(tupdesc, (AttrNumber) 2, "wal_fpi",
+                                          INT8OID, -1, 0);
+       TupleDescInitEntry(tupdesc, (AttrNumber) 3, "wal_bytes",
+                                          NUMERICOID, -1, 0);
+       TupleDescInitEntry(tupdesc, (AttrNumber) 4, "wal_buffers_full",
+                                          INT8OID, -1, 0);
+       TupleDescInitEntry(tupdesc, (AttrNumber) 5, "stats_reset",
                                           TIMESTAMPTZOID, -1, 0);
 
        BlessTupleDesc(tupdesc);
@@ -1726,8 +1733,18 @@ pg_stat_get_wal(PG_FUNCTION_ARGS)
        wal_stats = pgstat_fetch_stat_wal();
 
        /* Fill values and NULLs */
-       values[0] = Int64GetDatum(wal_stats->wal_buffers_full);
-       values[1] = TimestampTzGetDatum(wal_stats->stat_reset_timestamp);
+       values[0] = Int64GetDatum(wal_stats->wal_records);
+       values[1] = Int64GetDatum(wal_stats->wal_fpi);
+
+       /* Convert to numeric. */
+       snprintf(buf, sizeof buf, UINT64_FORMAT, wal_stats->wal_bytes);
+       values[2] = DirectFunctionCall3(numeric_in,
+                                                                       CStringGetDatum(buf),
+                                                                       ObjectIdGetDatum(0),
+                                                                       Int32GetDatum(-1));
+
+       values[3] = Int64GetDatum(wal_stats->wal_buffers_full);
+       values[4] = TimestampTzGetDatum(wal_stats->stat_reset_timestamp);
 
        /* Returns the record as Datum */
        PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));

diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index e7fbda9f81acc8bb71d8eeb94c3b74ce644a1695..fc2202b8436e2c9a27d473be27f99d43e4286743 100644 (file)
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5500,8 +5500,9 @@
 { oid => '1136', descr => 'statistics: information about WAL activity',
   proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's',
   proparallel => 'r', prorettype => 'record', proargtypes => '',
-  proallargtypes => '{int8,timestamptz}', proargmodes => '{o,o}',
-  proargnames => '{wal_buffers_full,stats_reset}',
+   proallargtypes => '{int8,int8,numeric,int8,timestamptz}',
+   proargmodes => '{o,o,o,o,o}',
+   proargnames => '{wal_records,wal_fpi,wal_bytes,wal_buffers_full,stats_reset}',
   prosrc => 'pg_stat_get_wal' },
 
 { oid => '2306', descr => 'statistics: information about SLRU caches',

diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index 257e515bfe75f67d92e9975cc6a8848815d07497..5954068dec534f5e837048c3e347656e7bbe2737 100644 (file)
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -459,6 +459,9 @@ typedef struct PgStat_MsgBgWriter
 typedef struct PgStat_MsgWal
 {
        PgStat_MsgHdr m_hdr;
+       PgStat_Counter m_wal_records;
+       PgStat_Counter m_wal_fpi;
+       uint64          m_wal_bytes;
        PgStat_Counter m_wal_buffers_full;
 } PgStat_MsgWal;
 
@@ -798,6 +801,9 @@ typedef struct PgStat_GlobalStats
  */
 typedef struct PgStat_WalStats
 {
+       PgStat_Counter wal_records;
+       PgStat_Counter wal_fpi;
+       uint64          wal_bytes;
        PgStat_Counter wal_buffers_full;
        TimestampTz stat_reset_timestamp;
 } PgStat_WalStats;

diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 097ff5d111f73cdd0ed55576998ed9af78850640..6293ab57bcf61fe3bad1d1a89bbd605d0e77925b 100644 (file)
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2138,9 +2138,12 @@ pg_stat_user_tables| SELECT pg_stat_all_tables.relid,
     pg_stat_all_tables.autoanalyze_count
    FROM pg_stat_all_tables
   WHERE ((pg_stat_all_tables.schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (pg_stat_all_tables.schemaname !~ '^pg_toast'::text));
-pg_stat_wal| SELECT w.wal_buffers_full,
+pg_stat_wal| SELECT w.wal_records,
+    w.wal_fpi,
+    w.wal_bytes,
+    w.wal_buffers_full,
     w.stats_reset
-   FROM pg_stat_get_wal() w(wal_buffers_full, stats_reset);
+   FROM pg_stat_get_wal() w(wal_records, wal_fpi, wal_bytes, wal_buffers_full, stats_reset);
 pg_stat_wal_receiver| SELECT s.pid,
     s.status,
     s.receive_start_lsn,