VACUUM: ignore indexing operations with CONCURRENTLY

As envisioned in commit c98763bf51bf, it is possible for VACUUM to
ignore certain transactions that are executing CREATE INDEX CONCURRENTLY
and REINDEX CONCURRENTLY for the purposes of computing Xmin; that's
because we know those transactions are not going to examine any other
tables, and are not going to execute anything else in the same
transaction.  (Only operations on "safe" indexes can be ignored: those
on indexes that are neither partial nor expressional).

This is extremely useful in cases where CIC/RC can run for a very long
time, because that used to be a significant headache for concurrent
vacuuming of other tables.

Reviewed-by: Matthias van de Meent <boekewurm+postgres@gmail.com>
Reviewed-by: Masahiko Sawada <sawada.mshk@gmail.com>
Discussion: https://postgr.es/m/20210115133858.GA18931@alvherre.pgsql

diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index a5271a9f8f2e20fc0e3e6b63a29a529999dfb80d..965dcf472cacc4775c1cb691c5afc7c83f6e3619 100644 (file)
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -855,6 +855,8 @@ Indexes:
    Like any long-running transaction, <command>CREATE INDEX</command> on a
    table can affect which tuples can be removed by concurrent
    <command>VACUUM</command> on any other table.
+   Excepted from this are operations with the <literal>CONCURRENTLY</literal>
+   option for indexes that are not partial and do not index any expressions.
   </para>
 
   <para>

diff --git a/doc/src/sgml/ref/reindex.sgml b/doc/src/sgml/ref/reindex.sgml
index 07795b573720d66965e04d3c44597075d56b120c..b22d39eba929e38080b954687c658f2e65524122 100644 (file)
--- a/doc/src/sgml/ref/reindex.sgml
+++ b/doc/src/sgml/ref/reindex.sgml
@@ -478,6 +478,8 @@ Indexes:
     Like any long-running transaction, <command>REINDEX</command> on a table
     can affect which tuples can be removed by concurrent
     <command>VACUUM</command> on any other table.
+    Excepted from this are operations with the <literal>CONCURRENTLY</literal>
+    option for indexes that are not partial and do not index any expressions.
    </para>
 
    <para>

diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index a5daea895713913d56c6cb69843397d5724cfefc..d736d06d2886dd56e4ad27fc5ee8d58236d0882a 100644 (file)
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -1610,7 +1610,13 @@ TransactionIdIsActive(TransactionId xid)
  * relations that's not required, since only backends in my own database could
  * ever see the tuples in them. Also, we can ignore concurrently running lazy
  * VACUUMs because (a) they must be working on other tables, and (b) they
- * don't need to do snapshot-based lookups.
+ * don't need to do snapshot-based lookups.  Similarly, for the non-catalog
+ * horizon, we can ignore CREATE INDEX CONCURRENTLY and REINDEX CONCURRENTLY
+ * when they are working on non-partial, non-expressional indexes, for the
+ * same reasons and because they can't run in transaction blocks.  (They are
+ * not possible to ignore for catalogs, because CIC and RC do some catalog
+ * operations.)  Do note that this means that CIC and RC must use a lock level
+ * that conflicts with VACUUM.
  *
  * This also computes a horizon used to truncate pg_subtrans. For that
  * backends in all databases have to be considered, and concurrently running
@@ -1660,9 +1666,6 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
    bool        in_recovery = RecoveryInProgress();
    TransactionId *other_xids = ProcGlobal->xids;
 
-   /* inferred after ProcArrayLock is released */
-   h->catalog_oldest_nonremovable = InvalidTransactionId;
-
    LWLockAcquire(ProcArrayLock, LW_SHARED);
 
    h->latest_completed = ShmemVariableCache->latestCompletedXid;
@@ -1682,6 +1685,7 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
 
        h->oldest_considered_running = initial;
        h->shared_oldest_nonremovable = initial;
+       h->catalog_oldest_nonremovable = initial;
        h->data_oldest_nonremovable = initial;
 
        /*
@@ -1752,7 +1756,7 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
        if (statusFlags & (PROC_IN_VACUUM | PROC_IN_LOGICAL_DECODING))
            continue;
 
-       /* shared tables need to take backends in all database into account */
+       /* shared tables need to take backends in all databases into account */
        h->shared_oldest_nonremovable =
            TransactionIdOlder(h->shared_oldest_nonremovable, xmin);
 
@@ -1773,11 +1777,26 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
            MyDatabaseId == InvalidOid || proc->databaseId == MyDatabaseId ||
            proc->databaseId == 0)  /* always include WalSender */
        {
-           h->data_oldest_nonremovable =
-               TransactionIdOlder(h->data_oldest_nonremovable, xmin);
+           /*
+            * We can ignore this backend if it's running CREATE INDEX
+            * CONCURRENTLY or REINDEX CONCURRENTLY on a "safe" index -- but
+            * only on vacuums of user-defined tables.
+            */
+           if (!(statusFlags & PROC_IN_SAFE_IC))
+               h->data_oldest_nonremovable =
+                   TransactionIdOlder(h->data_oldest_nonremovable, xmin);
+
+           /* Catalog tables need to consider all backends in this db */
+           h->catalog_oldest_nonremovable =
+               TransactionIdOlder(h->catalog_oldest_nonremovable, xmin);
+
        }
    }
 
+   /* catalog horizon should never be later than data */
+   Assert(TransactionIdPrecedesOrEquals(h->catalog_oldest_nonremovable,
+                                        h->data_oldest_nonremovable));
+
    /*
     * If in recovery fetch oldest xid in KnownAssignedXids, will be applied
     * after lock is released.
@@ -1799,6 +1818,8 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
            TransactionIdOlder(h->shared_oldest_nonremovable, kaxmin);
        h->data_oldest_nonremovable =
            TransactionIdOlder(h->data_oldest_nonremovable, kaxmin);
+       h->catalog_oldest_nonremovable =
+           TransactionIdOlder(h->catalog_oldest_nonremovable, kaxmin);
        /* temp relations cannot be accessed in recovery */
    }
    else
@@ -1825,6 +1846,9 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
        h->data_oldest_nonremovable =
            TransactionIdRetreatedBy(h->data_oldest_nonremovable,
                                     vacuum_defer_cleanup_age);
+       h->catalog_oldest_nonremovable =
+           TransactionIdRetreatedBy(h->catalog_oldest_nonremovable,
+                                    vacuum_defer_cleanup_age);
        /* defer doesn't apply to temp relations */
    }
 
@@ -1847,7 +1871,9 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
    h->shared_oldest_nonremovable =
        TransactionIdOlder(h->shared_oldest_nonremovable,
                           h->slot_catalog_xmin);
-   h->catalog_oldest_nonremovable = h->data_oldest_nonremovable;
+   h->catalog_oldest_nonremovable =
+       TransactionIdOlder(h->catalog_oldest_nonremovable,
+                          h->slot_xmin);
    h->catalog_oldest_nonremovable =
        TransactionIdOlder(h->catalog_oldest_nonremovable,
                           h->slot_catalog_xmin);

diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 260ec7b97e9a9f28e032b50c21e3be24fdfeb4d8..d626731723bdc3498241b7bee1b07682c1e07e4c 100644 (file)
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2583,7 +2583,7 @@ static struct config_int ConfigureNamesInt[] =
            NULL
        },
        &vacuum_defer_cleanup_age,
-       0, 0, 1000000,
+       0, 0, 1000000,      /* see ComputeXidHorizons */
        NULL, NULL, NULL
    },