Merge branch 'master' into fdw_tablefdw_table

4 files changed, 97 insertions, 61 deletions

diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c
index a5150363f9..0bd1865068 100644
--- a/src/backend/access/heap/rewriteheap.c
+++ b/src/backend/access/heap/rewriteheap.c
@@ -254,8 +254,6 @@ end_heap_rewrite(RewriteState state)
 	/*
 	 * Write any remaining tuples in the UnresolvedTups table. If we have any
 	 * left, they should in fact be dead, but let's err on the safe side.
-	 *
-	 * XXX this really is a waste of code no?
 	 */
 	hash_seq_init(&seq_status, state->rs_unresolved_tups);
 
@@ -502,8 +500,12 @@ rewrite_heap_tuple(RewriteState state,
  * Register a dead tuple with an ongoing rewrite. Dead tuples are not
  * copied to the new table, but we still make note of them so that we
  * can release some resources earlier.
+ *
+ * Returns true if a tuple was removed from the unresolved_tups table.
+ * This indicates that that tuple, previously thought to be "recently dead",
+ * is now known really dead and won't be written to the output.
  */
-void
+bool
 rewrite_heap_dead_tuple(RewriteState state, HeapTuple old_tuple)
 {
 	/*
@@ -539,7 +541,10 @@ rewrite_heap_dead_tuple(RewriteState state, HeapTuple old_tuple)
 		hash_search(state->rs_unresolved_tups, &hashkey,
 					HASH_REMOVE, &found);
 		Assert(found);
+		return true;
 	}
+
+	return false;
 }
 
 /*
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index f52e39fc36..bb7cd746b1 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -45,6 +45,7 @@
 #include "utils/inval.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
+#include "utils/pg_rusage.h"
 #include "utils/relcache.h"
 #include "utils/relmapper.h"
 #include "utils/snapmgr.h"
@@ -66,9 +67,9 @@ typedef struct
 
 
 static void rebuild_relation(Relation OldHeap, Oid indexOid,
-				 int freeze_min_age, int freeze_table_age);
+				 int freeze_min_age, int freeze_table_age, bool verbose);
 static void copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
-			   int freeze_min_age, int freeze_table_age,
+			   int freeze_min_age, int freeze_table_age, bool verbose,
 			   bool *pSwapToastByContent, TransactionId *pFreezeXid);
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static void reform_and_rewrite_tuple(HeapTuple tuple,
@@ -383,20 +384,9 @@ cluster_rel(Oid tableOid, Oid indexOid, bool recheck, bool verbose,
 	if (OidIsValid(indexOid))
 		check_index_is_clusterable(OldHeap, indexOid, recheck, AccessExclusiveLock);
 
-	/* Log what we're doing (this could use more effort) */
-	if (OidIsValid(indexOid))
-		ereport(verbose ? INFO : DEBUG2,
-				(errmsg("clustering \"%s.%s\"",
-						get_namespace_name(RelationGetNamespace(OldHeap)),
-						RelationGetRelationName(OldHeap))));
-	else
-		ereport(verbose ? INFO : DEBUG2,
-				(errmsg("vacuuming \"%s.%s\"",
-						get_namespace_name(RelationGetNamespace(OldHeap)),
-						RelationGetRelationName(OldHeap))));
-
 	/* rebuild_relation does all the dirty work */
-	rebuild_relation(OldHeap, indexOid, freeze_min_age, freeze_table_age);
+	rebuild_relation(OldHeap, indexOid, freeze_min_age, freeze_table_age,
+					 verbose);
 
 	/* NB: rebuild_relation does heap_close() on OldHeap */
 }
@@ -580,7 +570,7 @@ mark_index_clustered(Relation rel, Oid indexOid)
  */
 static void
 rebuild_relation(Relation OldHeap, Oid indexOid,
-				 int freeze_min_age, int freeze_table_age)
+				 int freeze_min_age, int freeze_table_age, bool verbose)
 {
 	Oid			tableOid = RelationGetRelid(OldHeap);
 	Oid			tableSpace = OldHeap->rd_rel->reltablespace;
@@ -604,7 +594,7 @@ rebuild_relation(Relation OldHeap, Oid indexOid,
 
 	/* Copy the heap data into the new table in the desired order */
 	copy_heap_data(OIDNewHeap, tableOid, indexOid,
-				   freeze_min_age, freeze_table_age,
+				   freeze_min_age, freeze_table_age, verbose,
 				   &swap_toast_by_content, &frozenXid);
 
 	/*
@@ -746,7 +736,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace)
  */
 static void
 copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
-			   int freeze_min_age, int freeze_table_age,
+			   int freeze_min_age, int freeze_table_age, bool verbose,
 			   bool *pSwapToastByContent, TransactionId *pFreezeXid)
 {
 	Relation	NewHeap,
@@ -766,6 +756,13 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 	RewriteState rwstate;
 	bool 		 use_sort;
 	Tuplesortstate *tuplesort;
+	double		num_tuples = 0,
+				tups_vacuumed = 0,
+				tups_recently_dead = 0;
+	int			elevel = verbose ? INFO : DEBUG2;
+	PGRUsage	ru0;
+
+	pg_rusage_init(&ru0);
 
 	/*
 	 * Open the relations we need.
@@ -887,6 +884,24 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 		indexScan = NULL;
 	}
 
+	/* Log what we're doing */
+	if (indexScan != NULL)
+		ereport(elevel,
+				(errmsg("clustering \"%s.%s\" using index scan on \"%s\"",
+						get_namespace_name(RelationGetNamespace(OldHeap)),
+						RelationGetRelationName(OldHeap),
+						RelationGetRelationName(OldIndex))));
+	else if (tuplesort != NULL)
+		ereport(elevel,
+				(errmsg("clustering \"%s.%s\" using sequential scan and sort",
+						get_namespace_name(RelationGetNamespace(OldHeap)),
+						RelationGetRelationName(OldHeap))));
+	else
+		ereport(elevel,
+				(errmsg("vacuuming \"%s.%s\"",
+						get_namespace_name(RelationGetNamespace(OldHeap)),
+						RelationGetRelationName(OldHeap))));
+
 	/*
 	 * Scan through the OldHeap, either in OldIndex order or sequentially;
 	 * copy each tuple into the NewHeap, or transiently to the tuplesort
@@ -930,8 +945,10 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 				/* Definitely dead */
 				isdead = true;
 				break;
-			case HEAPTUPLE_LIVE:
 			case HEAPTUPLE_RECENTLY_DEAD:
+				tups_recently_dead += 1;
+				/* fall through */
+			case HEAPTUPLE_LIVE:
 				/* Live or recently dead, must copy it */
 				isdead = false;
 				break;
@@ -963,6 +980,7 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 					elog(WARNING, "concurrent delete in progress within table \"%s\"",
 						 RelationGetRelationName(OldHeap));
 				/* treat as recently dead */
+				tups_recently_dead += 1;
 				isdead = false;
 				break;
 			default:
@@ -975,11 +993,18 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 
 		if (isdead)
 		{
+			tups_vacuumed += 1;
 			/* heap rewrite module still needs to see it... */
-			rewrite_heap_dead_tuple(rwstate, tuple);
+			if (rewrite_heap_dead_tuple(rwstate, tuple))
+			{
+				/* A previous recently-dead tuple is now known dead */
+				tups_vacuumed += 1;
+				tups_recently_dead -= 1;
+			}
 			continue;
 		}
 
+		num_tuples += 1;
 		if (tuplesort != NULL)
 			tuplesort_putheaptuple(tuplesort, tuple);
 		else
@@ -1031,6 +1056,17 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 	/* Reset rd_toastoid just to be tidy --- it shouldn't be looked at again */
 	NewHeap->rd_toastoid = InvalidOid;
 
+	/* Log what we did */
+	ereport(elevel,
+			(errmsg("\"%s\": found %.0f removable, %.0f nonremovable row versions in %u pages",
+					RelationGetRelationName(OldHeap),
+					tups_vacuumed, num_tuples,
+					RelationGetNumberOfBlocks(OldHeap)),
+			 errdetail("%.0f dead row versions cannot be removed yet.\n"
+					   "%s.",
+					   tups_recently_dead,
+					   pg_rusage_show(&ru0))));
+
 	/* Clean up */
 	pfree(values);
 	pfree(isnull);
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 0013703250..dae73f15af 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -120,9 +120,9 @@
 
 
 /* sort-type codes for sort__start probes */
-#define HEAP_SORT	0
-#define INDEX_SORT	1
-#define DATUM_SORT	2
+#define HEAP_SORT		0
+#define INDEX_SORT		1
+#define DATUM_SORT		2
 #define CLUSTER_SORT	3
 
 /* GUC variables */
@@ -435,6 +435,13 @@ struct Tuplesortstate
  * a lot better than what we were doing before 7.3.
  */
 
+/* When using this macro, beware of double evaluation of len */
+#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+	do { \
+		if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+			elog(ERROR, "unexpected end of data"); \
+	} while(0)
+
 
 static Tuplesortstate *tuplesort_begin_common(int workMem, bool randomAccess);
 static void puttuple_common(Tuplesortstate *state, SortTuple *tuple);
@@ -2576,8 +2583,8 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
 {
 	unsigned int len;
 
-	if (LogicalTapeRead(state->tapeset, tapenum, (void *) &len,
-						sizeof(len)) != sizeof(len))
+	if (LogicalTapeRead(state->tapeset, tapenum,
+						&len, sizeof(len)) != sizeof(len))
 		elog(ERROR, "unexpected end of tape");
 	if (len == 0 && !eofOK)
 		elog(ERROR, "unexpected end of data");
@@ -2810,14 +2817,11 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
 	USEMEM(state, GetMemoryChunkSpace(tuple));
 	/* read in the tuple proper */
 	tuple->t_len = tuplen;
-	if (LogicalTapeRead(state->tapeset, tapenum,
-						(void *) tupbody,
-						tupbodylen) != (size_t) tupbodylen)
-		elog(ERROR, "unexpected end of data");
+	LogicalTapeReadExact(state->tapeset, tapenum,
+						 tupbody, tupbodylen);
 	if (state->randomAccess)	/* need trailing length word? */
-		if (LogicalTapeRead(state->tapeset, tapenum, (void *) &tuplen,
-							sizeof(tuplen)) != sizeof(tuplen))
-			elog(ERROR, "unexpected end of data");
+		LogicalTapeReadExact(state->tapeset, tapenum,
+							 &tuplen, sizeof(tuplen));
 	stup->tuple = (void *) tuple;
 	/* set up first-column key value */
 	htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -2998,20 +3002,16 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
 	/* Reconstruct the HeapTupleData header */
 	tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
 	tuple->t_len = t_len;
-	if (LogicalTapeRead(state->tapeset, tapenum,
-						&tuple->t_self,
-						sizeof(ItemPointerData)) != sizeof(ItemPointerData))
-		elog(ERROR, "unexpected end of data");
+	LogicalTapeReadExact(state->tapeset, tapenum,
+						 &tuple->t_self, sizeof(ItemPointerData));
 	/* We don't currently bother to reconstruct t_tableOid */
 	tuple->t_tableOid = InvalidOid;
 	/* Read in the tuple body */
-	if (LogicalTapeRead(state->tapeset, tapenum,
-						tuple->t_data, tuple->t_len) != tuple->t_len)
-		elog(ERROR, "unexpected end of data");
+	LogicalTapeReadExact(state->tapeset, tapenum,
+						 tuple->t_data, tuple->t_len);
 	if (state->randomAccess)	/* need trailing length word? */
-		if (LogicalTapeRead(state->tapeset, tapenum, &tuplen,
-							sizeof(tuplen)) != sizeof(tuplen))
-			elog(ERROR, "unexpected end of data");
+		LogicalTapeReadExact(state->tapeset, tapenum,
+							 &tuplen, sizeof(tuplen));
 	stup->tuple = (void *) tuple;
 	/* set up first-column key value, if it's a simple column */
 	if (state->indexInfo->ii_KeyAttrNumbers[0] != 0)
@@ -3243,13 +3243,11 @@ readtup_index(Tuplesortstate *state, SortTuple *stup,
 	IndexTuple	tuple = (IndexTuple) palloc(tuplen);
 
 	USEMEM(state, GetMemoryChunkSpace(tuple));
-	if (LogicalTapeRead(state->tapeset, tapenum, (void *) tuple,
-						tuplen) != tuplen)
-		elog(ERROR, "unexpected end of data");
+	LogicalTapeReadExact(state->tapeset, tapenum,
+						 tuple, tuplen);
 	if (state->randomAccess)	/* need trailing length word? */
-		if (LogicalTapeRead(state->tapeset, tapenum, (void *) &tuplen,
-							sizeof(tuplen)) != sizeof(tuplen))
-			elog(ERROR, "unexpected end of data");
+		LogicalTapeReadExact(state->tapeset, tapenum,
+							 &tuplen, sizeof(tuplen));
 	stup->tuple = (void *) tuple;
 	/* set up first-column key value */
 	stup->datum1 = index_getattr(tuple,
@@ -3357,9 +3355,8 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
 	else if (state->datumTypeByVal)
 	{
 		Assert(tuplen == sizeof(Datum));
-		if (LogicalTapeRead(state->tapeset, tapenum, (void *) &stup->datum1,
-							tuplen) != tuplen)
-			elog(ERROR, "unexpected end of data");
+		LogicalTapeReadExact(state->tapeset, tapenum,
+							 &stup->datum1, tuplen);
 		stup->isnull1 = false;
 		stup->tuple = NULL;
 	}
@@ -3367,9 +3364,8 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
 	{
 		void	   *raddr = palloc(tuplen);
 
-		if (LogicalTapeRead(state->tapeset, tapenum, raddr,
-							tuplen) != tuplen)
-			elog(ERROR, "unexpected end of data");
+		LogicalTapeReadExact(state->tapeset, tapenum,
+							 raddr, tuplen);
 		stup->datum1 = PointerGetDatum(raddr);
 		stup->isnull1 = false;
 		stup->tuple = raddr;
@@ -3377,9 +3373,8 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
 	}
 
 	if (state->randomAccess)	/* need trailing length word? */
-		if (LogicalTapeRead(state->tapeset, tapenum, (void *) &tuplen,
-							sizeof(tuplen)) != sizeof(tuplen))
-			elog(ERROR, "unexpected end of data");
+		LogicalTapeReadExact(state->tapeset, tapenum,
+							 &tuplen, sizeof(tuplen));
 }
 
 static void
diff --git a/src/include/access/rewriteheap.h b/src/include/access/rewriteheap.h
index b89dc66676..db51d7cfaf 100644
--- a/src/include/access/rewriteheap.h
+++ b/src/include/access/rewriteheap.h
@@ -25,6 +25,6 @@ extern RewriteState begin_heap_rewrite(Relation NewHeap,
 extern void end_heap_rewrite(RewriteState state);
 extern void rewrite_heap_tuple(RewriteState state, HeapTuple oldTuple,
 				   HeapTuple newTuple);
-extern void rewrite_heap_dead_tuple(RewriteState state, HeapTuple oldTuple);
+extern bool rewrite_heap_dead_tuple(RewriteState state, HeapTuple oldTuple);
 
 #endif   /* REWRITE_HEAP_H */