Introduce CompactAttribute array in TupleDesc, take 2

The new compact_attrs array stores a few select fields from
FormData_pg_attribute in a more compact way, using only 16 bytes per
column instead of the 104 bytes that FormData_pg_attribute uses.  Using
CompactAttribute allows performance-critical operations such as tuple
deformation to be performed without looking at the FormData_pg_attribute
element in TupleDesc which means fewer cacheline accesses.

For some workloads, tuple deformation can be the most CPU intensive part
of processing the query.  Some testing with 16 columns on a table
where the first column is variable length showed around a 10% increase in
transactions per second for an OLAP type query performing aggregation on
the 16th column.  However, in certain cases, the increases were much
higher, up to ~25% on one AMD Zen4 machine.

This also makes pg_attribute.attcacheoff redundant.  A follow-on commit
will remove it, thus shrinking the FormData_pg_attribute struct by 4
bytes.

Author: David Rowley
Reviewed-by: Andres Freund, Victor Yegorov
Discussion: https://postgr.es/m/CAApHDvrBztXP3yx=NKNmo3xwFAFhEdyPnvrDg3=M0RhDs+4vYw@mail.gmail.com

4 files changed, 126 insertions, 17 deletions

diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h
index 5e38ef86969..0d1adff540f 100644
--- a/src/include/access/htup_details.h
+++ b/src/include/access/htup_details.h
@@ -758,9 +758,9 @@ fastgetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc, bool *isnull)
 	*isnull = false;
 	if (HeapTupleNoNulls(tup))
 	{
-		Form_pg_attribute att;
+		CompactAttribute *att;
 
-		att = TupleDescAttr(tupleDesc, attnum - 1);
+		att = TupleDescCompactAttr(tupleDesc, attnum - 1);
 		if (att->attcacheoff >= 0)
 			return fetchatt(att, (char *) tup->t_data + tup->t_data->t_hoff +
 							att->attcacheoff);
diff --git a/src/include/access/itup.h b/src/include/access/itup.h
index 94885751e59..4393b19a7fd 100644
--- a/src/include/access/itup.h
+++ b/src/include/access/itup.h
@@ -124,11 +124,13 @@ index_getattr(IndexTuple tup, int attnum, TupleDesc tupleDesc, bool *isnull)
 
 	if (!IndexTupleHasNulls(tup))
 	{
-		if (TupleDescAttr(tupleDesc, attnum - 1)->attcacheoff >= 0)
+		CompactAttribute *attr = TupleDescCompactAttr(tupleDesc, attnum - 1);
+
+		if (attr->attcacheoff >= 0)
 		{
-			return fetchatt(TupleDescAttr(tupleDesc, attnum - 1),
-							(char *) tup + IndexInfoFindDataOffset(tup->t_info)
-							+ TupleDescAttr(tupleDesc, attnum - 1)->attcacheoff);
+			return fetchatt(attr,
+							(char *) tup + IndexInfoFindDataOffset(tup->t_info) +
+							attr->attcacheoff);
 		}
 		else
 			return nocache_index_getattr(tup, attnum, tupleDesc);
diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h
index 8930a28d660..4406617fea6 100644
--- a/src/include/access/tupdesc.h
+++ b/src/include/access/tupdesc.h
@@ -46,6 +46,39 @@ typedef struct TupleConstr
 } TupleConstr;
 
 /*
+ * CompactAttribute
+ *		Cut-down version of FormData_pg_attribute for faster access for tasks
+ *		such as tuple deformation.  The fields of this struct are populated
+ *		using the populate_compact_attribute() function, which must be called
+ *		directly after the FormData_pg_attribute struct is populated or
+ *		altered in any way.
+ *
+ * Currently, this struct is 16 bytes.  Any code changes which enlarge this
+ * struct should be considered very carefully.
+ *
+ * Code which must access a TupleDesc's attribute data should always make use
+ * the fields of this struct when required fields are available here.  It's
+ * more efficient to access the memory in CompactAttribute due to it being a
+ * more compact representation of FormData_pg_attribute and also because
+ * accessing the FormData_pg_attribute requires an additional calculations to
+ * obtain the base address of the array within the TupleDesc.
+ */
+typedef struct CompactAttribute
+{
+	int32		attcacheoff;	/* fixed offset into tuple, if known, or -1 */
+	int16		attlen;			/* attr len in bytes or -1 = varlen, -2 =
+								 * cstring */
+	bool		attbyval;		/* as FormData_pg_attribute.attbyval */
+	bool		attispackable;	/* FormData_pg_attribute.attstorage !=
+								 * TYPSTORAGE_PLAIN */
+	bool		atthasmissing;	/* as FormData_pg_attribute.atthasmissing */
+	bool		attisdropped;	/* as FormData_pg_attribute.attisdropped */
+	bool		attgenerated;	/* FormData_pg_attribute.attgenerated != '\0' */
+	bool		attnotnull;		/* as FormData_pg_attribute.attnotnull */
+	char		attalign;		/* alignment requirement */
+} CompactAttribute;
+
+/*
  * This struct is passed around within the backend to describe the structure
  * of tuples.  For tuples coming from on-disk relations, the information is
  * collected from the pg_attribute, pg_attrdef, and pg_constraint catalogs.
@@ -75,6 +108,21 @@ typedef struct TupleConstr
  * context and go away when the context is freed.  We set the tdrefcount
  * field of such a descriptor to -1, while reference-counted descriptors
  * always have tdrefcount >= 0.
+ *
+ * Beyond the compact_attrs variable length array, the TupleDesc stores an
+ * array of FormData_pg_attribute.  The TupleDescAttr() function, as defined
+ * below, takes care of calculating the address of the elements of the
+ * FormData_pg_attribute array.
+ *
+ * The array of CompactAttribute is effectively an abbreviated version of the
+ * array of FormData_pg_attribute.  Because CompactAttribute is significantly
+ * smaller than FormData_pg_attribute, code, especially performance-critical
+ * code, should prioritize using the fields from the CompactAttribute over the
+ * equivalent fields in FormData_pg_attribute.
+ *
+ * Any code making changes manually to and fields in the FormData_pg_attribute
+ * array must subsequently call populate_compact_attribute() to flush the
+ * changes out to the corresponding 'compact_attrs' element.
  */
 typedef struct TupleDescData
 {
@@ -83,13 +131,68 @@ typedef struct TupleDescData
 	int32		tdtypmod;		/* typmod for tuple type */
 	int			tdrefcount;		/* reference count, or -1 if not counting */
 	TupleConstr *constr;		/* constraints, or NULL if none */
-	/* attrs[N] is the description of Attribute Number N+1 */
-	FormData_pg_attribute attrs[FLEXIBLE_ARRAY_MEMBER];
+	/* compact_attrs[N] is the compact metadata of Attribute Number N+1 */
+	CompactAttribute compact_attrs[FLEXIBLE_ARRAY_MEMBER];
 }			TupleDescData;
 typedef struct TupleDescData *TupleDesc;
 
-/* Accessor for the i'th attribute of tupdesc. */
-#define TupleDescAttr(tupdesc, i) (&(tupdesc)->attrs[(i)])
+extern void populate_compact_attribute(TupleDesc tupdesc, int attnum);
+
+/*
+ * Calculates the base address of the Form_pg_attribute at the end of the
+ * TupleDescData struct.
+ */
+#define TupleDescAttrAddress(desc) \
+	(Form_pg_attribute) ((char *) (desc) + \
+	 (offsetof(struct TupleDescData, compact_attrs) + \
+	 (desc)->natts * sizeof(CompactAttribute)))
+
+/* Accessor for the i'th FormData_pg_attribute element of tupdesc. */
+static inline FormData_pg_attribute *
+TupleDescAttr(TupleDesc tupdesc, int i)
+{
+	FormData_pg_attribute *attrs = TupleDescAttrAddress(tupdesc);
+
+	return &attrs[i];
+}
+
+#undef TupleDescAttrAddress
+
+/*
+ * Accessor for the i'th CompactAttribute element of tupdesc.
+ */
+static inline CompactAttribute *
+TupleDescCompactAttr(TupleDesc tupdesc, int i)
+{
+	CompactAttribute *cattr = &tupdesc->compact_attrs[i];
+#ifdef USE_ASSERT_CHECKING
+	CompactAttribute snapshot;
+
+	/*
+	 * In Assert enabled builds we verify that the CompactAttribute is
+	 * populated correctly.  This helps find bugs in places such as ALTER
+	 * TABLE where code makes changes to the FormData_pg_attribute but forgets
+	 * to call populate_compact_attribute.
+	 */
+
+	/*
+	 * Take a snapshot of how the CompactAttribute is now before calling
+	 * populate_compact_attribute to make it up-to-date with the
+	 * FormData_pg_attribute.
+	 */
+	memcpy(&snapshot, cattr, sizeof(CompactAttribute));
+
+	populate_compact_attribute(tupdesc, i);
+
+	/* reset attcacheoff back to what it was */
+	cattr->attcacheoff = snapshot.attcacheoff;
+
+	/* Ensure the snapshot matches the freshly populated CompactAttribute */
+	Assert(memcmp(&snapshot, cattr, sizeof(CompactAttribute)) == 0);
+#endif
+
+	return cattr;
+}
 
 extern TupleDesc CreateTemplateTupleDesc(int natts);
 
@@ -97,10 +200,13 @@ extern TupleDesc CreateTupleDesc(int natts, Form_pg_attribute *attrs);
 
 extern TupleDesc CreateTupleDescCopy(TupleDesc tupdesc);
 
+extern TupleDesc CreateTupleDescTruncatedCopy(TupleDesc tupdesc, int natts);
+
 extern TupleDesc CreateTupleDescCopyConstr(TupleDesc tupdesc);
 
 #define TupleDescSize(src) \
-	(offsetof(struct TupleDescData, attrs) + \
+	(offsetof(struct TupleDescData, compact_attrs) + \
+	 (src)->natts * sizeof(CompactAttribute) + \
 	 (src)->natts * sizeof(FormData_pg_attribute))
 
 extern void TupleDescCopy(TupleDesc dst, TupleDesc src);
diff --git a/src/include/access/tupmacs.h b/src/include/access/tupmacs.h
index 58b3a58cfd0..fcf09ed95f4 100644
--- a/src/include/access/tupmacs.h
+++ b/src/include/access/tupmacs.h
@@ -30,13 +30,14 @@ att_isnull(int ATT, const bits8 *BITS)
 
 #ifndef FRONTEND
 /*
- * Given a Form_pg_attribute and a pointer into a tuple's data area,
- * return the correct value or pointer.
+ * Given an attbyval and an attlen from either a Form_pg_attribute or
+ * CompactAttribute and a pointer into a tuple's data area, return the
+ * correct value or pointer.
  *
- * We return a Datum value in all cases.  If the attribute has "byval" false,
- * we return the same pointer into the tuple data area that we're passed.
- * Otherwise, we return the correct number of bytes fetched from the data
- * area and extended to Datum form.
+ * We return a Datum value in all cases.  If attbyval is false,  we return the
+ * same pointer into the tuple data area that we're passed.  Otherwise, we
+ * return the correct number of bytes fetched from the data area and extended
+ * to Datum form.
  *
  * On machines where Datum is 8 bytes, we support fetching 8-byte byval
  * attributes; otherwise, only 1, 2, and 4-byte values are supported.