varattrib_4b design proposal to make it extended to support multiple compression algorithms.

Author: Nikhil Kumar Veldanda

contrib/amcheck/verify_heapam.c

@@ -1786,7 +1786,8 @@ check_tuple_attribute(HeapCheckContext *ctx)
 		bool		valid = false;
 
 		/* Compressed attributes should have a valid compression method */
-		cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
+		cmid = toast_get_compression_id(attr);
+
 		switch (cmid)
 		{
 				/* List of all valid compression method IDs */

src/backend/access/brin/brin_tuple.c

@@ -223,6 +223,7 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
 			{
 				Datum		cvalue;
 				char		compression;
+				CompressionInfo cmp;
 				Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc,
 													  keyno);
 
@@ -237,7 +238,8 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
 				else
 					compression = InvalidCompressionMethod;
 
-				cvalue = toast_compress_datum(value, compression);
+				cmp = setup_compression_info(compression, att);
+				cvalue = toast_compress_datum(value, cmp);
 
 				if (DatumGetPointer(cvalue) != NULL)
 				{

src/backend/access/common/detoast.c

@@ -478,7 +478,7 @@ toast_decompress_datum(struct varlena *attr)
 	 * Fetch the compression method id stored in the compression header and
 	 * decompress the data using the appropriate decompression routine.
 	 */
-	cmid = TOAST_COMPRESS_METHOD(attr);
+	cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
 	switch (cmid)
 	{
 		case TOAST_PGLZ_COMPRESSION_ID:
@@ -514,14 +514,14 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
 	 * have been seen to give wrong results if passed an output size that is
 	 * more than the data's true decompressed size.
 	 */
-	if ((uint32) slicelength >= TOAST_COMPRESS_EXTSIZE(attr))
+	if ((uint32) slicelength >= VARDATA_COMPRESSED_GET_EXTSIZE(attr))
 		return toast_decompress_datum(attr);
 
 	/*
 	 * Fetch the compression method id stored in the compression header and
 	 * decompress the data slice using the appropriate decompression routine.
 	 */
-	cmid = TOAST_COMPRESS_METHOD(attr);
+	cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
 	switch (cmid)
 	{
 		case TOAST_PGLZ_COMPRESSION_ID:

src/backend/access/common/indextuple.c

@@ -123,9 +123,10 @@ index_form_tuple_context(TupleDesc tupleDescriptor,
 			 att->attstorage == TYPSTORAGE_MAIN))
 		{
 			Datum		cvalue;
+			CompressionInfo cmp;
 
-			cvalue = toast_compress_datum(untoasted_values[i],
-										  att->attcompression);
+			cmp = setup_compression_info(att->attcompression, att);
+			cvalue = toast_compress_datum(untoasted_values[i], cmp);
 
 			if (DatumGetPointer(cvalue) != NULL)
 			{

src/backend/access/common/toast_compression.c

@@ -266,7 +266,9 @@ toast_get_compression_id(struct varlena *attr)
 
 		VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
-		if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+		if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) && VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) > TOAST_LAST_COMPRESSION_ID_BEFORE_EXT)
+			cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(detoast_external_attr(attr));
+		else
 			cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
 	}
 	else if (VARATT_IS_COMPRESSED(attr))
@@ -314,3 +316,25 @@ GetCompressionMethodName(char method)
 			return NULL;		/* keep compiler quiet */
 	}
 }
+
+CompressionInfo
+setup_compression_info(char cmethod, Form_pg_attribute att)
+{
+	CompressionInfo info;
+
+	/* initialize from the attribute’s default settings */
+	info.cmethod = cmethod;
+	info.cmp_ext = NULL;
+
+	if (!CompressionMethodIsValid(cmethod))
+		info.cmethod = default_toast_compression;
+
+	return info;
+}
+
+void
+free_compression_info(CompressionInfo *info)
+{
+	if (info->cmp_ext != NULL)
+		pfree(info->cmp_ext);
+}

src/backend/access/common/toast_internals.c

@@ -43,25 +43,22 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
  * ----------
  */
 Datum
-toast_compress_datum(Datum value, char cmethod)
+toast_compress_datum(Datum value, CompressionInfo cmp)
 {
 	struct varlena *tmp = NULL;
 	int32		valsize;
 	ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
+	varatt_cmp_extended *cmp_ext = cmp.cmp_ext;
 
 	Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value)));
 	Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
 
 	valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
 
-	/* If the compression method is not valid, use the current default */
-	if (!CompressionMethodIsValid(cmethod))
-		cmethod = default_toast_compression;
-
 	/*
 	 * Call appropriate compression routine for the compression method.
 	 */
-	switch (cmethod)
+	switch (cmp.cmethod)
 	{
 		case TOAST_PGLZ_COMPRESSION:
 			tmp = pglz_compress_datum((const struct varlena *) value);
@@ -72,11 +69,14 @@ toast_compress_datum(Datum value, char cmethod)
 			cmid = TOAST_LZ4_COMPRESSION_ID;
 			break;
 		default:
-			elog(ERROR, "invalid compression method %c", cmethod);
+			elog(ERROR, "invalid compression method %c", cmp.cmethod);
 	}
 
 	if (tmp == NULL)
+	{
+		free_compression_info(&cmp);
 		return PointerGetDatum(NULL);
+	}
 
 	/*
 	 * We recheck the actual size even if compression reports success, because
@@ -92,13 +92,15 @@ toast_compress_datum(Datum value, char cmethod)
 	{
 		/* successful compression */
 		Assert(cmid != TOAST_INVALID_COMPRESSION_ID);
-		TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid);
+		TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid, cmp_ext);
+		free_compression_info(&cmp);
 		return PointerGetDatum(tmp);
 	}
 	else
 	{
 		/* incompressible data */
 		pfree(tmp);
+		free_compression_info(&cmp);
 		return PointerGetDatum(NULL);
 	}
 }

src/backend/access/table/toast_helper.c

@@ -229,8 +229,10 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
 	Datum	   *value = &ttc->ttc_values[attribute];
 	Datum		new_value;
 	ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
+	Form_pg_attribute att = TupleDescAttr(ttc->ttc_rel->rd_att, attribute);
+	CompressionInfo cmp = setup_compression_info(attr->tai_compression, att);
 
-	new_value = toast_compress_datum(*value, attr->tai_compression);
+	new_value = toast_compress_datum(*value, cmp);
 
 	if (DatumGetPointer(new_value) != NULL)
 	{

src/include/access/toast_compression.h

@@ -13,6 +13,10 @@
 #ifndef TOAST_COMPRESSION_H
 #define TOAST_COMPRESSION_H
 
+#include "varatt.h"
+#include "toast_compression.h"
+#include "catalog/pg_attribute.h"
+
 /*
  * GUC support.
  *
@@ -23,24 +27,38 @@
 extern PGDLLIMPORT int default_toast_compression;
 
 /*
- * Built-in compression method ID.  The toast compression header will store
- * this in the first 2 bits of the raw length.  These built-in compression
- * method IDs are directly mapped to the built-in compression methods.
+ * TOAST compression methods enumeration.
+ *
+ * Each entry defines:
+ *   - NAME         : identifier for the compression algorithm
+ *   - VALUE        : numeric enum value
+ *   - METADATA type: struct type holding extra info (void when none)
  *
- * Don't use these values for anything other than understanding the meaning
- * of the raw bits from a varlena; in particular, if the goal is to identify
- * a compression method, use the constants TOAST_PGLZ_COMPRESSION, etc.
- * below. We might someday support more than 4 compression methods, but
- * we can never have more than 4 values in this enum, because there are
- * only 2 bits available in the places where this is stored.
+ * The INVALID entry is a sentinel and must remain last.
  */
+#define TOAST_COMPRESSION_LIST                                 \
+    X(PGLZ,         0,    void)    /* PostgreSQL LZ-based */   \
+    X(LZ4,          1,    void)    /* LZ4 algorithm */         \
+    X(INVALID,      2,    void) /* sentinel, must be last */
+
+
 typedef enum ToastCompressionId
 {
-	TOAST_PGLZ_COMPRESSION_ID = 0,
-	TOAST_LZ4_COMPRESSION_ID = 1,
-	TOAST_INVALID_COMPRESSION_ID = 2,
+#define X(name,val,struct) TOAST_##name##_COMPRESSION_ID = (val),
+	TOAST_COMPRESSION_LIST
+#undef X
 } ToastCompressionId;
 
+#define TOAST_LAST_COMPRESSION_ID_BEFORE_EXT  TOAST_LZ4_COMPRESSION_ID
+
+typedef struct CompressionInfo
+{
+	char		cmethod;
+	/* Extended compression meta info */
+	varatt_cmp_extended *cmp_ext;
+} CompressionInfo;
+
+
 /*
  * Built-in compression methods.  pg_attribute will store these in the
  * attcompression column.  In attcompression, InvalidCompressionMethod
@@ -69,5 +87,7 @@ extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
 extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
 extern char CompressionNameToMethod(const char *compression);
 extern const char *GetCompressionMethodName(char method);
+extern CompressionInfo setup_compression_info(char cmethod, Form_pg_attribute att);
+extern void free_compression_info(CompressionInfo *info);
 
 #endif							/* TOAST_COMPRESSION_H */

src/include/access/toast_internals.h

@@ -31,21 +31,26 @@ typedef struct toast_compress_header
  * Utilities for manipulation of header information for compressed
  * toast entries.
  */
-#define TOAST_COMPRESS_EXTSIZE(ptr) \
-	(((toast_compress_header *) (ptr))->tcinfo & VARLENA_EXTSIZE_MASK)
-#define TOAST_COMPRESS_METHOD(ptr) \
-	(((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
-
-#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
-	do { \
-		Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
-		Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
-			   (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
-		((toast_compress_header *) (ptr))->tcinfo = \
-			(len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method, cmp_ext) 				\
+	do { 																						\
+		Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); 									\
+		Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || 										\
+			   (cm_method) == TOAST_LZ4_COMPRESSION_ID); 										\
+		if ((cm_method) <= TOAST_LAST_COMPRESSION_ID_BEFORE_EXT) { 								\
+			((toast_compress_header *) (ptr))->tcinfo = 										\
+				(len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); 						\
+		} else { 																				\
+			/* For compression methods after lz4, use 11 in the top bits of tcinfo 				\
+				to indicate compression algorithm is stored in extended format. */ 				\
+			((toast_compress_header *) (ptr))->tcinfo = 										\
+				(len) | ((uint32) (VARATT_4BCE_MASK) << VARLENA_EXTSIZE_BITS);					\
+			Assert((cmp_ext) != NULL);															\
+			memcpy(VARATT_4BCE_HDR_PTR(ptr), (cmp_ext), 										\
+					sizeof(varatt_cmp_extended) + VARATT_4BCE_META_SIZE( cmp_ext->ext_hdr ));	\
+		} 																						\
 	} while (0)
 
-extern Datum toast_compress_datum(Datum value, char cmethod);
+extern Datum toast_compress_datum(Datum value, CompressionInfo cmp);
 extern Oid	toast_get_valid_index(Oid toastoid, LOCKMODE lock);
 
 extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);

src/include/varatt.h

@@ -328,7 +328,8 @@ typedef struct
 #define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
 	(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
 #define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
-	(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
+	( (VARATT_IS_4BCE(PTR)) ? (VARATT_4BCE_CMP_METHOD(VARATT_4BCE_HDR_PTR(PTR)->ext_hdr)) \
+	: (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS))
 
 /* Same for external Datums; but note argument is a struct varatt_external */
 #define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
@@ -340,8 +341,17 @@ typedef struct
 	do { \
 		Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
 			   (cm) == TOAST_LZ4_COMPRESSION_ID); \
-		((toast_pointer).va_extinfo = \
-			(len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
+		if ((cm) <= TOAST_LAST_COMPRESSION_ID_BEFORE_EXT) \
+		{ \
+			/* Store the actual method in va_extinfo */ \
+			((toast_pointer).va_extinfo = \
+				(len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
+		} \
+		else \
+		{ \
+			/* Store 11 in the top bits, meaning "extended" method. */ 				\
+			(toast_pointer).va_extinfo = (uint32)(len) | (VARATT_4BCE_MASK << VARLENA_EXTSIZE_BITS ); \
+		} \
 	} while (0)
 
 /*
@@ -355,4 +365,61 @@ typedef struct
 	(VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
 	 (toast_pointer).va_rawsize - VARHDRSZ)
 
+typedef struct varatt_cmp_extended
+{
+	uint32		ext_hdr;		/* [ size:24 | type:8 ] */
+	char		ext_data[FLEXIBLE_ARRAY_MEMBER];	/* algorithm-specific meta */
+} varatt_cmp_extended;
+
+/*--------------------------------------------------------------------*/
+/* 1) Detect the extended compression					              */
+/*    (top-2 mode bits of va_tcinfo are 0b11)                         */
+#define VARATT_4BCE_MASK   0x0003
+
+#define VARATT_IS_4BCE(ptr)                                            			\
+	((((varattrib_4b*)(ptr))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) 	\
+		== VARATT_4BCE_MASK)
+
+/*--------------------------------------------------------------------*/
+/* 2) Pointer to varatt_cmp_extended header (just after the 8-byte varlena head) */
+#define VARATT_4BCE_HDR_PTR(ptr) ((varatt_cmp_extended*)(((char*)(ptr)) + VARHDRSZ_COMPRESSED))
+#define VARATT_4BCE_GET_HDR(ptr) ((uint32)(VARATT_4BCE_HDR_PTR(ptr)->ext_hdr))
+
+/*--------------------------------------------------------------------*/
+/* 3) The 32-bit ext_hdr */
+/*    Layout:   [ meta size:24 bits | type:8 bits ] */
+#define VARATT_4BCE_TYPE_MASK  0x000000FF	/* low-order 8 bits  */
+#define VARATT_4BCE_SIZE_MASK  0xFFFFFF00	/* high-order 24 bits */
+
+#define VARATT_4BCE_SET_HDR(hdr, type, size24)                                  \
+	do {                                                                        \
+		Assert((uint32)(type)   <= VARATT_4BCE_TYPE_MASK);      /* 8 bits  */   \
+		Assert((uint32)(size24) <= (VARATT_4BCE_SIZE_MASK >> 8)); 				\
+		(hdr) = ( ((uint32)(type)) ) | ( ((uint32)(size24) << 8) );   			\
+	} while (0)
+
+#define VARATT_4BCE_CMP_METHOD(hdr)   ( (uint8)  ((hdr) & VARATT_4BCE_TYPE_MASK) )
+#define VARATT_4BCE_META_SIZE(hdr)   ( ((hdr) & VARATT_4BCE_SIZE_MASK) >> 8)
+
+/*--------------------------------------------------------------------*/
+/* 4) Derived helpers to jump inside the extension block */
+
+/* -> metadata begins immediately after the 4-byte ext header */
+#define VARATT_4BCE_META_PTR(ptr)    ( (void*) VARATT_4BCE_HDR_PTR(ptr)->ext_data )
+
+/* -> compressed bytes begins after metadata */
+#define VARATT_4BCE_DATA_PTR(ptr)                                      \
+	( (void*)( (char*)VARATT_4BCE_META_PTR(ptr)                        \
+			   + VARATT_4BCE_META_SIZE(VARATT_4BCE_HDR_PTR(ptr)->ext_hdr) ) )
+
+/* -> payload byte count */
+#define VARATT_4BCE_DATA_SIZE(ptr)                                    \
+	( VARSIZE_4B(ptr)                                                 \
+	  - VARHDRSZ_COMPRESSED                                           \
+	  - sizeof(varatt_cmp_extended)                                   \
+	  - VARATT_4BCE_META_SIZE(VARATT_4BCE_HDR_PTR(ptr)->ext_hdr) )
+
+/* Expects varatt_cmp_extended pointer */
+#define	VARATT_4BCE_HDRSZ(ptr) (VARHDRSZ_COMPRESSED + sizeof(varatt_cmp_extended) + VARATT_4BCE_META_SIZE((ptr)->ext_hdr))
+
 #endif

src/tools/pgindent/typedefs.list

@@ -482,6 +482,7 @@ CompositeIOData
 CompositeTypeStmt
 CompoundAffixFlag
 CompressFileHandle
+CompressionInfo
 CompressionLocation
 CompressorState
 ComputeXidHorizonsResult
@@ -4153,6 +4154,7 @@ uuid_t
 va_list
 vacuumingOptions
 validate_string_relopt
+varatt_cmp_extended
 varatt_expanded
 varattrib_1b
 varattrib_1b_e