Support varlena fields with single-byte headers and unaligned storage.

This commit breaks any code that assumes that the mere act of forming a tuple
(without writing it to disk) does not "toast" any fields.  While all available
regression tests pass, I'm not totally sure that we've fixed every nook and
cranny, especially in contrib.

Greg Stark with some help from Tom Lane

10 files changed, 336 insertions, 94 deletions

diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h
index 6c7c98b3f2..1fbb713b55 100644
--- a/src/include/access/heapam.h
+++ b/src/include/access/heapam.h
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.121 2007/03/29 00:15:39 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.122 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -200,7 +200,8 @@ extern Size heap_compute_data_size(TupleDesc tupleDesc,
 					   Datum *values, bool *isnull);
 extern void heap_fill_tuple(TupleDesc tupleDesc,
 				Datum *values, bool *isnull,
-				char *data, uint16 *infomask, bits8 *bit);
+				char *data, Size data_size,
+				uint16 *infomask, bits8 *bit);
 extern bool heap_attisnull(HeapTuple tup, int attnum);
 extern Datum nocachegetattr(HeapTuple tup, int attnum,
 			   TupleDesc att, bool *isnull);
diff --git a/src/include/access/htup.h b/src/include/access/htup.h
index c37501d910..ee816c568a 100644
--- a/src/include/access/htup.h
+++ b/src/include/access/htup.h
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.92 2007/02/27 23:48:09 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.93 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -160,9 +160,8 @@ typedef HeapTupleHeaderData *HeapTupleHeader;
 #define HEAP_HASNULL			0x0001	/* has null attribute(s) */
 #define HEAP_HASVARWIDTH		0x0002	/* has variable-width attribute(s) */
 #define HEAP_HASEXTERNAL		0x0004	/* has external stored attribute(s) */
-#define HEAP_HASCOMPRESSED		0x0008	/* has compressed stored attribute(s) */
-#define HEAP_HASEXTENDED		0x000C	/* the two above combined */
-#define HEAP_HASOID				0x0010	/* has an object-id field */
+#define HEAP_HASOID				0x0008	/* has an object-id field */
+/* bit 0x0010 is available */
 #define HEAP_COMBOCID			0x0020	/* t_cid is a combo cid */
 #define HEAP_XMAX_EXCL_LOCK		0x0040	/* xmax is exclusive locker */
 #define HEAP_XMAX_SHARED_LOCK	0x0080	/* xmax is shared locker */
@@ -341,7 +340,7 @@ do { \
  * MaxAttrSize is a somewhat arbitrary upper limit on the declared size of
  * data fields of char(n) and similar types.  It need not have anything
  * directly to do with the *actual* upper limit of varlena values, which
- * is currently 1Gb (see struct varattrib in postgres.h).  I've set it
+ * is currently 1Gb (see TOAST structures in postgres.h).  I've set it
  * at 10Mb which seems like a reasonable number --- tgl 8/6/00.
  */
 #define MaxAttrSize		(10 * 1024 * 1024)
@@ -485,12 +484,6 @@ typedef HeapTupleData *HeapTuple;
 #define HeapTupleHasExternal(tuple) \
 		(((tuple)->t_data->t_infomask & HEAP_HASEXTERNAL) != 0)
 
-#define HeapTupleHasCompressed(tuple) \
-		(((tuple)->t_data->t_infomask & HEAP_HASCOMPRESSED) != 0)
-
-#define HeapTupleHasExtended(tuple) \
-		(((tuple)->t_data->t_infomask & HEAP_HASEXTENDED) != 0)
-
 #define HeapTupleGetOid(tuple) \
 		HeapTupleHeaderGetOid((tuple)->t_data)
 
diff --git a/src/include/access/tupmacs.h b/src/include/access/tupmacs.h
index 80c2d436f0..f4fb8c7b33 100644
--- a/src/include/access/tupmacs.h
+++ b/src/include/access/tupmacs.h
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.32 2007/02/27 23:48:09 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.33 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -91,26 +91,83 @@
 #endif   /* SIZEOF_DATUM == 8 */
 
 /*
- * att_align aligns the given offset as needed for a datum of alignment
- * requirement attalign.  The cases are tested in what is hopefully something
- * like their frequency of occurrence.
+ * att_align_datum aligns the given offset as needed for a datum of alignment
+ * requirement attalign and typlen attlen.  attdatum is the Datum variable
+ * we intend to pack into a tuple (it's only accessed if we are dealing with
+ * a varlena type).  Note that this assumes the Datum will be stored as-is;
+ * callers that are intending to convert non-short varlena datums to short
+ * format have to account for that themselves.
  */
-#define att_align(cur_offset, attalign) \
+#define att_align_datum(cur_offset, attalign, attlen, attdatum) \
+( \
+	((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? (long) (cur_offset) : \
+	att_align_nominal(cur_offset, attalign) \
+)
+
+/*
+ * att_align_pointer performs the same calculation as att_align_datum,
+ * but is used when walking a tuple.  attptr is the current actual data
+ * pointer; when accessing a varlena field we have to "peek" to see if we
+ * are looking at a pad byte or the first byte of a 1-byte-header datum.
+ * (A zero byte must be either a pad byte, or the first byte of a correctly
+ * aligned 4-byte length word; in either case we can align safely.  A non-zero
+ * byte must be either a 1-byte length word, or the first byte of a correctly
+ * aligned 4-byte length word; in either case we need not align.)
+ *
+ * Note: some callers pass a "char *" pointer for cur_offset.  This is
+ * a bit of a hack but works OK on all known platforms.  It ought to be
+ * cleaned up someday, though.
+ */
+#define att_align_pointer(cur_offset, attalign, attlen, attptr) \
+( \
+	((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? (long) (cur_offset) : \
+	att_align_nominal(cur_offset, attalign) \
+)
+
+/*
+ * att_align_nominal aligns the given offset as needed for a datum of alignment
+ * requirement attalign, ignoring any consideration of packed varlena datums.
+ * There are three main use cases for using this macro directly:
+ * 	* we know that the att in question is not varlena (attlen != -1);
+ *	  in this case it is cheaper than the above macros and just as good.
+ *	* we need to estimate alignment padding cost abstractly, ie without
+ *	  reference to a real tuple.  We must assume the worst case that
+ *	  all varlenas are aligned.
+ *	* within arrays, we unconditionally align varlenas (XXX this should be
+ *	  revisited, probably).
+ *
+ * The attalign cases are tested in what is hopefully something like their
+ * frequency of occurrence.
+ */
+#define att_align_nominal(cur_offset, attalign) \
 ( \
 	((attalign) == 'i') ? INTALIGN(cur_offset) : \
-	 (((attalign) == 'c') ? ((long)(cur_offset)) : \
+	 (((attalign) == 'c') ? (long) (cur_offset) : \
 	  (((attalign) == 'd') ? DOUBLEALIGN(cur_offset) : \
-		( \
+	   ( \
 			AssertMacro((attalign) == 's'), \
 			SHORTALIGN(cur_offset) \
-		))) \
+	   ))) \
 )
 
 /*
- * att_addlength increments the given offset by the length of the attribute.
- * attval is only accessed if we are dealing with a variable-length attribute.
+ * att_addlength_datum increments the given offset by the space needed for
+ * the given Datum variable.  attdatum is only accessed if we are dealing
+ * with a variable-length attribute.
+ */
+#define att_addlength_datum(cur_offset, attlen, attdatum) \
+	att_addlength_pointer(cur_offset, attlen, DatumGetPointer(attdatum))
+
+/*
+ * att_addlength_pointer performs the same calculation as att_addlength_datum,
+ * but is used when walking a tuple --- attptr is the pointer to the field
+ * within the tuple.
+ *
+ * Note: some callers pass a "char *" pointer for cur_offset.  This is
+ * actually perfectly OK, but probably should be cleaned up along with
+ * the same practice for att_align_pointer.
  */
-#define att_addlength(cur_offset, attlen, attval) \
+#define att_addlength_pointer(cur_offset, attlen, attptr) \
 ( \
 	((attlen) > 0) ? \
 	( \
@@ -118,12 +175,12 @@
 	) \
 	: (((attlen) == -1) ? \
 	( \
-		(cur_offset) + VARSIZE(DatumGetPointer(attval)) \
+		(cur_offset) + VARSIZE_ANY(attptr) \
 	) \
 	: \
 	( \
 		AssertMacro((attlen) == -2), \
-		(cur_offset) + (strlen(DatumGetCString(attval)) + 1) \
+		(cur_offset) + (strlen((char *) (attptr)) + 1) \
 	)) \
 )
 
diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h
index 3ab2997946..ea870aff11 100644
--- a/src/include/access/tuptoaster.h
+++ b/src/include/access/tuptoaster.h
@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2000-2007, PostgreSQL Global Development Group
  *
- * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.34 2007/04/03 04:14:26 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.35 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -111,7 +111,7 @@ extern void toast_delete(Relation rel, HeapTuple oldtup);
  *		in compressed format.
  * ----------
  */
-extern varattrib *heap_tuple_fetch_attr(varattrib *attr);
+extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr);
 
 /* ----------
  * heap_tuple_untoast_attr() -
@@ -120,7 +120,7 @@ extern varattrib *heap_tuple_fetch_attr(varattrib *attr);
  *		it as needed.
  * ----------
  */
-extern varattrib *heap_tuple_untoast_attr(varattrib *attr);
+extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr);
 
 /* ----------
  * heap_tuple_untoast_attr_slice() -
@@ -129,7 +129,7 @@ extern varattrib *heap_tuple_untoast_attr(varattrib *attr);
  *		(Handles all cases for attribute storage)
  * ----------
  */
-extern varattrib *heap_tuple_untoast_attr_slice(varattrib *attr,
+extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr,
 							  int32 sliceoffset,
 							  int32 slicelength);
 
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index 9525a198fe..68486709cc 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -37,7 +37,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.399 2007/04/02 03:49:40 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.400 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -53,6 +53,6 @@
  */
 
 /*							yyyymmddN */
-#define CATALOG_VERSION_NO	200704012
+#define CATALOG_VERSION_NO	200704051
 
 #endif
diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h
index 6971319600..519be08afc 100644
--- a/src/include/catalog/pg_type.h
+++ b/src/include/catalog/pg_type.h
@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/pg_type.h,v 1.181 2007/04/02 03:49:41 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/pg_type.h,v 1.182 2007/04/06 04:21:43 tgl Exp $
  *
  * NOTES
  *	  the genbki.sh script reads this file and generates .bki
@@ -130,8 +130,10 @@ CATALOG(pg_type,1247) BKI_BOOTSTRAP
 	 * 'i' = INT alignment (4 bytes on most machines).
 	 * 'd' = DOUBLE alignment (8 bytes on many machines, but by no means all).
 	 *
-	 * See include/utils/memutils.h for the macros that compute these
-	 * alignment requirements.
+	 * See include/access/tupmacs.h for the macros that compute these
+	 * alignment requirements.  Note also that we allow the nominal alignment
+	 * to be violated when storing "packed" varlenas; the TOAST mechanism
+	 * takes care of hiding that from most code.
 	 *
 	 * NOTE: for types used in system tables, it is critical that the
 	 * size and alignment defined in pg_type agree with the way that the
@@ -398,10 +400,10 @@ DATA(insert OID = 791 (  _money    PGNSP PGUID	-1 f b t \054 0  790 array_in arr
 DATA(insert OID = 829 ( macaddr    PGNSP PGUID	6 f b t \054 0 0 macaddr_in macaddr_out macaddr_recv macaddr_send - - - i p f 0 -1 0 _null_ _null_ ));
 DESCR("XX:XX:XX:XX:XX:XX, MAC address");
 #define MACADDROID 829
-DATA(insert OID = 869 ( inet	   PGNSP PGUID	-1 f b t \054 0 0 inet_in inet_out inet_recv inet_send - - - i p f 0 -1 0 _null_ _null_ ));
+DATA(insert OID = 869 ( inet	   PGNSP PGUID	-1 f b t \054 0 0 inet_in inet_out inet_recv inet_send - - - i m f 0 -1 0 _null_ _null_ ));
 DESCR("IP address/netmask, host address, netmask optional");
 #define INETOID 869
-DATA(insert OID = 650 ( cidr	   PGNSP PGUID	-1 f b t \054 0 0 cidr_in cidr_out cidr_recv cidr_send - - - i p f 0 -1 0 _null_ _null_ ));
+DATA(insert OID = 650 ( cidr	   PGNSP PGUID	-1 f b t \054 0 0 cidr_in cidr_out cidr_recv cidr_send - - - i m f 0 -1 0 _null_ _null_ ));
 DESCR("network IP address/netmask, network address");
 #define CIDROID 650
 
diff --git a/src/include/fmgr.h b/src/include/fmgr.h
index f6462d43a8..298cd12c77 100644
--- a/src/include/fmgr.h
+++ b/src/include/fmgr.h
@@ -11,7 +11,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/fmgr.h,v 1.49 2007/01/05 22:19:50 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/fmgr.h,v 1.50 2007/04/06 04:21:44 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -153,6 +153,11 @@ extern void fmgr_info_copy(FmgrInfo *dstinfo, FmgrInfo *srcinfo,
  * if you need a modifiable copy of the input.	Caller is expected to have
  * checked for null inputs first, if necessary.
  *
+ * pg_detoast_datum_packed() will return packed (1-byte header) datums
+ * unmodified.  It will still expand an externally toasted or compressed datum.
+ * The resulting datum can be accessed using VARSIZE_ANY() and VARDATA_ANY()
+ * (beware of multiple evaluations in those macros!)
+ *
  * Note: it'd be nice if these could be macros, but I see no way to do that
  * without evaluating the arguments multiple times, which is NOT acceptable.
  */
@@ -160,6 +165,7 @@ extern struct varlena *pg_detoast_datum(struct varlena * datum);
 extern struct varlena *pg_detoast_datum_copy(struct varlena * datum);
 extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 					   int32 first, int32 count);
+extern struct varlena *pg_detoast_datum_packed(struct varlena * datum);
 
 #define PG_DETOAST_DATUM(datum) \
 	pg_detoast_datum((struct varlena *) DatumGetPointer(datum))
@@ -168,6 +174,8 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 #define PG_DETOAST_DATUM_SLICE(datum,f,c) \
 		pg_detoast_datum_slice((struct varlena *) DatumGetPointer(datum), \
 		(int32) f, (int32) c)
+#define PG_DETOAST_DATUM_PACKED(datum) \
+	pg_detoast_datum_packed((struct varlena *) DatumGetPointer(datum))
 
 /*
  * Support for cleaning up detoasted copies of inputs.	This must only
@@ -207,9 +215,13 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 #define PG_GETARG_VARLENA_P(n) PG_DETOAST_DATUM(PG_GETARG_DATUM(n))
 /* DatumGetFoo macros for varlena types will typically look like this: */
 #define DatumGetByteaP(X)			((bytea *) PG_DETOAST_DATUM(X))
+#define DatumGetByteaPP(X)			((bytea *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetTextP(X)			((text *) PG_DETOAST_DATUM(X))
+#define DatumGetTextPP(X)			((text *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetBpCharP(X)			((BpChar *) PG_DETOAST_DATUM(X))
+#define DatumGetBpCharPP(X)			((BpChar *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetVarCharP(X)			((VarChar *) PG_DETOAST_DATUM(X))
+#define DatumGetVarCharPP(X)		((VarChar *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetHeapTupleHeader(X)	((HeapTupleHeader) PG_DETOAST_DATUM(X))
 /* And we also offer variants that return an OK-to-write copy */
 #define DatumGetByteaPCopy(X)		((bytea *) PG_DETOAST_DATUM_COPY(X))
@@ -224,9 +236,13 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 #define DatumGetVarCharPSlice(X,m,n) ((VarChar *) PG_DETOAST_DATUM_SLICE(X,m,n))
 /* GETARG macros for varlena types will typically look like this: */
 #define PG_GETARG_BYTEA_P(n)		DatumGetByteaP(PG_GETARG_DATUM(n))
+#define PG_GETARG_BYTEA_PP(n)		DatumGetByteaPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_TEXT_P(n)			DatumGetTextP(PG_GETARG_DATUM(n))
+#define PG_GETARG_TEXT_PP(n)		DatumGetTextPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_BPCHAR_P(n)		DatumGetBpCharP(PG_GETARG_DATUM(n))
+#define PG_GETARG_BPCHAR_PP(n)		DatumGetBpCharPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_VARCHAR_P(n)		DatumGetVarCharP(PG_GETARG_DATUM(n))
+#define PG_GETARG_VARCHAR_PP(n)		DatumGetVarCharPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_HEAPTUPLEHEADER(n)	DatumGetHeapTupleHeader(PG_GETARG_DATUM(n))
 /* And we also offer variants that return an OK-to-write copy */
 #define PG_GETARG_BYTEA_P_COPY(n)	DatumGetByteaPCopy(PG_GETARG_DATUM(n))
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 9b107e27f7..35c1be6f9c 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -671,6 +671,10 @@
 /* Define to select Win32-style shared memory. */
 #undef USE_WIN32_SHARED_MEMORY
 
+/* Define to 1 if your processor stores words with the most significant byte
+   first (like Motorola and SPARC, unlike Intel and VAX). */
+#undef WORDS_BIGENDIAN
+
 /* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS
 
diff --git a/src/include/postgres.h b/src/include/postgres.h
index 5234c34e29..1ff0e97e9b 100644
--- a/src/include/postgres.h
+++ b/src/include/postgres.h
@@ -10,7 +10,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1995, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/postgres.h,v 1.78 2007/03/23 20:24:41 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/postgres.h,v 1.79 2007/04/06 04:21:44 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -54,56 +54,219 @@
  * ----------------------------------------------------------------
  */
 
-/* ----------------
- * struct varattrib is the header of a varlena object that may have been
- * TOASTed.  Generally, only the code closely associated with TOAST logic
- * should mess directly with struct varattrib or use the VARATT_FOO macros.
- * ----------------
+/*
+ * struct varatt_external is a "TOAST pointer", that is, the information
+ * needed to fetch a stored-out-of-line Datum.  The data is compressed
+ * if and only if va_extsize < va_rawsize - VARHDRSZ.  This struct must not
+ * contain any padding, because we sometimes compare pointers using memcmp.
+ *
+ * Note that this information is stored unaligned within actual tuples, so
+ * you need to memcpy from the tuple into a local struct variable before
+ * you can look at these fields!  (The reason we use memcmp is to avoid
+ * having to do that just to detect equality of two TOAST pointers...)
+ */
+struct varatt_external
+{
+	int32		va_rawsize;			/* Original data size (includes header) */
+	int32		va_extsize;			/* External saved size (doesn't) */
+	Oid			va_valueid;			/* Unique ID of value within TOAST table */
+	Oid			va_toastrelid;		/* RelID of TOAST table containing it */
+};
+
+/*
+ * These structs describe the header of a varlena object that may have been
+ * TOASTed.  Generally, don't reference these structs directly, but use the
+ * macros below.
+ *
+ * We use separate structs for the aligned and unaligned cases because the
+ * compiler might otherwise think it could generate code that assumes
+ * alignment while touching fields of a 1-byte-header varlena.
  */
-typedef struct varattrib
+typedef union
 {
-	int32		va_header_;		/* External/compressed storage */
-	/* flags and item size */
-	union
+	struct							/* Normal varlena (4-byte length) */
+	{
+		uint32	va_header;
+		char	va_data[1];
+	} va_4byte;
+	struct							/* Compressed-in-line format */
 	{
-		struct
-		{
-			int32		va_rawsize;		/* Plain data size */
-			char		va_data[1];		/* Compressed data */
-		}			va_compressed;		/* Compressed stored attribute */
-
-		struct
-		{
-			int32		va_rawsize;		/* Plain data size */
-			int32		va_extsize;		/* External saved size */
-			Oid			va_valueid;		/* Unique identifier of value */
-			Oid			va_toastrelid;	/* RelID where to find chunks */
-		}			va_external;	/* External stored attribute */
-
-		char		va_data[1]; /* Plain stored attribute */
-	}			va_content;
-} varattrib;
-
-#define VARATT_FLAG_EXTERNAL	0x80000000
-#define VARATT_FLAG_COMPRESSED	0x40000000
-#define VARATT_MASK_FLAGS		0xc0000000
-#define VARATT_MASK_SIZE		0x3fffffff
-
-#define VARATT_SIZEP_DEPRECATED(PTR)	(((varattrib *) (PTR))->va_header_)
-
-#define VARATT_IS_EXTENDED(PTR)		\
-				((VARATT_SIZEP_DEPRECATED(PTR) & VARATT_MASK_FLAGS) != 0)
-#define VARATT_IS_EXTERNAL(PTR)		\
-				((VARATT_SIZEP_DEPRECATED(PTR) & VARATT_FLAG_EXTERNAL) != 0)
-#define VARATT_IS_COMPRESSED(PTR)	\
-				((VARATT_SIZEP_DEPRECATED(PTR) & VARATT_FLAG_COMPRESSED) != 0)
-
-/* These macros are the ones for non-TOAST code to use */
-
-#define VARSIZE(PTR)	(VARATT_SIZEP_DEPRECATED(PTR) & VARATT_MASK_SIZE)
-#define VARDATA(PTR)	(((varattrib *) (PTR))->va_content.va_data)
-
-#define SET_VARSIZE(PTR,SIZE)	(VARATT_SIZEP_DEPRECATED(PTR) = (SIZE))
+		uint32	va_header;
+		uint32	va_rawsize;			/* Original data size (excludes header) */
+		char	va_data[1];			/* Compressed data */
+	} va_compressed;
+} varattrib_4b;
+
+typedef struct
+{
+	uint8		va_header;
+	char		va_data[1];			/* Data or TOAST pointer */
+} varattrib_1b;
+
+typedef struct
+{
+	uint8		va_header;
+	char		va_data[sizeof(struct varatt_external)];
+} varattrib_pointer;
+
+/*
+ * Bit layouts for varlena headers on big-endian machines:
+ *
+ * 00xxxxxx	4-byte length word, aligned, uncompressed data (up to 1G)
+ * 01xxxxxx	4-byte length word, aligned, *compressed* data (up to 1G)
+ * 10000000	1-byte length word, unaligned, TOAST pointer
+ * 1xxxxxxx	1-byte length word, unaligned, uncompressed data (up to 126b)
+ *
+ * Bit layouts for varlena headers on little-endian machines:
+ *
+ * xxxxxx00	4-byte length word, aligned, uncompressed data (up to 1G)
+ * xxxxxx10	4-byte length word, aligned, *compressed* data (up to 1G)
+ * 00000001	1-byte length word, unaligned, TOAST pointer
+ * xxxxxxx1	1-byte length word, unaligned, uncompressed data (up to 126b)
+ *
+ * The "xxx" bits are the length field (which includes itself in all cases).
+ * In the big-endian case we mask to extract the length, in the little-endian
+ * case we shift.  Note that in both cases the flag bits are in the physically
+ * first byte.  Also, it is not possible for a 1-byte length word to be zero;
+ * this lets us disambiguate alignment padding bytes from the start of an
+ * unaligned datum.  (We now *require* pad bytes to be filled with zero!)
+ */
+
+/*
+ * Endian-dependent macros.  These are considered internal --- use the
+ * external macros below instead of using these directly.
+ *
+ * Note: IS_1B is true for external toast records but VARSIZE_1B will return 0
+ * for such records. Hence you should usually check for IS_EXTERNAL before
+ * checking for IS_1B.
+ */
+
+#ifdef WORDS_BIGENDIAN
+
+#define VARATT_IS_4B(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00)
+#define VARATT_IS_4B_U(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00)
+#define VARATT_IS_4B_C(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40)
+#define VARATT_IS_1B(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80)
+#define VARATT_IS_1B_E(PTR) \
+	((((varattrib_1b *) (PTR))->va_header) == 0x80)
+#define VARATT_NOT_PAD_BYTE(PTR) \
+	(*((uint8 *) (PTR)) != 0)
+
+/* VARSIZE_4B() should only be used on known-aligned data */
+#define VARSIZE_4B(PTR) \
+	(((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
+#define VARSIZE_1B(PTR) \
+	(((varattrib_1b *) (PTR))->va_header & 0x7F)
+/* Currently there is only one size of toast pointer, but someday maybe not */
+#define VARSIZE_1B_E(PTR) \
+	(sizeof(varattrib_pointer))
+
+#define SET_VARSIZE_4B(PTR,len) \
+	(((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
+#define SET_VARSIZE_4B_C(PTR,len) \
+	(((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
+#define SET_VARSIZE_1B(PTR,len) \
+	(((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
+#define SET_VARSIZE_1B_E(PTR) \
+	(((varattrib_1b *) (PTR))->va_header = 0x80)
+
+#else  /* !WORDS_BIGENDIAN */
+
+#define VARATT_IS_4B(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00)
+#define VARATT_IS_4B_U(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00)
+#define VARATT_IS_4B_C(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02)
+#define VARATT_IS_1B(PTR) \
+	((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01)
+#define VARATT_IS_1B_E(PTR) \
+	((((varattrib_1b *) (PTR))->va_header) == 0x01)
+#define VARATT_NOT_PAD_BYTE(PTR) \
+	(*((uint8 *) (PTR)) != 0)
+
+/* VARSIZE_4B() should only be used on known-aligned data */
+#define VARSIZE_4B(PTR) \
+	((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
+#define VARSIZE_1B(PTR) \
+	((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
+/* Currently there is only one size of toast pointer, but someday maybe not */
+#define VARSIZE_1B_E(PTR) \
+	(sizeof(varattrib_pointer))
+
+#define SET_VARSIZE_4B(PTR,len) \
+	(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
+#define SET_VARSIZE_4B_C(PTR,len) \
+	(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
+#define SET_VARSIZE_1B(PTR,len) \
+	(((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
+#define SET_VARSIZE_1B_E(PTR) \
+	(((varattrib_1b *) (PTR))->va_header = 0x01)
+
+#endif /* WORDS_BIGENDIAN */
+
+#define VARHDRSZ_SHORT			1
+#define VARATT_SHORT_MAX		0x7F
+#define VARATT_CAN_MAKE_SHORT(PTR) \
+	(VARATT_IS_4B_U(PTR) && \
+	 (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX)
+#define VARATT_CONVERTED_SHORT_SIZE(PTR) \
+	(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
+
+#define VARDATA_4B(PTR)		(((varattrib_4b *) (PTR))->va_4byte.va_data)
+#define VARDATA_4B_C(PTR)	(((varattrib_4b *) (PTR))->va_compressed.va_data)
+#define VARDATA_1B(PTR)		(((varattrib_1b *) (PTR))->va_data)
+
+#define VARRAWSIZE_4B_C(PTR) \
+	(((varattrib_4b *) (PTR))->va_compressed.va_rawsize)
+
+/* Externally visible macros */
+
+/*
+ * VARDATA, VARSIZE, and SET_VARSIZE are the recommended API for most code
+ * for varlena datatypes.  Note that they only work on untoasted,
+ * 4-byte-header Datums!
+ *
+ * Code that wants to use 1-byte-header values without detoasting should
+ * use VARSIZE_ANY/VARSIZE_ANY_EXHDR/VARDATA_ANY.  The other macros here
+ * should usually be used only by tuple assembly/disassembly code and
+ * code that specifically wants to work with still-toasted Datums.
+ */
+#define VARDATA(PTR)						VARDATA_4B(PTR)
+#define VARSIZE(PTR)						VARSIZE_4B(PTR)
+
+#define VARSIZE_SHORT(PTR)					VARSIZE_1B(PTR)
+#define VARDATA_SHORT(PTR)					VARDATA_1B(PTR)
+
+#define VARSIZE_EXTERNAL(PTR)				VARSIZE_1B_E(PTR)
+
+#define VARATT_IS_COMPRESSED(PTR)			VARATT_IS_4B_C(PTR)
+#define VARATT_IS_EXTERNAL(PTR)				VARATT_IS_1B_E(PTR)
+#define VARATT_IS_SHORT(PTR)				VARATT_IS_1B(PTR)
+#define VARATT_IS_EXTENDED(PTR)				(!VARATT_IS_4B_U(PTR))
+
+#define SET_VARSIZE(PTR, len)				SET_VARSIZE_4B(PTR, len)
+#define SET_VARSIZE_SHORT(PTR, len)			SET_VARSIZE_1B(PTR, len)
+#define SET_VARSIZE_COMPRESSED(PTR, len)	SET_VARSIZE_4B_C(PTR, len)
+#define SET_VARSIZE_EXTERNAL(PTR)			SET_VARSIZE_1B_E(PTR)
+
+#define VARSIZE_ANY(PTR) \
+	(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \
+	 (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \
+	  VARSIZE_4B(PTR)))
+
+#define VARSIZE_ANY_EXHDR(PTR) \
+	(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-1 : \
+	 (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-1 : \
+	  VARSIZE_4B(PTR)-4))
+
+/* caution: this will not work on an external or compressed-in-line Datum */
+#define VARDATA_ANY(PTR) \
+	 (VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR))
 
 
 /* ----------------------------------------------------------------
diff --git a/src/include/utils/inet.h b/src/include/utils/inet.h
index ca92f4ca9e..d94855bb22 100644
--- a/src/include/utils/inet.h
+++ b/src/include/utils/inet.h
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/utils/inet.h,v 1.25 2007/01/05 22:19:59 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/utils/inet.h,v 1.26 2007/04/06 04:21:44 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -39,13 +39,19 @@ typedef struct
 
 /*
  * Both INET and CIDR addresses are represented within Postgres as varlena
- * objects, ie, there is a varlena header (basically a length word) in front
- * of the struct type depicted above.
- *
- * Although these types are variable-length, the maximum length
- * is pretty short, so we make no provision for TOASTing them.
+ * objects, ie, there is a varlena header in front of the struct type
+ * depicted above.  This struct depicts what we actually have in memory
+ * in "uncompressed" cases.  Note that since the maximum data size is only
+ * 18 bytes, INET/CIDR will invariably be stored into tuples using the
+ * 1-byte-header varlena format.  However, we have to be prepared to cope
+ * with the 4-byte-header format too, because various code may helpfully
+ * try to "decompress" 1-byte-header datums.
  */
-typedef struct varlena inet;
+typedef struct
+{
+	int32		vl_len_;		/* Do not touch this field directly! */
+	inet_struct	inet_data;
+} inet;
 
 
 /*
@@ -64,7 +70,7 @@ typedef struct macaddr
 /*
  * fmgr interface macros
  */
-#define DatumGetInetP(X)	((inet *) DatumGetPointer(X))
+#define DatumGetInetP(X)	((inet *) PG_DETOAST_DATUM_PACKED(X))
 #define InetPGetDatum(X)	PointerGetDatum(X)
 #define PG_GETARG_INET_P(n) DatumGetInetP(PG_GETARG_DATUM(n))
 #define PG_RETURN_INET_P(x) return InetPGetDatum(x)