summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/backend/access/heap/tuptoaster.c110
-rw-r--r--src/include/access/tuptoaster.h5
-rw-r--r--src/include/postgres.h84
-rw-r--r--src/test/regress/expected/indirect_toast.out151
-rw-r--r--src/test/regress/input/create_function_1.source5
-rw-r--r--src/test/regress/output/create_function_1.source4
-rw-r--r--src/test/regress/parallel_schedule2
-rw-r--r--src/test/regress/regress.c92
-rw-r--r--src/test/regress/serial_schedule1
-rw-r--r--src/test/regress/sql/indirect_toast.sql61
10 files changed, 472 insertions, 43 deletions
diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c
index fc37ceb4a3e..445a7ed9fbc 100644
--- a/src/backend/access/heap/tuptoaster.c
+++ b/src/backend/access/heap/tuptoaster.c
@@ -44,9 +44,6 @@
#undef TOAST_DEBUG
-/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
-#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_external))
-
/*
* Testing whether an externally-stored value is compressed now requires
* comparing extsize (the actual length of the external data) to rawsize
@@ -87,11 +84,11 @@ static struct varlena *toast_fetch_datum_slice(struct varlena * attr,
* heap_tuple_fetch_attr -
*
* Public entry point to get back a toasted value from
- * external storage (possibly still in compressed format).
+ * external source (possibly still in compressed format).
*
* This will return a datum that contains all the data internally, ie, not
- * relying on external storage, but it can still be compressed or have a short
- * header.
+ * relying on external storage or memory, but it can still be compressed or
+ * have a short header.
----------
*/
struct varlena *
@@ -99,13 +96,35 @@ heap_tuple_fetch_attr(struct varlena * attr)
{
struct varlena *result;
- if (VARATT_IS_EXTERNAL(attr))
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
/*
* This is an external stored plain value
*/
result = toast_fetch_datum(attr);
}
+ else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ {
+ /*
+ * copy into the caller's memory context. That's not required in all
+ * cases but sufficient for now since this is mainly used when we need
+ * to persist a Datum for unusually long time, like in a HOLD cursor.
+ */
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, attr);
+ attr = (struct varlena *)redirect.pointer;
+
+ /* nested indirect Datums aren't allowed */
+ Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
+
+ /* doesn't make much sense, but better handle it */
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
+ return heap_tuple_fetch_attr(attr);
+
+ /* copy datum verbatim */
+ result = (struct varlena *) palloc(VARSIZE_ANY(attr));
+ memcpy(result, attr, VARSIZE_ANY(attr));
+ }
else
{
/*
@@ -128,7 +147,7 @@ heap_tuple_fetch_attr(struct varlena * attr)
struct varlena *
heap_tuple_untoast_attr(struct varlena * attr)
{
- if (VARATT_IS_EXTERNAL(attr))
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
/*
* This is an externally stored datum --- fetch it back from there
@@ -145,6 +164,17 @@ heap_tuple_untoast_attr(struct varlena * attr)
pfree(tmp);
}
}
+ else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ {
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, attr);
+ attr = (struct varlena *)redirect.pointer;
+
+ /* nested indirect Datums aren't allowed */
+ Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
+
+ attr = heap_tuple_untoast_attr(attr);
+ }
else if (VARATT_IS_COMPRESSED(attr))
{
/*
@@ -191,7 +221,7 @@ heap_tuple_untoast_attr_slice(struct varlena * attr,
char *attrdata;
int32 attrsize;
- if (VARATT_IS_EXTERNAL(attr))
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
struct varatt_external toast_pointer;
@@ -204,6 +234,17 @@ heap_tuple_untoast_attr_slice(struct varlena * attr,
/* fetch it back (compressed marker will get set automatically) */
preslice = toast_fetch_datum(attr);
}
+ else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ {
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, attr);
+
+ /* nested indirect Datums aren't allowed */
+ Assert(!VARATT_IS_EXTERNAL_INDIRECT(redirect.pointer));
+
+ return heap_tuple_untoast_attr_slice(redirect.pointer,
+ sliceoffset, slicelength);
+ }
else
preslice = attr;
@@ -267,7 +308,7 @@ toast_raw_datum_size(Datum value)
struct varlena *attr = (struct varlena *) DatumGetPointer(value);
Size result;
- if (VARATT_IS_EXTERNAL(attr))
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
/* va_rawsize is the size of the original datum -- including header */
struct varatt_external toast_pointer;
@@ -275,6 +316,16 @@ toast_raw_datum_size(Datum value)
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
result = toast_pointer.va_rawsize;
}
+ else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ {
+ struct varatt_indirect toast_pointer;
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+
+ /* nested indirect Datums aren't allowed */
+ Assert(!VARATT_IS_EXTERNAL_INDIRECT(toast_pointer.pointer));
+
+ return toast_raw_datum_size(PointerGetDatum(toast_pointer.pointer));
+ }
else if (VARATT_IS_COMPRESSED(attr))
{
/* here, va_rawsize is just the payload size */
@@ -308,7 +359,7 @@ toast_datum_size(Datum value)
struct varlena *attr = (struct varlena *) DatumGetPointer(value);
Size result;
- if (VARATT_IS_EXTERNAL(attr))
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
/*
* Attribute is stored externally - return the extsize whether
@@ -320,6 +371,16 @@ toast_datum_size(Datum value)
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
result = toast_pointer.va_extsize;
}
+ else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ {
+ struct varatt_indirect toast_pointer;
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+
+ /* nested indirect Datums aren't allowed */
+ Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
+
+ return toast_datum_size(PointerGetDatum(toast_pointer.pointer));
+ }
else if (VARATT_IS_SHORT(attr))
{
result = VARSIZE_SHORT(attr);
@@ -387,8 +448,12 @@ toast_delete(Relation rel, HeapTuple oldtup)
{
Datum value = toast_values[i];
- if (!toast_isnull[i] && VARATT_IS_EXTERNAL(PointerGetDatum(value)))
+ if (toast_isnull[i])
+ continue;
+ else if (VARATT_IS_EXTERNAL_ONDISK(PointerGetDatum(value)))
toast_delete_datum(rel, value);
+ else if (VARATT_IS_EXTERNAL_INDIRECT(PointerGetDatum(value)))
+ elog(ERROR, "attempt to delete tuple containing indirect datums");
}
}
}
@@ -490,13 +555,13 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
new_value = (struct varlena *) DatumGetPointer(toast_values[i]);
/*
- * If the old value is an external stored one, check if it has
- * changed so we have to delete it later.
+ * If the old value is stored on disk, check if it has changed so
+ * we have to delete it later.
*/
if (att[i]->attlen == -1 && !toast_oldisnull[i] &&
- VARATT_IS_EXTERNAL(old_value))
+ VARATT_IS_EXTERNAL_ONDISK(old_value))
{
- if (toast_isnull[i] || !VARATT_IS_EXTERNAL(new_value) ||
+ if (toast_isnull[i] || !VARATT_IS_EXTERNAL_ONDISK(new_value) ||
memcmp((char *) old_value, (char *) new_value,
VARSIZE_EXTERNAL(old_value)) != 0)
{
@@ -1258,6 +1323,8 @@ toast_save_datum(Relation rel, Datum value,
int32 data_todo;
Pointer dval = DatumGetPointer(value);
+ Assert(!VARATT_IS_EXTERNAL(value));
+
/*
* Open the toast relation and its index. We can use the index to check
* uniqueness of the OID we assign to the toasted item, even though it has
@@ -1341,7 +1408,7 @@ toast_save_datum(Relation rel, Datum value,
{
struct varatt_external old_toast_pointer;
- Assert(VARATT_IS_EXTERNAL(oldexternal));
+ Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
/* Must copy to access aligned fields */
VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
@@ -1456,7 +1523,7 @@ toast_save_datum(Relation rel, Datum value,
* Create the TOAST pointer value that we'll return
*/
result = (struct varlena *) palloc(TOAST_POINTER_SIZE);
- SET_VARSIZE_EXTERNAL(result, TOAST_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK);
memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
return PointerGetDatum(result);
@@ -1480,7 +1547,7 @@ toast_delete_datum(Relation rel, Datum value)
SysScanDesc toastscan;
HeapTuple toasttup;
- if (!VARATT_IS_EXTERNAL(attr))
+ if (!VARATT_IS_EXTERNAL_ONDISK(attr))
return;
/* Must copy to access aligned fields */
@@ -1608,6 +1675,9 @@ toast_fetch_datum(struct varlena * attr)
char *chunkdata;
int32 chunksize;
+ if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ elog(ERROR, "shouldn't be called for indirect tuples");
+
/* Must copy to access aligned fields */
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
@@ -1775,7 +1845,7 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length)
int32 chcpystrt;
int32 chcpyend;
- Assert(VARATT_IS_EXTERNAL(attr));
+ Assert(VARATT_IS_EXTERNAL_ONDISK(attr));
/* Must copy to access aligned fields */
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h
index 6f4fc4545da..d0c17fde036 100644
--- a/src/include/access/tuptoaster.h
+++ b/src/include/access/tuptoaster.h
@@ -94,6 +94,11 @@
sizeof(int32) - \
VARHDRSZ)
+/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
+#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_external))
+
+/* Size of an indirect datum that contains an indirect TOAST pointer */
+#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_indirect))
/* ----------
* toast_insert_or_update -
diff --git a/src/include/postgres.h b/src/include/postgres.h
index 30e1dee1870..cb9d196c519 100644
--- a/src/include/postgres.h
+++ b/src/include/postgres.h
@@ -54,23 +54,53 @@
*/
/*
- * struct varatt_external is a "TOAST pointer", that is, the information
- * needed to fetch a stored-out-of-line Datum. The data is compressed
- * if and only if va_extsize < va_rawsize - VARHDRSZ. This struct must not
- * contain any padding, because we sometimes compare pointers using memcmp.
+ * struct varatt_external is a "TOAST pointer", that is, the information needed
+ * to fetch a Datum stored in an out-of-line on-disk Datum. The data is
+ * compressed if and only if va_extsize < va_rawsize - VARHDRSZ. This struct
+ * must not contain any padding, because we sometimes compare pointers using
+ * memcmp.
*
* Note that this information is stored unaligned within actual tuples, so
* you need to memcpy from the tuple into a local struct variable before
* you can look at these fields! (The reason we use memcmp is to avoid
* having to do that just to detect equality of two TOAST pointers...)
*/
-struct varatt_external
+typedef struct varatt_external
{
int32 va_rawsize; /* Original data size (includes header) */
int32 va_extsize; /* External saved size (doesn't) */
Oid va_valueid; /* Unique ID of value within TOAST table */
Oid va_toastrelid; /* RelID of TOAST table containing it */
-};
+} varatt_external;
+
+/*
+ * Out-of-line Datum thats stored in memory in contrast to varatt_external
+ * pointers which points to data in an external toast relation.
+ *
+ * Note that just as varatt_external's this is stored unaligned within the
+ * tuple.
+ */
+typedef struct varatt_indirect
+{
+ struct varlena *pointer; /* Pointer to in-memory varlena */
+} varatt_indirect;
+
+
+/*
+ * Type of external toast datum stored. The peculiar value for VARTAG_ONDISK
+ * comes from the requirement for on-disk compatibility with the older
+ * definitions of varattrib_1b_e where v_tag was named va_len_1be...
+ */
+typedef enum vartag_external
+{
+ VARTAG_INDIRECT = 1,
+ VARTAG_ONDISK = 18
+} vartag_external;
+
+#define VARTAG_SIZE(tag) \
+ ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
+ (tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \
+ TrapMacro(true, "unknown vartag"))
/*
* These structs describe the header of a varlena object that may have been
@@ -102,11 +132,12 @@ typedef struct
char va_data[1]; /* Data begins here */
} varattrib_1b;
+/* inline portion of a short varlena pointing to an external resource */
typedef struct
{
uint8 va_header; /* Always 0x80 or 0x01 */
- uint8 va_len_1be; /* Physical length of datum */
- char va_data[1]; /* Data (for now always a TOAST pointer) */
+ uint8 va_tag; /* Type of datum */
+ char va_data[1]; /* Data (of the type indicated by va_tag) */
} varattrib_1b_e;
/*
@@ -130,6 +161,9 @@ typedef struct
* first byte. Also, it is not possible for a 1-byte length word to be zero;
* this lets us disambiguate alignment padding bytes from the start of an
* unaligned datum. (We now *require* pad bytes to be filled with zero!)
+ *
+ * In TOAST datums the tag field in varattrib_1b_e is used to discern whether
+ * its an indirection pointer or more commonly an on-disk tuple.
*/
/*
@@ -161,8 +195,8 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
(((varattrib_1b *) (PTR))->va_header & 0x7F)
-#define VARSIZE_1B_E(PTR) \
- (((varattrib_1b_e *) (PTR))->va_len_1be)
+#define VARTAG_1B_E(PTR) \
+ (((varattrib_1b_e *) (PTR))->va_tag)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
@@ -170,9 +204,9 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
-#define SET_VARSIZE_1B_E(PTR,len) \
+#define SET_VARTAG_1B_E(PTR,tag) \
(((varattrib_1b_e *) (PTR))->va_header = 0x80, \
- ((varattrib_1b_e *) (PTR))->va_len_1be = (len))
+ ((varattrib_1b_e *) (PTR))->va_tag = (tag))
#else /* !WORDS_BIGENDIAN */
#define VARATT_IS_4B(PTR) \
@@ -193,8 +227,8 @@ typedef struct
((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
-#define VARSIZE_1B_E(PTR) \
- (((varattrib_1b_e *) (PTR))->va_len_1be)
+#define VARTAG_1B_E(PTR) \
+ (((varattrib_1b_e *) (PTR))->va_tag)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
@@ -202,12 +236,12 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
-#define SET_VARSIZE_1B_E(PTR,len) \
+#define SET_VARTAG_1B_E(PTR,tag) \
(((varattrib_1b_e *) (PTR))->va_header = 0x01, \
- ((varattrib_1b_e *) (PTR))->va_len_1be = (len))
+ ((varattrib_1b_e *) (PTR))->va_tag = (tag))
#endif /* WORDS_BIGENDIAN */
-#define VARHDRSZ_SHORT 1
+#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
#define VARATT_SHORT_MAX 0x7F
#define VARATT_CAN_MAKE_SHORT(PTR) \
(VARATT_IS_4B_U(PTR) && \
@@ -215,7 +249,7 @@ typedef struct
#define VARATT_CONVERTED_SHORT_SIZE(PTR) \
(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
-#define VARHDRSZ_EXTERNAL 2
+#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
@@ -249,26 +283,32 @@ typedef struct
#define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR)
#define VARDATA_SHORT(PTR) VARDATA_1B(PTR)
-#define VARSIZE_EXTERNAL(PTR) VARSIZE_1B_E(PTR)
+#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR)
+#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR)))
#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR)
#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
+#define VARATT_IS_EXTERNAL_ONDISK(PTR) \
+ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK)
+#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
+ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR)
#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR))
#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len)
#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len)
#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len)
-#define SET_VARSIZE_EXTERNAL(PTR, len) SET_VARSIZE_1B_E(PTR, len)
+
+#define SET_VARTAG_EXTERNAL(PTR, tag) SET_VARTAG_1B_E(PTR, tag)
#define VARSIZE_ANY(PTR) \
- (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \
+ (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \
VARSIZE_4B(PTR)))
#define VARSIZE_ANY_EXHDR(PTR) \
- (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-VARHDRSZ_EXTERNAL : \
+ (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \
VARSIZE_4B(PTR)-VARHDRSZ))
diff --git a/src/test/regress/expected/indirect_toast.out b/src/test/regress/expected/indirect_toast.out
new file mode 100644
index 00000000000..4f4bf41973a
--- /dev/null
+++ b/src/test/regress/expected/indirect_toast.out
@@ -0,0 +1,151 @@
+CREATE TABLE toasttest(descr text, cnt int DEFAULT 0, f1 text, f2 text);
+INSERT INTO toasttest(descr, f1, f2) VALUES('two-compressed', repeat('1234567890',1000), repeat('1234567890',1000));
+INSERT INTO toasttest(descr, f1, f2) VALUES('two-toasted', repeat('1234567890',30000), repeat('1234567890',50000));
+INSERT INTO toasttest(descr, f1, f2) VALUES('one-compressed,one-null', NULL, repeat('1234567890',1000));
+INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null', NULL, repeat('1234567890',50000));
+-- check whether indirect tuples works on the most basic level
+SELECT descr, substring(make_tuple_indirect(toasttest)::text, 1, 200) FROM toasttest;
+ descr | substring
+-------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ two-compressed | (two-compressed,0,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ two-toasted | (two-toasted,0,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ one-compressed,one-null | ("one-compressed,one-null",0,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ one-toasted,one-null | ("one-toasted,one-null",0,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+-- modification without changing varlenas
+UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,1,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ (two-toasted,1,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ ("one-compressed,one-null",1,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",1,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+-- modification without modifying asigned value
+UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,2,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ (two-toasted,2,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ ("one-compressed,one-null",2,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",2,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+-- modification modifying, but effectively not changing
+UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,3,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ (two-toasted,3,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ ("one-compressed,one-null",3,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",3,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
+ (two-toasted,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ ("one-compressed,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
+ (two-toasted,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ ("one-compressed,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+-- check we didn't screw with main/toast tuple visiblity
+VACUUM FREEZE toasttest;
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
+ (two-toasted,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ ("one-compressed,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+-- now create a trigger that forces all Datums to be indirect ones
+CREATE FUNCTION update_using_indirect()
+ RETURNS trigger
+ LANGUAGE plpgsql AS $$
+BEGIN
+ NEW := make_tuple_indirect(NEW);
+ RETURN NEW;
+END$$;
+CREATE TRIGGER toasttest_update_indirect
+ BEFORE INSERT OR UPDATE
+ ON toasttest
+ FOR EACH ROW
+ EXECUTE PROCEDURE update_using_indirect();
+-- modification without changing varlenas
+UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,5,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
+ (two-toasted,5,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ ("one-compressed,one-null",5,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",5,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+-- modification without modifying asigned value
+UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,6,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
+ (two-toasted,6,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ ("one-compressed,one-null",6,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",6,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+-- modification modifying, but effectively not changing
+UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,7,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
+ (two-toasted,7,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ ("one-compressed,one-null",7,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",7,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200);
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ (two-toasted,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+ ("one-compressed,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+(4 rows)
+
+INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null, via indirect', repeat('1234567890',30000), NULL);
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ (two-toasted,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+ ("one-compressed,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+ ("one-toasted,one-null, via indirect",0,1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+(5 rows)
+
+-- check we didn't screw with main/toast tuple visiblity
+VACUUM FREEZE toasttest;
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+ substring
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ (two-compressed,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ (two-toasted,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+ ("one-compressed,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ ("one-toasted,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123
+ ("one-toasted,one-null, via indirect",0,1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+(5 rows)
+
+DROP TABLE toasttest;
+DROP FUNCTION update_using_indirect();
diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source
index a72dd9861c5..aef15182874 100644
--- a/src/test/regress/input/create_function_1.source
+++ b/src/test/regress/input/create_function_1.source
@@ -52,6 +52,11 @@ CREATE FUNCTION set_ttdummy (int4)
AS '@libdir@/regress@DLSUFFIX@'
LANGUAGE C STRICT;
+CREATE FUNCTION make_tuple_indirect (record)
+ RETURNS record
+ AS '@libdir@/regress@DLSUFFIX@'
+ LANGUAGE C STRICT;
+
-- Things that shouldn't work:
CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL
diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source
index 61b87ed953a..9761d127e1f 100644
--- a/src/test/regress/output/create_function_1.source
+++ b/src/test/regress/output/create_function_1.source
@@ -47,6 +47,10 @@ CREATE FUNCTION set_ttdummy (int4)
RETURNS int4
AS '@libdir@/regress@DLSUFFIX@'
LANGUAGE C STRICT;
+CREATE FUNCTION make_tuple_indirect (record)
+ RETURNS record
+ AS '@libdir@/regress@DLSUFFIX@'
+ LANGUAGE C STRICT;
-- Things that shouldn't work:
CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL
AS 'SELECT ''not an integer'';';
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 2af28b15029..4bb9cc78579 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -98,7 +98,7 @@ test: event_trigger
# ----------
# Another group of parallel tests
# ----------
-test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock json
+test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock json indirect_toast
# ----------
# Another group of parallel tests
diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c
index e5136cfa7c8..3bd8a152859 100644
--- a/src/test/regress/regress.c
+++ b/src/test/regress/regress.c
@@ -7,7 +7,9 @@
#include <float.h>
#include <math.h>
+#include "access/htup_details.h"
#include "access/transam.h"
+#include "access/tuptoaster.h"
#include "access/xact.h"
#include "catalog/pg_type.h"
#include "commands/sequence.h"
@@ -17,6 +19,8 @@
#include "utils/builtins.h"
#include "utils/geo_decls.h"
#include "utils/rel.h"
+#include "utils/typcache.h"
+#include "utils/memutils.h"
#define P_MAXDIG 12
@@ -35,6 +39,7 @@ extern char *reverse_name(char *string);
extern int oldstyle_length(int n, text *t);
extern Datum int44in(PG_FUNCTION_ARGS);
extern Datum int44out(PG_FUNCTION_ARGS);
+extern Datum make_tuple_indirect(PG_FUNCTION_ARGS);
#ifdef PG_MODULE_MAGIC
PG_MODULE_MAGIC;
@@ -737,3 +742,90 @@ int44out(PG_FUNCTION_ARGS)
*--walk = '\0';
PG_RETURN_CSTRING(result);
}
+
+PG_FUNCTION_INFO_V1(make_tuple_indirect);
+Datum
+make_tuple_indirect(PG_FUNCTION_ARGS)
+{
+ HeapTupleHeader rec = PG_GETARG_HEAPTUPLEHEADER(0);
+ HeapTupleData tuple;
+ int ncolumns;
+ Datum *values;
+ bool *nulls;
+
+ Oid tupType;
+ int32 tupTypmod;
+ TupleDesc tupdesc;
+
+ HeapTuple newtup;
+
+ int i;
+
+ MemoryContext old_context;
+
+ /* Extract type info from the tuple itself */
+ tupType = HeapTupleHeaderGetTypeId(rec);
+ tupTypmod = HeapTupleHeaderGetTypMod(rec);
+ tupdesc = lookup_rowtype_tupdesc(tupType, tupTypmod);
+ ncolumns = tupdesc->natts;
+
+ /* Build a temporary HeapTuple control structure */
+ tuple.t_len = HeapTupleHeaderGetDatumLength(rec);
+ ItemPointerSetInvalid(&(tuple.t_self));
+ tuple.t_tableOid = InvalidOid;
+ tuple.t_data = rec;
+
+ values = (Datum *) palloc(ncolumns * sizeof(Datum));
+ nulls = (bool *) palloc(ncolumns * sizeof(bool));
+
+ heap_deform_tuple(&tuple, tupdesc, values, nulls);
+
+ old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+ for (i = 0; i < ncolumns; i++)
+ {
+ struct varlena *attr;
+ struct varlena *new_attr;
+ struct varatt_indirect redirect_pointer;
+
+ /* only work on existing, not-null varlenas */
+ if (tupdesc->attrs[i]->attisdropped ||
+ nulls[i] ||
+ tupdesc->attrs[i]->attlen != -1)
+ continue;
+
+ attr = (struct varlena *) DatumGetPointer(values[i]);
+
+ /* don't recursively indirect */
+ if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ continue;
+
+ /* copy datum, so it still lives later */
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
+ attr = heap_tuple_fetch_attr(attr);
+ else
+ {
+ struct varlena *oldattr = attr;
+ attr = palloc0(VARSIZE_ANY(oldattr));
+ memcpy(attr, oldattr, VARSIZE_ANY(oldattr));
+ }
+
+ /* build indirection Datum */
+ new_attr = (struct varlena *) palloc0(INDIRECT_POINTER_SIZE);
+ redirect_pointer.pointer = attr;
+ SET_VARTAG_EXTERNAL(new_attr, VARTAG_INDIRECT);
+ memcpy(VARDATA_EXTERNAL(new_attr), &redirect_pointer,
+ sizeof(redirect_pointer));
+
+ values[i] = PointerGetDatum(new_attr);
+ }
+
+ newtup = heap_form_tuple(tupdesc, values, nulls);
+ pfree(values);
+ pfree(nulls);
+ ReleaseTupleDesc(tupdesc);
+
+ MemoryContextSwitchTo(old_context);
+
+ PG_RETURN_HEAPTUPLEHEADER(newtup->t_data);
+}
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index d6eaa7aa4da..ceeca734d3a 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -117,6 +117,7 @@ test: xmlmap
test: functional_deps
test: advisory_lock
test: json
+test: indirect_toast
test: plancache
test: limit
test: plpgsql
diff --git a/src/test/regress/sql/indirect_toast.sql b/src/test/regress/sql/indirect_toast.sql
new file mode 100644
index 00000000000..d502480ad3f
--- /dev/null
+++ b/src/test/regress/sql/indirect_toast.sql
@@ -0,0 +1,61 @@
+CREATE TABLE toasttest(descr text, cnt int DEFAULT 0, f1 text, f2 text);
+
+INSERT INTO toasttest(descr, f1, f2) VALUES('two-compressed', repeat('1234567890',1000), repeat('1234567890',1000));
+INSERT INTO toasttest(descr, f1, f2) VALUES('two-toasted', repeat('1234567890',30000), repeat('1234567890',50000));
+INSERT INTO toasttest(descr, f1, f2) VALUES('one-compressed,one-null', NULL, repeat('1234567890',1000));
+INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null', NULL, repeat('1234567890',50000));
+
+-- check whether indirect tuples works on the most basic level
+SELECT descr, substring(make_tuple_indirect(toasttest)::text, 1, 200) FROM toasttest;
+
+-- modification without changing varlenas
+UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200);
+
+-- modification without modifying asigned value
+UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200);
+
+-- modification modifying, but effectively not changing
+UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200);
+
+UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200);
+
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+-- check we didn't screw with main/toast tuple visiblity
+VACUUM FREEZE toasttest;
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+
+-- now create a trigger that forces all Datums to be indirect ones
+CREATE FUNCTION update_using_indirect()
+ RETURNS trigger
+ LANGUAGE plpgsql AS $$
+BEGIN
+ NEW := make_tuple_indirect(NEW);
+ RETURN NEW;
+END$$;
+
+CREATE TRIGGER toasttest_update_indirect
+ BEFORE INSERT OR UPDATE
+ ON toasttest
+ FOR EACH ROW
+ EXECUTE PROCEDURE update_using_indirect();
+
+-- modification without changing varlenas
+UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200);
+
+-- modification without modifying asigned value
+UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200);
+
+-- modification modifying, but effectively not changing
+UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200);
+
+UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200);
+
+INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null, via indirect', repeat('1234567890',30000), NULL);
+
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+-- check we didn't screw with main/toast tuple visiblity
+VACUUM FREEZE toasttest;
+SELECT substring(toasttest::text, 1, 200) FROM toasttest;
+
+DROP TABLE toasttest;
+DROP FUNCTION update_using_indirect();
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-20 22:36:39 +0000