Added long-standing transaction when restoring BLOBS (uses commit every BLOB_BATCH_SIZE)

Prevent dumping of languages from template1.

diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index b6487d342596cd041088c91b0637cf0d8172158b..d8a969b41e0279623081d0869737c869be389845 100644 (file)
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -23,6 +23,10 @@
  * Modifications - 31-Jul-2000 - pjw@rhyme.com.au (1.46, 1.47)
  *     Fixed version number initialization in _allocAH (pg_backup_archiver.c)
  *
+ *
+ * Modifications - 30-Oct-2000 - pjw@rhyme.com.au
+ *     Added {Start,End}RestoreBlobs to allow extended TX during BLOB restore.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -590,6 +594,34 @@ int EndBlob(Archive* AHX, int oid)
  * BLOB Restoration
  **********/
 
+/*
+ * Called by a format handler before any blobs are restored 
+ */
+void StartRestoreBlobs(ArchiveHandle* AH)
+{
+   AH->blobCount = 0;
+}
+
+/*
+ * Called by a format handler after all blobs are restored 
+ */
+void EndRestoreBlobs(ArchiveHandle* AH)
+{
+   if (AH->txActive)
+   {
+       ahlog(AH, 2, "Committing BLOB transactions\n");
+       CommitTransaction(AH);
+   }
+
+   if (AH->blobTxActive)
+   {
+       CommitTransactionXref(AH);
+   }
+
+   ahlog(AH, 1, "Restored %d BLOBs\n", AH->blobCount);
+}
+
+
 /*
  * Called by a format handler to initiate restoration of a blob
  */
@@ -597,6 +629,8 @@ void StartRestoreBlob(ArchiveHandle* AH, int oid)
 {
    int         loOid;
 
+   AH->blobCount++;
+
    if (!AH->createdBlobXref)
    {
        if (!AH->connection)
@@ -606,7 +640,18 @@ void StartRestoreBlob(ArchiveHandle* AH, int oid)
        AH->createdBlobXref = 1;
    }
 
-   StartTransaction(AH);
+   /*
+    * Start long-running TXs if necessary
+    */
+   if (!AH->txActive)
+   {
+       ahlog(AH, 2, "Starting BLOB transactions\n");
+       StartTransaction(AH);
+   }
+   if (!AH->blobTxActive)
+   {
+       StartTransactionXref(AH);
+   }
 
    loOid = lo_creat(AH->connection, INV_READ | INV_WRITE);
    if (loOid == 0)
@@ -628,7 +673,15 @@ void EndRestoreBlob(ArchiveHandle* AH, int oid)
     lo_close(AH->connection, AH->loFd);
     AH->writingBlob = 0;
 
-   CommitTransaction(AH);
+   /*
+    * Commit every BLOB_BATCH_SIZE blobs...
+    */
+   if ( ((AH->blobCount / BLOB_BATCH_SIZE) * BLOB_BATCH_SIZE) == AH->blobCount) 
+   {
+       ahlog(AH, 2, "Committing BLOB transactions\n");
+       CommitTransaction(AH);
+       CommitTransactionXref(AH);
+   }
 }
 
 /***********

diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 41fbb5c9c06a0918aeb480969105629cd0974a94..2c7291e6c6917f678c6711c2ecac34c532546412 100644 (file)
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -62,7 +62,7 @@ typedef z_stream *z_streamp;
 
 #define K_VERS_MAJOR 1
 #define K_VERS_MINOR 4 
-#define K_VERS_REV 21 
+#define K_VERS_REV 22 
 
 /* Data block types */
 #define BLK_DATA 1
@@ -76,6 +76,9 @@ typedef z_stream *z_streamp;
 #define K_VERS_1_4 (( (1 * 256 + 4) * 256 + 0) * 256 + 0) /* Date & name in header */
 #define K_VERS_MAX (( (1 * 256 + 4) * 256 + 255) * 256 + 0)
 
+/* No of BLOBs to restore in 1 TX */
+#define BLOB_BATCH_SIZE    100
+
 struct _archiveHandle;
 struct _tocEntry;
 struct _restoreList;
@@ -186,6 +189,8 @@ typedef struct _archiveHandle {
    char                *pgport;
    PGconn              *connection;
    PGconn              *blobConnection;    /* Connection for BLOB xref */
+   int                 txActive;           /* Flag set if TX active on connection */
+   int                 blobTxActive;       /* Flag set if TX active on blobConnection */
    int                 connectToDB;        /* Flag to indicate if direct DB connection is required */
    int                 pgCopyIn;           /* Currently in libpq 'COPY IN' mode. */
    PQExpBuffer         pgCopyBuf;          /* Left-over data from incomplete lines in COPY IN */
@@ -193,6 +198,7 @@ typedef struct _archiveHandle {
    int                 loFd;               /* BLOB fd */
    int                 writingBlob;        /* Flag */
    int                 createdBlobXref;    /* Flag */
+   int                 blobCount;          /* # of blobs restored */
 
    int                 lastID;             /* Last internal ID for a TOC entry */
    char*               fSpec;              /* Archive File Spec */
@@ -256,8 +262,10 @@ extern int                 ReadInt(ArchiveHandle* AH);
 extern char*           ReadStr(ArchiveHandle* AH);
 extern int                 WriteStr(ArchiveHandle* AH, char* s);
 
+extern void                StartRestoreBlobs(ArchiveHandle* AH);
 extern void            StartRestoreBlob(ArchiveHandle* AH, int oid);
 extern void            EndRestoreBlob(ArchiveHandle* AH, int oid);
+extern void                EndRestoreBlobs(ArchiveHandle* AH);
 
 extern void            InitArchiveFmt_Custom(ArchiveHandle* AH);
 extern void            InitArchiveFmt_Files(ArchiveHandle* AH);

diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index f5b208e233ebc933b11e00a73390c51a1ce858e7..e44f02259c00e572ddbc5da2dd66d83779075e04 100644 (file)
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -585,6 +585,8 @@ static void _LoadBlobs(ArchiveHandle* AH)
 {
     int        oid;
 
+   StartRestoreBlobs(AH);
+
     oid = ReadInt(AH);
     while(oid != 0)
     {
@@ -593,6 +595,9 @@ static void _LoadBlobs(ArchiveHandle* AH)
        EndRestoreBlob(AH, oid);
        oid = ReadInt(AH);
     }
+
+   EndRestoreBlobs(AH);
+
 }
 
 /*
@@ -608,8 +613,8 @@ static void _skipBlobs(ArchiveHandle* AH)
     oid = ReadInt(AH);
     while(oid != 0)
     {
-   _skipData(AH);
-   oid = ReadInt(AH);
+       _skipData(AH);
+       oid = ReadInt(AH);
     }
 }
 

diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c
index d606508a36a4c8d80520be8cbbcab0e7effe3fb9..4b8873c3a2360e420ff520128148d2481b84c490 100644 (file)
--- a/src/bin/pg_dump/pg_backup_db.c
+++ b/src/bin/pg_dump/pg_backup_db.c
@@ -675,6 +675,17 @@ void StartTransaction(ArchiveHandle* AH)
    appendPQExpBuffer(qry, "Begin;");
 
    ExecuteSqlCommand(AH, qry, "can not start database transaction");
+   AH->txActive = true;
+}
+
+void StartTransactionXref(ArchiveHandle* AH)
+{
+   PQExpBuffer     qry = createPQExpBuffer();
+
+   appendPQExpBuffer(qry, "Begin;");
+
+   _executeSqlCommand(AH, AH->blobConnection, qry, "can not start BLOB xref transaction");
+   AH->blobTxActive = true;
 }
 
 void CommitTransaction(ArchiveHandle* AH)
@@ -684,6 +695,15 @@ void CommitTransaction(ArchiveHandle* AH)
     appendPQExpBuffer(qry, "Commit;");
 
     ExecuteSqlCommand(AH, qry, "can not commit database transaction");
+   AH->txActive = false;
 }
 
+void CommitTransactionXref(ArchiveHandle* AH)
+{
+   PQExpBuffer     qry = createPQExpBuffer();
 
+   appendPQExpBuffer(qry, "Commit;");
+
+   _executeSqlCommand(AH, AH->blobConnection, qry, "can not commit BLOB xref transaction");
+   AH->blobTxActive = false;
+}

diff --git a/src/bin/pg_dump/pg_backup_db.h b/src/bin/pg_dump/pg_backup_db.h
index 5d03967f583df5c1524116682c91922d8318fb64..3dfc6664fc934f349342ddf25b0ef28cde541f9e 100644 (file)
--- a/src/bin/pg_dump/pg_backup_db.h
+++ b/src/bin/pg_dump/pg_backup_db.h
@@ -12,5 +12,7 @@ extern int ExecuteSqlCommandBuf(ArchiveHandle* AH, void *qry, int bufLen);
 extern void CreateBlobXrefTable(ArchiveHandle* AH);
 extern void InsertBlobXref(ArchiveHandle* AH, int old, int new);
 extern void StartTransaction(ArchiveHandle* AH);
+extern void StartTransactionXref(ArchiveHandle* AH);
 extern void CommitTransaction(ArchiveHandle* AH);
+extern void CommitTransactionXref(ArchiveHandle* AH);
 

diff --git a/src/bin/pg_dump/pg_backup_files.c b/src/bin/pg_dump/pg_backup_files.c
index 1583a497b9c87e4c27f1d040c60e681327f75a91..1624bf14355a791fa02a53c57e13caff09df5c60 100644 (file)
--- a/src/bin/pg_dump/pg_backup_files.c
+++ b/src/bin/pg_dump/pg_backup_files.c
@@ -318,6 +318,8 @@ static void _LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt)
    lclContext*     ctx = (lclContext*)AH->formatData;
    char            fname[K_STD_BUF_SIZE];
 
+   StartRestoreBlobs(AH);
+
    ctx->blobToc = fopen("blobs.toc", PG_BINARY_R);
 
    _getBlobTocEntry(AH, &oid, fname);
@@ -331,6 +333,8 @@ static void _LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt)
     }
 
    fclose(ctx->blobToc);
+
+   EndRestoreBlobs(AH);
 }
 
 

diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index a137513e78c2e6f15bcb1352df851465bd4a8347..cb4a9e906d22e23952e12e70b44cd3d646216ea8 100644 (file)
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -627,6 +627,8 @@ static void _LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt)
    int             cnt;
    char            buf[4096];
 
+   StartRestoreBlobs(AH);
+
    th = tarOpen(AH, NULL, 'r'); /* Open next file */
    while (th != NULL)
    {
@@ -652,21 +654,8 @@ static void    _LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt)
        th = tarOpen(AH, NULL, 'r');
    }
 
-   /*
-    * ctx->blobToc = tarOpen(AH, "blobs.toc", 'r');
-    *
-    * _getBlobTocEntry(AH, &oid, fname);
-    *
-     * while(oid != 0)
-     * {
-    *      StartRestoreBlob(AH, oid);
-    *      _PrintFileData(AH, fname, ropt);
-    *      EndRestoreBlob(AH, oid);
-    *      _getBlobTocEntry(AH, &oid, fname);
-     * }
-    *
-    * tarClose(AH, ctx->blobToc);
-    */ 
+   EndRestoreBlobs(AH);
+
 }
 
 

diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 0ae0ee00144182773c2843f96627b660b03c27e2..4b765f528806ad1c216b60ed65ba70888766a79d 100644 (file)
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -22,7 +22,7 @@
  *
  *
  * IDENTIFICATION
- *   $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_dump.c,v 1.176 2000/10/24 13:24:30 pjw Exp $
+ *   $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_dump.c,v 1.177 2000/10/31 14:20:30 pjw Exp $
  *
  * Modifications - 6/10/96 - dave@bensoft.com - version 1.13.dhb
  *
@@ -2872,6 +2872,7 @@ dumpProcLangs(Archive *fout, FuncInfo *finfo, int numFuncs,
    int         i_lanpltrusted;
    int         i_lanplcallfoid;
    int         i_lancompiler;
+   Oid         lanoid;
    char       *lanname;
    char       *lancompiler;
    const char *lanplcallfoid;
@@ -2898,7 +2899,13 @@ dumpProcLangs(Archive *fout, FuncInfo *finfo, int numFuncs,
 
    for (i = 0; i < ntups; i++)
    {
+       lanoid = atoi(PQgetvalue(res, i, i_oid));
+       if (lanoid <= g_last_builtin_oid)
+           continue;
+
        lanplcallfoid = PQgetvalue(res, i, i_lanplcallfoid);
+
+
        for (fidx = 0; fidx < numFuncs; fidx++)
        {
            if (!strcmp(finfo[fidx].oid, lanplcallfoid))