Speed improvement for large object restore.

Mario Weilguni

diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 2c8ea7d828f899992e07f98cc3008bd7d9eb1094..8d2c2792bfe3ffc9bfa8e2d68a3adc741af775fb 100644 (file)
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -15,7 +15,7 @@
  *
  *
  * IDENTIFICATION
- *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.42 2002/02/11 00:18:20 tgl Exp $
+ *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.43 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -819,6 +819,9 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
        AH->createdBlobXref = 1;
    }
 
+   /* Initialize the LO Buffer */
+   AH->lo_buf_used = 0;
+
    /*
     * Start long-running TXs if necessary
     */
@@ -848,6 +851,19 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
 void
 EndRestoreBlob(ArchiveHandle *AH, Oid oid)
 {
+        if(AH->lo_buf_used > 0) {
+     /* Write remaining bytes from the LO buffer */
+     int res;
+          res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf, AH->lo_buf_used);
+
+     ahlog(AH, 5, "wrote remaining %d bytes of large object data (result = %d)\n",
+         (int)AH->lo_buf_used, res);
+     if (res != AH->lo_buf_used)
+       die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
+            res, AH->lo_buf_used);
+          AH->lo_buf_used = 0;
+        }
+
    lo_close(AH->connection, AH->loFd);
    AH->writingBlob = 0;
 
@@ -1228,14 +1244,27 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
 
    if (AH->writingBlob)
    {
-       res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
-       ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
-             (int) (size * nmemb), res);
-       if (res != size * nmemb)
-           die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
-                        res, (int) (size * nmemb));
+           if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
+         /* Split LO buffer */
+         int remaining = AH->lo_buf_size - AH->lo_buf_used;
+         int slack = nmemb * size - remaining;
 
-       return res;
+         memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
+         res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
+         ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
+                   AH->lo_buf_size, res);
+         if (res != AH->lo_buf_size)
+           die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
+                        res, AH->lo_buf_size);
+             memcpy(AH->lo_buf, ptr + remaining, slack);
+         AH->lo_buf_used = slack;
+          } else {
+            /* LO Buffer is still large enough, buffer it */
+        memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
+        AH->lo_buf_used += size * nmemb;
+          }
+
+          return size * nmemb;
    }
    else if (AH->gzOut)
    {

diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index cb7671ca69d5a745dcead83e0e7e5c31f9f932ae..0fbce03f52c8b79a053ce043b69323959504a88d 100644 (file)
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -17,7 +17,7 @@
  *
  *
  * IDENTIFICATION
- *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.41 2001/11/05 17:46:30 momjian Exp $
+ *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.42 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  * -   Initial version.
@@ -41,6 +41,7 @@
 #include <errno.h>
 
 #include "pqexpbuffer.h"
+#define LOBBUFSIZE 32768
 
 #ifdef HAVE_LIBZ
 #include <zlib.h>
@@ -240,6 +241,9 @@ typedef struct _archiveHandle
 
    RestoreOptions *ropt;       /* Used to check restore options in
                                 * ahwrite etc */
+   void                    *lo_buf;
+   int                     lo_buf_used;
+   int                     lo_buf_size;
 } ArchiveHandle;
 
 typedef struct _tocEntry

diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 314e90d4df7fbf456fdfa5d829cdfd18bb5168f1..93e58378f4f620a978e4f1173832363b5c0da7e4 100644 (file)
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -19,7 +19,7 @@
  *
  *
  * IDENTIFICATION
- *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.17 2001/11/27 23:48:12 tgl Exp $
+ *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.18 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -153,6 +153,12 @@ InitArchiveFmt_Custom(ArchiveHandle *AH)
    if (ctx->zp == NULL)
        die_horribly(AH, modulename, "out of memory\n");
 
+   /* Initialize LO buffering */
+   AH->lo_buf_size = LOBBUFSIZE;
+   AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+   if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
+
    /*
     * zlibOutSize is the buffer size we tell zlib it can output to.  We
     * actually allocate one extra byte because some routines want to

diff --git a/src/bin/pg_dump/pg_backup_files.c b/src/bin/pg_dump/pg_backup_files.c
index dfc1fdd2a1cf7b5b9dde7546cb80adebe83b8f48..33e49b2d38027f18a15c4f8c9dbcefe2e28ded04 100644 (file)
--- a/src/bin/pg_dump/pg_backup_files.c
+++ b/src/bin/pg_dump/pg_backup_files.c
@@ -20,7 +20,7 @@
  *
  *
  * IDENTIFICATION
- *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.14 2001/10/25 05:49:52 momjian Exp $
+ *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.15 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -113,6 +113,12 @@ InitArchiveFmt_Files(ArchiveHandle *AH)
    AH->formatData = (void *) ctx;
    ctx->filePos = 0;
 
+   /* Initialize LO buffering */
+   AH->lo_buf_size = LOBBUFSIZE;
+   AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+   if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
+
    /*
     * Now open the TOC file
     */

diff --git a/src/bin/pg_dump/pg_backup_null.c b/src/bin/pg_dump/pg_backup_null.c
index ee3b85eb4c85c1c7ee3faa5ec04767e72f5b5506..af8f52a7c206a6f8ab275669db833520248d6ca4 100644 (file)
--- a/src/bin/pg_dump/pg_backup_null.c
+++ b/src/bin/pg_dump/pg_backup_null.c
@@ -17,7 +17,7 @@
  *
  *
  * IDENTIFICATION
- *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.7 2001/06/27 21:21:37 petere Exp $
+ *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.8 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 09-Jul-2000 - pjw@rhyme.com.au
  *
@@ -64,7 +64,6 @@ InitArchiveFmt_Null(ArchiveHandle *AH)
     */
    if (AH->mode == archModeRead)
        die_horribly(AH, NULL, "this format cannot be read\n");
-
 }
 
 /*

diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 557ccbecbdc4a7c528165efe3f007c6e488e0e83..efd8a03013358764b833a47fa1de6722f43198f7 100644 (file)
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -16,7 +16,7 @@
  *
  *
  * IDENTIFICATION
- *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.20 2001/10/28 06:25:58 momjian Exp $
+ *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.21 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -157,6 +157,12 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
    ctx = (lclContext *) malloc(sizeof(lclContext));
    AH->formatData = (void *) ctx;
    ctx->filePos = 0;
+   
+   /* Initialize LO buffering */
+   AH->lo_buf_size = LOBBUFSIZE;
+   AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+   if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
 
    /*
     * Now open the TOC file