In pg_upgrade, use cached copy of directory listing, rather than calling

scandir() with a pattern for every table.

Optimization after report of pg_upgrade slowness with 150k tables.

diff --git a/contrib/pg_upgrade/info.c b/contrib/pg_upgrade/info.c
index b4e414ec8d97874bfb44923c2b128d45ce050f1f..a8d0bddc1912e90fe7969900ad1073da64cadcdf 100644 (file)
--- a/contrib/pg_upgrade/info.c
+++ b/contrib/pg_upgrade/info.c
@@ -30,8 +30,8 @@ static void map_rel_by_id(Oid oldid, Oid newid,
              const char *old_tablespace, const DbInfo *old_db,
              const DbInfo *new_db, const char *olddata,
              const char *newdata, FileNameMap *map);
-static RelInfo *relarr_lookup_reloid(
-                    RelInfoArr *rel_arr, Oid oid, Cluster whichCluster);
+static RelInfo *relarr_lookup_reloid(RelInfoArr *rel_arr,
+               Oid oid, Cluster whichCluster);
 static RelInfo *relarr_lookup_rel(RelInfoArr *rel_arr,
                  const char *nspname, const char *relname,
                  Cluster whichCluster);
@@ -172,8 +172,8 @@ map_rel_by_id(Oid oldid, Oid newid,
         * relation belongs to the default tablespace, hence relfiles would
         * exist in the data directories.
         */
-       snprintf(map->old_file, sizeof(map->old_file), "%s/base/%u", olddata, old_db->db_oid);
-       snprintf(map->new_file, sizeof(map->new_file), "%s/base/%u", newdata, new_db->db_oid);
+       snprintf(map->old_dir, sizeof(map->old_dir), "%s/base/%u", olddata, old_db->db_oid);
+       snprintf(map->new_dir, sizeof(map->new_dir), "%s/base/%u", newdata, new_db->db_oid);
    }
    else
    {
@@ -181,9 +181,9 @@ map_rel_by_id(Oid oldid, Oid newid,
         * relation belongs to some tablespace, hence copy its physical
         * location
         */
-       snprintf(map->old_file, sizeof(map->old_file), "%s%s/%u", old_tablespace,
+       snprintf(map->old_dir, sizeof(map->old_dir), "%s%s/%u", old_tablespace,
                 old_cluster.tablespace_suffix, old_db->db_oid);
-       snprintf(map->new_file, sizeof(map->new_file), "%s%s/%u", old_tablespace,
+       snprintf(map->new_dir, sizeof(map->new_dir), "%s%s/%u", old_tablespace,
                 new_cluster.tablespace_suffix, new_db->db_oid);
    }
 }
@@ -318,6 +318,7 @@ get_rel_infos(const DbInfo *dbinfo,
     * pg_largeobject_loid_pn_index's relfilenode can change if the table was
     * reindexed so we get the relfilenode for each database and migrate it as
     * a normal user table.
+    * Order by tablespace so we can cache the directory contents efficiently.
     */
 
    snprintf(query, sizeof(query),
@@ -338,7 +339,7 @@ get_rel_infos(const DbInfo *dbinfo,
             "GROUP BY  c.oid, n.nspname, c.relname, c.relfilenode,"
             "          c.reltoastrelid, t.spclocation, "
             "          n.nspname "
-            "ORDER BY n.nspname, c.relname;",
+            "ORDER BY t.spclocation, n.nspname, c.relname;",
             FirstNormalObjectId,
    /* see the comment at the top of old_8_3_create_sequence_script() */
             (GET_MAJOR_VERSION(old_cluster.major_version) <= 803) ?

diff --git a/contrib/pg_upgrade/pg_upgrade.h b/contrib/pg_upgrade/pg_upgrade.h
index 3756f5cbb0a4f410a651ee0b3052fa491a2e724f..2aa220815820fa3921599230a412dd666402a6c6 100644 (file)
--- a/contrib/pg_upgrade/pg_upgrade.h
+++ b/contrib/pg_upgrade/pg_upgrade.h
@@ -85,10 +85,10 @@ typedef struct
  */
 typedef struct
 {
+   char        old_dir[MAXPGPATH];
+   char        new_dir[MAXPGPATH];
    Oid         old_relfilenode;    /* Relfilenode of the old relation */
    Oid         new_relfilenode;    /* Relfilenode of the new relation */
-   char        old_file[MAXPGPATH];
-   char        new_file[MAXPGPATH];
    char        old_nspname[NAMEDATALEN];       /* old name of the namespace */
    char        old_relname[NAMEDATALEN];       /* old name of the relation */
    char        new_nspname[NAMEDATALEN];       /* new name of the namespace */
@@ -255,10 +255,8 @@ void check_old_cluster(bool live_check,
                  char **sequence_script_file_name);
 void       check_new_cluster(void);
 void       report_clusters_compatible(void);
-void issue_warnings(
-              char *sequence_script_file_name);
-void output_completion_banner(
-                        char *deletion_script_file_name);
+void issue_warnings(char *sequence_script_file_name);
+void output_completion_banner(char *deletion_script_file_name);
 void       check_cluster_versions(void);
 void       check_cluster_compatibility(bool live_check);
 void       create_script_for_old_cluster_deletion(char **deletion_script_file_name);
@@ -319,14 +317,12 @@ typedef void *pageCnvCtx;
 #endif
 
 int            dir_matching_filenames(const struct dirent * scan_ent);
-int pg_scandir(const char *dirname,
-          struct dirent *** namelist,
-          int (*selector) (const struct dirent *));
-const char *copyAndUpdateFile(
-                 pageCnvCtx *pageConverter, const char *src,
+int pg_scandir(const char *dirname, struct dirent *** namelist,
+              int (*selector) (const struct dirent *));
+const char *copyAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
                  const char *dst, bool force);
-const char *linkAndUpdateFile(
-               pageCnvCtx *pageConverter, const char *src, const char *dst);
+const char *linkAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
+                             const char *dst);
 
 void       check_hard_link(void);
 
@@ -374,8 +370,7 @@ PGresult *executeQueryOrDie(PGconn *conn,
 
 void       start_postmaster(Cluster whichCluster, bool quiet);
 void       stop_postmaster(bool fast, bool quiet);
-uint32 get_major_server_version(char **verstr,
-                        Cluster whichCluster);
+uint32 get_major_server_version(char **verstr, Cluster whichCluster);
 void       check_for_libpq_envvars(void);
 
 

diff --git a/contrib/pg_upgrade/relfilenode.c b/contrib/pg_upgrade/relfilenode.c
index ff1a73cdec2c18dab0084b9b064a13531068408c..7b73b5e91f5603d4a83b130adcd6458cb2efd1d7 100644 (file)
--- a/contrib/pg_upgrade/relfilenode.c
+++ b/contrib/pg_upgrade/relfilenode.c
@@ -110,20 +110,38 @@ static void
 transfer_single_new_db(pageCnvCtx *pageConverter,
                       FileNameMap *maps, int size)
 {
+   char        old_dir[MAXPGPATH];
+   struct dirent **namelist = NULL;
+   int         numFiles = 0;
    int         mapnum;
+   int         fileno;
+
+   old_dir[0] = '\0';
 
    for (mapnum = 0; mapnum < size; mapnum++)
    {
        char        old_file[MAXPGPATH];
        char        new_file[MAXPGPATH];
-       struct dirent **namelist = NULL;
-       int         numFiles;
+
+       /* Changed tablespaces?  Need a new directory scan? */
+       if (strcmp(maps[mapnum].old_dir, old_dir) != 0)
+       {
+           if (numFiles > 0)
+           {
+               for (fileno = 0; fileno < numFiles; fileno++)
+                   pg_free(namelist[fileno]);
+               pg_free(namelist);
+           }
+
+           snprintf(old_dir, sizeof(old_dir), "%s", maps[mapnum].old_dir);
+           numFiles = pg_scandir(old_dir, &namelist, NULL);
+       }
 
        /* Copying files might take some time, so give feedback. */
 
-       snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_file,
+       snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_dir,
                 maps[mapnum].old_relfilenode);
-       snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_file,
+       snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_dir,
                 maps[mapnum].new_relfilenode);
        pg_log(PG_REPORT, OVERWRITE_MESSAGE, old_file);
 
@@ -139,28 +157,27 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
        if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804)
        {
            /*
-            * Now copy/link any fsm and vm files, if they exist
+            * Copy/link any fsm and vm files, if they exist
             */
            snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u_",
                     maps[mapnum].old_relfilenode);
-           numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
 
-           while (numFiles--)
+           for (fileno = 0; fileno < numFiles; fileno++)
            {
-               snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
-                        namelist[numFiles]->d_name);
-               snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
-                        maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '_'));
-
-               unlink(new_file);
-               transfer_relfile(pageConverter, old_file, new_file,
-                         maps[mapnum].old_nspname, maps[mapnum].old_relname,
-                        maps[mapnum].new_nspname, maps[mapnum].new_relname);
-
-               pg_free(namelist[numFiles]);
+               if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
+                   strlen(scandir_file_pattern)) == 0)
+               {
+                   snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
+                            namelist[fileno]->d_name);
+                   snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
+                            maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '_'));
+   
+                   unlink(new_file);
+                   transfer_relfile(pageConverter, old_file, new_file,
+                             maps[mapnum].old_nspname, maps[mapnum].old_relname,
+                            maps[mapnum].new_nspname, maps[mapnum].new_relname);
+               }
            }
-
-           pg_free(namelist);
        }
 
        /*
@@ -172,23 +189,30 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
         */
        snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u.",
                 maps[mapnum].old_relfilenode);
-       numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
 
-       while (numFiles--)
+       for (fileno = 0; fileno < numFiles; fileno++)
        {
-           snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
-                    namelist[numFiles]->d_name);
-           snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
-                    maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '.'));
-
-           unlink(new_file);
-           transfer_relfile(pageConverter, old_file, new_file,
-                         maps[mapnum].old_nspname, maps[mapnum].old_relname,
-                        maps[mapnum].new_nspname, maps[mapnum].new_relname);
+           if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
+               strlen(scandir_file_pattern)) == 0)
+           {
+               snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
+                        namelist[fileno]->d_name);
+               snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
+                        maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '.'));
 
-           pg_free(namelist[numFiles]);
+               unlink(new_file);
+               transfer_relfile(pageConverter, old_file, new_file,
+                             maps[mapnum].old_nspname, maps[mapnum].old_relname,
+                            maps[mapnum].new_nspname, maps[mapnum].new_relname);
+           }
        }
+   }
 
+
+   if (numFiles > 0)
+   {
+       for (fileno = 0; fileno < numFiles; fileno++)
+           pg_free(namelist[fileno]);
        pg_free(namelist);
    }
 }