Reduce memory used by partitionwise joins

In try_partitionwise_join, we aim to break down the join between two
partitioned relations into joins between matching partitions.  To
achieve this, we iterate through each pair of partitions from the two
joining relations and create child-join relations for them.  With
potentially thousands of partitions, the local objects allocated in
each iteration can accumulate significant memory usage.  Therefore, we
opt to eagerly free these local objects at the end of each iteration.

In line with this approach, this patch frees the bitmap set that
represents the relids of child-join relations at the end of each
iteration.  Additionally, it modifies build_child_join_rel() to reuse
the AppendRelInfo structures generated within each iteration.

Author: Ashutosh Bapat
Reviewed-by: David Christensen, Richard Guo
Discussion: https://postgr.es/m/CAExHW5s4EqY43oB=ne6B2=-xLgrs9ZGeTr1NXwkGFt2j-OmaQQ@mail.gmail.com

diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c
index a3677f824fe8e567a1743a7164126f7a3de91cac..7db5e30eef8c2a93504c530b7a3d039c2de80ce1 100644 (file)
--- a/src/backend/optimizer/path/joinrels.c
+++ b/src/backend/optimizer/path/joinrels.c
@@ -1547,6 +1547,7 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
        RelOptInfo *child_joinrel;
        AppendRelInfo **appinfos;
        int         nappinfos;
+       Relids      child_relids;
 
        if (joinrel->partbounds_merged)
        {
@@ -1642,9 +1643,8 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
                                               child_rel2->relids);
 
        /* Find the AppendRelInfo structures */
-       appinfos = find_appinfos_by_relids(root,
-                                          bms_union(child_rel1->relids,
-                                                    child_rel2->relids),
+       child_relids = bms_union(child_rel1->relids, child_rel2->relids);
+       appinfos = find_appinfos_by_relids(root, child_relids,
                                           &nappinfos);
 
        /*
@@ -1662,7 +1662,7 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
        {
            child_joinrel = build_child_join_rel(root, child_rel1, child_rel2,
                                                 joinrel, child_restrictlist,
-                                                child_sjinfo);
+                                                child_sjinfo, nappinfos, appinfos);
            joinrel->part_rels[cnt_parts] = child_joinrel;
            joinrel->live_parts = bms_add_member(joinrel->live_parts, cnt_parts);
            joinrel->all_partrels = bms_add_members(joinrel->all_partrels,
@@ -1679,7 +1679,14 @@ try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
                                    child_joinrel, child_sjinfo,
                                    child_restrictlist);
 
+       /*
+        * When there are thousands of partitions involved, this loop will
+        * accumulate a significant amount of memory usage from objects that
+        * are only needed within the loop.  Free these local objects eagerly
+        * at the end of each iteration.
+        */
        pfree(appinfos);
+       bms_free(child_relids);
        free_child_join_sjinfo(child_sjinfo);
    }
 }

diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c
index e05b21c884e57d8bee0629a24a4a0a2bff9021f4..971d1c7aae50dc51adc297f80cc4c8fcd5a1ac52 100644 (file)
--- a/src/backend/optimizer/util/relnode.c
+++ b/src/backend/optimizer/util/relnode.c
@@ -876,15 +876,15 @@ build_join_rel(PlannerInfo *root,
  * 'restrictlist': list of RestrictInfo nodes that apply to this particular
  *     pair of joinable relations
  * 'sjinfo': child join's join-type details
+ * 'nappinfos' and 'appinfos': AppendRelInfo array for child relids
  */
 RelOptInfo *
 build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel,
                     RelOptInfo *inner_rel, RelOptInfo *parent_joinrel,
-                    List *restrictlist, SpecialJoinInfo *sjinfo)
+                    List *restrictlist, SpecialJoinInfo *sjinfo,
+                    int nappinfos, AppendRelInfo **appinfos)
 {
    RelOptInfo *joinrel = makeNode(RelOptInfo);
-   AppendRelInfo **appinfos;
-   int         nappinfos;
 
    /* Only joins between "other" relations land here. */
    Assert(IS_OTHER_REL(outer_rel) && IS_OTHER_REL(inner_rel));
@@ -892,16 +892,6 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel,
    /* The parent joinrel should have consider_partitionwise_join set. */
    Assert(parent_joinrel->consider_partitionwise_join);
 
-   /*
-    * Find the AppendRelInfo structures for the child baserels.  We'll need
-    * these for computing the child join's relid set, and later for mapping
-    * Vars to the child rel.
-    */
-   appinfos = find_appinfos_by_relids(root,
-                                      bms_union(outer_rel->relids,
-                                                inner_rel->relids),
-                                      &nappinfos);
-
    joinrel->reloptkind = RELOPT_OTHER_JOINREL;
    joinrel->relids = adjust_child_relids(parent_joinrel->relids,
                                          nappinfos, appinfos);
@@ -1017,8 +1007,6 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel,
                                        nappinfos, appinfos,
                                        parent_joinrel, joinrel);
 
-   pfree(appinfos);
-
    return joinrel;
 }
 

diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h
index 112e7c23d4e1a7e5edc4ee5e4e387d219265b992..f00bd55f3938db26b948ec5025652e1d4510016f 100644 (file)
--- a/src/include/optimizer/pathnode.h
+++ b/src/include/optimizer/pathnode.h
@@ -346,6 +346,7 @@ extern Bitmapset *get_param_path_clause_serials(Path *path);
 extern RelOptInfo *build_child_join_rel(PlannerInfo *root,
                                        RelOptInfo *outer_rel, RelOptInfo *inner_rel,
                                        RelOptInfo *parent_joinrel, List *restrictlist,
-                                       SpecialJoinInfo *sjinfo);
+                                       SpecialJoinInfo *sjinfo,
+                                       int nappinfos, AppendRelInfo **appinfos);
 
 #endif                         /* PATHNODE_H */