Add log_verbosity = 'silent' support to COPY command.

Previously, when the on_error option was set to ignore, the COPY command
would always log NOTICE messages for input rows discarded due to
data type incompatibility. Users had no way to suppress these messages.

This commit introduces a new log_verbosity setting, 'silent',
which prevents the COPY command from emitting NOTICE messages
when on_error = 'ignore' is used, even if rows are discarded.
This feature is particularly useful when processing malformed files
frequently, where a flood of NOTICE messages can be undesirable.

For example, when frequently loading malformed files via the COPY command
or querying foreign tables using file_fdw (with an upcoming patch to
add on_error support for file_fdw), users may prefer to suppress
these messages to reduce log noise and improve clarity.

Author: Atsushi Torikoshi
Reviewed-by: Masahiko Sawada, Fujii Masao
Discussion: https://postgr.es/m/ab59dad10490ea3734cf022b16c24cfd@oss.nttdata.com

diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml
index 33633447f16f32ee813a23d739d2dde3ffb0d927..b9413d48925aa579511ca7e9e2c0f803ec84e2d3 100644 (file)
--- a/doc/src/sgml/ref/copy.sgml
+++ b/doc/src/sgml/ref/copy.sgml
@@ -407,6 +407,8 @@ COPY { <replaceable class="parameter">table_name</replaceable> [ ( <replaceable
       <literal>verbose</literal>, a <literal>NOTICE</literal> message
       containing the line of the input file and the column name whose input
       conversion has failed is emitted for each discarded row.
+      When it is set to <literal>silent</literal>, no message is emitted
+      regarding ignored rows.
      </para>
     </listitem>
    </varlistentry>
@@ -428,9 +430,11 @@ COPY { <replaceable class="parameter">table_name</replaceable> [ ( <replaceable
     <listitem>
      <para>
       Specifies the amount of messages emitted by a <command>COPY</command>
-      command: <literal>default</literal> or <literal>verbose</literal>. If
-      <literal>verbose</literal> is specified, additional messages are emitted
-      during processing.
+      command: <literal>default</literal>, <literal>verbose</literal>, or
+      <literal>silent</literal>.
+      If <literal>verbose</literal> is specified, additional messages are
+      emitted during processing.
+      <literal>silent</literal> suppresses both verbose and default messages.
      </para>
      <para>
       This is currently used in <command>COPY FROM</command> command when

diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 3bb579a3a4464f3efd0af8d27e0215210104d902..03eb7a4ebacbe9223308c8cb0c7def50cab45f28 100644 (file)
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -427,9 +427,11 @@ defGetCopyLogVerbosityChoice(DefElem *def, ParseState *pstate)
    char       *sval;
 
    /*
-    * Allow "default", or "verbose" values.
+    * Allow "silent", "default", or "verbose" values.
     */
    sval = defGetString(def);
+   if (pg_strcasecmp(sval, "silent") == 0)
+       return COPY_LOG_VERBOSITY_SILENT;
    if (pg_strcasecmp(sval, "default") == 0)
        return COPY_LOG_VERBOSITY_DEFAULT;
    if (pg_strcasecmp(sval, "verbose") == 0)

diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 2d3462913e17309e60eaa5a454d840b1a876386a..47879994f7848ccc87af86f733191f94265c3c35 100644 (file)
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1320,7 +1320,8 @@ CopyFrom(CopyFromState cstate)
    error_context_stack = errcallback.previous;
 
    if (cstate->opts.on_error != COPY_ON_ERROR_STOP &&
-       cstate->num_errors > 0)
+       cstate->num_errors > 0 &&
+       cstate->opts.log_verbosity >= COPY_LOG_VERBOSITY_DEFAULT)
        ereport(NOTICE,
                errmsg_plural("%llu row was skipped due to data type incompatibility",
                              "%llu rows were skipped due to data type incompatibility",

diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c
index a7ccde6d7dffc7c67bb01d12eb962849b246d916..6530b0f1ce3c1b8c1794bbb36ed50f6e575a71ad 100644 (file)
--- a/src/bin/psql/tab-complete.c
+++ b/src/bin/psql/tab-complete.c
@@ -2916,7 +2916,7 @@ psql_completion(const char *text, int start, int end)
 
    /* Complete COPY <sth> FROM filename WITH (LOG_VERBOSITY */
    else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "LOG_VERBOSITY"))
-       COMPLETE_WITH("default", "verbose");
+       COMPLETE_WITH("silent", "default", "verbose");
 
    /* Complete COPY <sth> FROM <sth> WITH (<options>) */
    else if (Matches("COPY|\\copy", MatchAny, "FROM", MatchAny, "WITH", MatchAny))

diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 141fd48dc10dfb30263af96625d503acc142b7a1..6f64d97fdd94396bc5e61041fed19b4621be4186 100644 (file)
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -45,7 +45,9 @@ typedef enum CopyOnErrorChoice
  */
 typedef enum CopyLogVerbosityChoice
 {
-   COPY_LOG_VERBOSITY_DEFAULT = 0, /* logs no additional messages, default */
+   COPY_LOG_VERBOSITY_SILENT = -1, /* logs none */
+   COPY_LOG_VERBOSITY_DEFAULT = 0, /* logs no additional messages. As this is
+                                    * the default, assign 0 */
    COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */
 } CopyLogVerbosityChoice;
 

diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out
index 61a19cdc4ca78d515d2a523ff6ccd3c674cd3b96..4e752977b53a5c29a3caf0821f9ed2a90910f4e3 100644 (file)
--- a/src/test/regress/expected/copy2.out
+++ b/src/test/regress/expected/copy2.out
@@ -760,6 +760,7 @@ COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity verbose);
 NOTICE:  skipping row due to data type incompatibility at line 2 for column "l": null input
 CONTEXT:  COPY check_ign_err2
 NOTICE:  1 row was skipped due to data type incompatibility
+COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity silent);
 -- reset context choice
 \set SHOW_CONTEXT errors
 SELECT * FROM check_ign_err;
@@ -774,7 +775,8 @@ SELECT * FROM check_ign_err2;
  n |  m  | k |   l   
 ---+-----+---+-------
  1 | {1} | 1 | 'foo'
-(1 row)
+ 3 | {3} | 3 | 'bar'
+(2 rows)
 
 -- test datatype error that can't be handled as soft: should fail
 CREATE TABLE hard_err(foo widget);

diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql
index 8b1496219471020ee248a26fbd626e8025d03db1..fa6aa17344a25308c108929c35edf2799ce621a4 100644 (file)
--- a/src/test/regress/sql/copy2.sql
+++ b/src/test/regress/sql/copy2.sql
@@ -533,6 +533,10 @@ COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity verbose);
 1  {1} 1   'foo'
 2  {2} 2   \N
 \.
+COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity silent);
+3  {3} 3   'bar'
+4  {4} 4   \N
+\.
 
 -- reset context choice
 \set SHOW_CONTEXT errors