Import named regex captures as fish variables

CHANGELOG.rst

@@ -38,6 +38,7 @@ Notable improvements and fixes
    or non-matching wildcards, as these could be defined differently at
    runtime (especially for functions). This makes it usable as a static syntax checker (#977).
 -  ``type`` is now a builtin and therefore much faster (#7342).
+-  ``string match --regex`` now imports named PCRE2 capture groups as fish variables (#7459).
 
 Syntax changes and new commands
 -------------------------------

doc_src/cmds/string-match.rst

@@ -27,6 +27,8 @@ If ``--index`` or ``-n`` is given, each match is reported as a 1-based start pos
 
 If ``--regex`` or ``-r`` is given, PATTERN is interpreted as a Perl-compatible regular expression, which does not have to match the entire STRING. For a regular expression containing capturing groups, multiple items will be reported for each match, one for the entire match and one for each capturing group. With this, only the matching part of the STRING will be reported, unless ``--entire`` is given.
 
+When matching via regular expressions, it is possible to directly import matches as fish variables by means of named capture groups using the PCRE2 syntax. This behavior is automatic and occurs in addition to any other match reporting. The default behavior with `--regex` results in the initialization of fish variables in the default scope containing the matched text corresponding to each named capture group. A named capture group matching a zero-length string will be initialized as a fish variable containing a likewise empty string (i.e. the equivalent of `""`), but a named capture group that did not match will result in an empty (null) fish variable. When `--regex` is used in conjunction with `--all`, this behavior changes slightly: for each of the *n* matching sequences, the *n*th index of each named variable will contain the contents of the corresponding named capture group for the *n*th match, but will contain an empty string if the group was empty or if the group did not match.
+
 If ``--invert`` or ``-v`` is used the selected lines will be only those which do not match the given glob pattern or regular expression.
 
 Exit status: 0 if at least one match was found, or 1 otherwise.
@@ -104,4 +106,21 @@ Match Regex Examples
     >_ string match -r -i '0x[0-9a-f]{1,8}' 'int magic = 0xBadC0de;'
     0xBadC0de
 
+    >_ echo $version
+    3.1.2-1575-ga2ff32d90
+    >_ string match -rq '(?<major>\d+).(?<minor>\d+).(?<revision>\d+)' -- $version
+    >_ echo "You are using fish $major!"
+    You are using fish 3!
+
+    >_ string match -raq ' *(?<sentence>[^.!?]+)(?<punctuation>[.!?])?' "hello, friend. goodbye"
+    >_ printf "%s\n" -- $sentence
+    hello, friend
+    goodbye
+    >_ printf "%s\n" -- $punctuation
+    .
+
+    >_ string match -rq '(?<word>hello)' 'hi'
+    >_ count $word
+    0
+
 .. END EXAMPLES

src/builtin_string.cpp

@@ -1,6 +1,8 @@
 // Implementation of the string builtin.
 #include "config.h"  // IWYU pragma: keep
 
+#include <functional>
+
 #define PCRE2_CODE_UNIT_WIDTH WCHAR_T_BITS
 #ifdef _WIN32
 #define PCRE2_STATIC
@@ -23,18 +25,18 @@
 
 #include "builtin.h"
 #include "common.h"
+#include "env.h"
 #include "fallback.h"  // IWYU pragma: keep
 #include "future_feature_flags.h"
 #include "io.h"
 #include "parse_util.h"
+#include "parser.h"
 #include "pcre2.h"
 #include "wcstringutil.h"
 #include "wgetopt.h"
 #include "wildcard.h"
 #include "wutil.h"  // IWYU pragma: keep
 
-class parser_t;
-
 // How many bytes we read() at once.
 // Bash uses 128 here, so we do too (see READ_CHUNK_SIZE).
 // This should be about the size of a line.
@@ -827,9 +829,15 @@ struct compiled_regex_t {
 class pcre2_matcher_t : public string_matcher_t {
     const wchar_t *argv0;
     compiled_regex_t regex;
+    parser_t &parser;
+
+    enum class match_result_t {
+        pcre2_error = -1,
+        no_match = 0,
+        match = 1,
+    };
 
-    int report_match(const wcstring &arg, int pcre2_rc) {
-        // Return values: -1 = error, 0 = no match, 1 = match.
+    match_result_t report_match(const wcstring &arg, int pcre2_rc) {
         if (pcre2_rc == PCRE2_ERROR_NOMATCH) {
             if (opts.invert_match && !opts.quiet) {
                 if (opts.index) {
@@ -840,17 +848,17 @@ class pcre2_matcher_t : public string_matcher_t {
                 }
             }
 
-            return opts.invert_match ? 1 : 0;
+            return opts.invert_match ? match_result_t::match : match_result_t::no_match;
         } else if (pcre2_rc < 0) {
             string_error(streams, _(L"%ls: Regular expression match error: %ls\n"), argv0,
                          pcre2_strerror(pcre2_rc).c_str());
-            return -1;
+            return match_result_t::pcre2_error;
         } else if (pcre2_rc == 0) {
             // The output vector wasn't big enough. Should not happen.
             string_error(streams, _(L"%ls: Regular expression internal error\n"), argv0);
-            return -1;
+            return match_result_t::pcre2_error;
         } else if (opts.invert_match) {
-            return 0;
+            return match_result_t::no_match;
         }
 
         if (opts.entire && !opts.quiet) {
@@ -874,15 +882,154 @@ class pcre2_matcher_t : public string_matcher_t {
             }
         }
 
-        return opts.invert_match ? 0 : 1;
+        return opts.invert_match ? match_result_t::no_match : match_result_t::match;
     }
 
+    class regex_importer_t {
+       private:
+        std::map<wcstring, std::vector<wcstring>> matches_;
+        parser_t &parser_;
+        const wcstring &haystack_;
+        const compiled_regex_t &regex_;
+        /// fish variables may be empty, but there's no such thing as a fish array that contains
+        /// an empty value/index. Since a match may evaluate to a literal empty string, we can't
+        /// use that as a sentinel value in place of null/none to indicate that no matches were
+        /// found, which is required to determine whether, in the case of a single
+        /// `string match -r` invocation without `--all` we export a variable set to "" or an
+        /// empty variable.
+        bool match_found_ = false;
+        bool skip_import_ = true;
+
+       public:
+        regex_importer_t(parser_t &parser, const wcstring &haystack, const compiled_regex_t &regex)
+            : parser_(parser), haystack_(haystack), regex_(regex) {}
+
+        /// Enumerates the named groups in the compiled PCRE2 expression, validates the names of
+        /// the groups as variable names, and initializes their value (overriding any previous
+        /// contents).
+        bool init(io_streams_t &streams) {
+            PCRE2_SPTR name_table;
+            uint32_t name_entry_size;
+            uint32_t name_count;
+
+            pcre2_pattern_info(regex_.code, PCRE2_INFO_NAMETABLE, &name_table);
+            pcre2_pattern_info(regex_.code, PCRE2_INFO_NAMEENTRYSIZE, &name_entry_size);
+            pcre2_pattern_info(regex_.code, PCRE2_INFO_NAMECOUNT, &name_count);
+
+            struct name_table_entry_t {
+#if PCRE2_CODE_UNIT_WIDTH == 8
+                uint8_t match_index_msb;
+                uint8_t match_index_lsb;
+                char name[];
+#elif PCRE2_CODE_UNIT_WIDTH == 16
+                uint16_t match_index;
+                char16_t name[];
+#else
+                uint32_t match_index;
+#if WCHAR_T_BITS == PCRE2_CODE_UNIT_WIDTH
+                wchar_t name[];
+#else
+                char32_t name[];
+#endif  // WCHAR_T_BITS
+#endif  // PCRE2_CODE_UNIT_WIDTH
+            };
+
+            auto *names = static_cast<name_table_entry_t *>((void *)(name_table));
+            for (uint32_t i = 0; i < name_count; ++i) {
+                auto &name_entry = names[i * name_entry_size];
+
+                if (env_var_t::flags_for(name_entry.name) & env_var_t::flag_read_only) {
+                    // Modification of read-only variables is not allowed
+                    streams.err.append_format(
+                        L"Modification of read-only variable \"%S\" is not allowed\n",
+                        name_entry.name);
+                    return false;
+                }
+                matches_.emplace(name_entry.name, std::vector<wcstring>{});
+            }
+
+            skip_import_ = false;
+            return true;
+        }
+
+        /// This member function should be called each time a match is found
+        void import_vars(bool match_found) {
+            match_found_ |= match_found;
+            if (!match_found) {
+                return;
+            }
+
+            PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(regex_.match);
+            for (const auto &kv : matches_) {
+                const auto &name = kv.first;
+                // A named group may actually correspond to multiple group numbers, each of which
+                // might have to be enumerated.
+                PCRE2_SPTR first = nullptr;
+                PCRE2_SPTR last = nullptr;
+                int entry_size = pcre2_substring_nametable_scan(
+                    regex_.code, (PCRE2_SPTR)(name.c_str()), &first, &last);
+                if (entry_size <= 0) {
+                    FLOGF(warning, L"PCRE2 failure retrieving named matches");
+                    continue;
+                }
+
+                if (!match_found) {
+                    matches_[name].emplace_back(L"");
+                    continue;
+                }
+
+                bool value_found = false;
+                for (auto group_ptr = first; group_ptr <= last; group_ptr += entry_size) {
+                    int group_num = group_ptr[0];
+
+                    PCRE2_SIZE *capture = ovector + (2 * group_num);
+                    PCRE2_SIZE begin = capture[0];
+                    PCRE2_SIZE end = capture[1];
+
+                    if (begin != PCRE2_UNSET && end != PCRE2_UNSET && end >= begin) {
+                        matches_[name].emplace_back(haystack_.substr(begin, end - begin));
+                        value_found = true;
+                        break;
+                    }
+                }
+
+                // If there are multiple named groups and --all was used, we need to ensure that the
+                // indexes are always in sync between the variables. If an optional named group
+                // didn't match but its brethren did, we need to make sure to put *something* in the
+                // resulting array, and unfortunately fish doesn't support empty/null members so
+                // we're going to have to use an empty string as the sentinel value.
+                if (!value_found) {
+                    matches_[name].emplace_back(wcstring{});
+                }
+            }
+        }
+
+        ~regex_importer_t() {
+            if (skip_import_) {
+                return;
+            }
+
+            auto &vars = parser_.vars();
+            for (const auto &kv : matches_) {
+                const auto &name = kv.first;
+                const auto &value = kv.second;
+
+                if (!match_found_) {
+                    vars.set_empty(name, ENV_DEFAULT);
+                } else {
+                    vars.set(name, ENV_DEFAULT, value);
+                }
+            }
+        }
+    };
+
    public:
     pcre2_matcher_t(const wchar_t *argv0_, const wcstring &pattern, const options_t &opts,
-                    io_streams_t &streams)
+                    io_streams_t &streams, parser_t &parser_)
         : string_matcher_t(opts, streams),
           argv0(argv0_),
-          regex(argv0_, pattern, opts.ignore_case, streams) {}
+          regex(argv0_, pattern, opts.ignore_case, streams),
+          parser(parser_) {}
 
     ~pcre2_matcher_t() override = default;
 
@@ -894,17 +1041,30 @@ class pcre2_matcher_t : public string_matcher_t {
             return false;
         }
 
+        regex_importer_t var_importer(this->parser, arg, this->regex);
+
+        // We must manually init the importer rather than relegating this to the constructor
+        // because it will validate the names it is importing to make sure they're all legal and
+        // writeable.
+        if (!var_importer.init(streams)) {
+            // init() directly reports errors itself so it can specify the problem variable
+            return false;
+        }
+
         // See pcre2demo.c for an explanation of this logic.
         PCRE2_SIZE arglen = arg.length();
-        int rc = report_match(arg, pcre2_match(regex.code, PCRE2_SPTR(arg.c_str()), arglen, 0, 0,
+        auto rc = report_match(arg, pcre2_match(regex.code, PCRE2_SPTR(arg.c_str()), arglen, 0, 0,
                                                regex.match, nullptr));
+        var_importer.import_vars(rc == match_result_t::match);
 
-        if (rc < 0 /* pcre2 error */)
-            return false;
-        else if (rc == 0 /* no match */)
-            return true;
-        else
-            total_matched++;
+        switch (rc) {
+            case match_result_t::pcre2_error:
+                return false;
+            case match_result_t::no_match:
+                return true;
+            case match_result_t::match:
+                total_matched++;
+        }
 
         if (opts.invert_match) return true;
 
@@ -921,14 +1081,22 @@ class pcre2_matcher_t : public string_matcher_t {
             rc = report_match(arg, pcre2_match(regex.code, PCRE2_SPTR(arg.c_str()), arglen, offset,
                                                options, regex.match, nullptr));
 
-            if (rc < 0 /* pcre2 error */)
+            if (rc == match_result_t::pcre2_error) {
+                // This shouldn't happen as we've already validated the regex above
                 return false;
-            else if (rc == 0 /* no matches */) {
+            }
+
+            // Call import_vars() before modifying the ovector
+            if (rc == match_result_t::match) {
+                var_importer.import_vars(true /* match found */);
+            }
+
+            if (rc == match_result_t::no_match) {
                 if (options == 0 /* all matches found now */) break;
                 ovector[1] = offset + 1;
-                continue;
             }
         }
+
         return true;
     }
 };
@@ -957,7 +1125,7 @@ static int string_match(parser_t &parser, io_streams_t &streams, int argc, wchar
 
     std::unique_ptr<string_matcher_t> matcher;
     if (opts.regex) {
-        matcher = make_unique<pcre2_matcher_t>(cmd, pattern, opts, streams);
+        matcher = make_unique<pcre2_matcher_t>(cmd, pattern, opts, streams, parser);
     } else {
         matcher = make_unique<wildcard_matcher_t>(cmd, pattern, opts, streams);
     }

tests/checks/regex-import.fish

@@ -0,0 +1,40 @@
+#RUN: %fish %s
+# Tests for importing named regex groups as fish variables
+
+# Capture first match
+echo "hello world" | string match --regex -q -- '(?<words>[^ ]+) ?'
+printf "%s\n" $words
+# CHECK: hello
+
+# Capture multiple matches
+echo "hello world" | string match --regex -q --all -- '(?<words>[^ ]+) ?'
+printf "%s\n" $words
+# CHECK: hello
+# CHECK: world
+
+# Capture multiple variables
+echo "hello world" | string match -rq -- '^(?<word1>[^ ]+) (?<word2>.*)$'
+printf "%s\n" $word1 $word2
+# CHECK: hello
+# CHECK: world
+
+# Clear variables on no match
+set foo foo
+echo "foo" | string match -rq -- '^(?<foo>bar)$'
+echo $foo
+# CHECK:
+
+# Named group may be empty in some of the matches
+set word
+set punctuation
+echo "hello world, boy!" | string match -a -qr -- '(?<word>[^ .,!;]+)(?<punctuation>[.,!;])?'
+echo $word
+# CHECK: hello world boy
+printf "%s\n" $punctuation
+# CHECK:
+# CHECK: ,
+# CHECK: !
+
+# Verify read-only variables may not be imported
+echo hello | string match -rq "(?<version>.*)"
+# CHECKERR: Modification of read-only variable "version" is not allowed