diff options
| author | Marko Kreen | 2010-12-03 15:26:10 +0000 |
|---|---|---|
| committer | Marko Kreen | 2010-12-03 15:26:10 +0000 |
| commit | f2887be34e81ad75c0befb2c67d30b0e8b6a49f9 (patch) | |
| tree | 6278b57876aa6670beb6c44dfdfecb2bef3e753f /python/skytools/parsing.py | |
| parent | bfc38278b72e10e2f4e85f15dcbac743f25d2060 (diff) | |
sql_tokenizer: options to returns quoted idents separately
Diffstat (limited to 'python/skytools/parsing.py')
| -rw-r--r-- | python/skytools/parsing.py | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/python/skytools/parsing.py b/python/skytools/parsing.py index d45bb839..2187addc 100644 --- a/python/skytools/parsing.py +++ b/python/skytools/parsing.py @@ -248,7 +248,7 @@ _std_sql_rc = _ext_sql_rc = None _std_sql_fq_rc = _ext_sql_fq_rc = None def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False, - fqident = False, show_location = False): + fqident = False, show_location = False, use_qident = False): r"""Parser SQL to tokens. Iterator, returns (toktype, tokstr) tuples. @@ -288,10 +288,13 @@ def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False, typ = m.lastgroup if ignore_whitespace and typ == "ws": continue + tk = m.group() + if use_qident and typ == 'ident' and tk[0] == '"': + typ = 'qident' if show_location: - yield (typ, m.group(), pos) + yield (typ, tk, pos) else: - yield (typ, m.group()) + yield (typ, tk) _copy_from_stdin_re = "copy.*from\s+stdin" _copy_from_stdin_rc = None |