diff options
author | Asko Oja | 2009-05-04 14:02:39 +0000 |
---|---|---|
committer | Marko Kreen | 2009-06-01 07:49:44 +0000 |
commit | 9cbd16ab343c28c55012127a1cb643b793547416 (patch) | |
tree | 6a5b351db839396c8d6912d2a0eb90fe1c94d734 /scripts/queue_loader.py | |
parent | 16e20c4daccb4f0e95249d8fcfe32e89bae41882 (diff) |
python scripts: Move template config to docstring.
* Template config is now in docstring to make to more
easy to keep up-to-date.
* dbscript: --ini option to show template config.
Diffstat (limited to 'scripts/queue_loader.py')
-rwxr-xr-x | scripts/queue_loader.py | 84 |
1 files changed, 80 insertions, 4 deletions
diff --git a/scripts/queue_loader.py b/scripts/queue_loader.py index c14ecd01..c71b20e9 100755 --- a/scripts/queue_loader.py +++ b/scripts/queue_loader.py @@ -1,6 +1,83 @@ #! /usr/bin/env python -"""Load data from queue into tables, with optional partitioning.""" +"""Load data from queue into tables, with optional partitioning. + +Config template:: + + [queue_loader] + job_name = + logfile = + pidfile = + + db = + + #rename_tables = + + [DEFAULT] + + # fields - which fields to send through + #fields = col1, col2, col3:renamed3 + #fields = * + + # table_mode - how to handle a table + # + # ignore - ignore this table + # direct - update table directly + # split - split data into partitions + #table_mode = ignore + + # split_mode - how to split, if requested + # + # by-batch-time: use batch time for splitting + # by-event-time: use event time for splitting + # by-date-field:fld - use fld for splitting + #split_mode = by-batch-time + + # split_part - partition name format + # + # %(table_name)s %(year)s %(month)s %(day)s %(hour)s + #split_part = %(table_name)s_%(year)s_%(month)s_%(day)s + + # split_part_template - How to create new partition tables + # + # Available fields: + # %(part)s + # %(parent)s + # %(pkey)s + # + ### Non-inherited partitions + #split_part_template = + # create table %%(part)s (like %%(parent)s); + # alter table only %%(part)s add primary key (%%(pkey)s); + # + ### Inherited partitions + #split_part_template = + # create table %%(part)s () inherits (%%(parent)s); + # alter table only %%(part)s add primary key (%%(pkey)s); + + + # row_mode - How to apply the events + # + # plain - each event creates SQL statement to run + # keep_latest - change updates to DELETE + INSERT + # keep_all - change updates to inserts, ignore deletes + # bulk - instead of statement-per-row, do bulk updates + #row_mode = plain + + + # bulk_mode - How to do the bulk update + # + # correct - inserts as COPY into table, + # update as COPY into temp table and single UPDATE from there + # delete as COPY into temp table and single DELETE from there + # delete - as 'correct', but do update as DELETE + COPY + # merged - as 'delete', but merge insert rows with update rows + #bulk_mode=correct + + [table public.foo] + mode = + create_sql = +""" import sys, time, skytools @@ -253,9 +330,8 @@ class BulkLoader(BasicLoader): self.log.debug(del_sql) curs.execute(del_sql) self.log.debug("%s - %d" % (curs.statusmessage, curs.rowcount)) - self.log.debug(curs.statusmessage) if len(del_list) != curs.rowcount: - self.log.warning("Delete mismatch: expected=%s updated=%d" + self.log.warning("Delete mismatch: expected=%s deleted=%d" % (len(del_list), curs.rowcount)) temp_used = True @@ -274,7 +350,7 @@ class BulkLoader(BasicLoader): # update main table self.log.debug(upd_sql) curs.execute(upd_sql) - self.log.debug(curs.statusmessage) + self.log.debug("%s - %d" % (curs.statusmessage, curs.rowcount)) # check count if len(upd_list) != curs.rowcount: self.log.warning("Update mismatch: expected=%s updated=%d" |