summaryrefslogtreecommitdiff
path: root/scripts/queue_loader.py
diff options
context:
space:
mode:
authorAsko Oja2009-05-04 14:02:39 +0000
committerMarko Kreen2009-06-01 07:49:44 +0000
commit9cbd16ab343c28c55012127a1cb643b793547416 (patch)
tree6a5b351db839396c8d6912d2a0eb90fe1c94d734 /scripts/queue_loader.py
parent16e20c4daccb4f0e95249d8fcfe32e89bae41882 (diff)
python scripts: Move template config to docstring.
* Template config is now in docstring to make to more easy to keep up-to-date. * dbscript: --ini option to show template config.
Diffstat (limited to 'scripts/queue_loader.py')
-rwxr-xr-xscripts/queue_loader.py84
1 files changed, 80 insertions, 4 deletions
diff --git a/scripts/queue_loader.py b/scripts/queue_loader.py
index c14ecd01..c71b20e9 100755
--- a/scripts/queue_loader.py
+++ b/scripts/queue_loader.py
@@ -1,6 +1,83 @@
#! /usr/bin/env python
-"""Load data from queue into tables, with optional partitioning."""
+"""Load data from queue into tables, with optional partitioning.
+
+Config template::
+
+ [queue_loader]
+ job_name =
+ logfile =
+ pidfile =
+
+ db =
+
+ #rename_tables =
+
+ [DEFAULT]
+
+ # fields - which fields to send through
+ #fields = col1, col2, col3:renamed3
+ #fields = *
+
+ # table_mode - how to handle a table
+ #
+ # ignore - ignore this table
+ # direct - update table directly
+ # split - split data into partitions
+ #table_mode = ignore
+
+ # split_mode - how to split, if requested
+ #
+ # by-batch-time: use batch time for splitting
+ # by-event-time: use event time for splitting
+ # by-date-field:fld - use fld for splitting
+ #split_mode = by-batch-time
+
+ # split_part - partition name format
+ #
+ # %(table_name)s %(year)s %(month)s %(day)s %(hour)s
+ #split_part = %(table_name)s_%(year)s_%(month)s_%(day)s
+
+ # split_part_template - How to create new partition tables
+ #
+ # Available fields:
+ # %(part)s
+ # %(parent)s
+ # %(pkey)s
+ #
+ ### Non-inherited partitions
+ #split_part_template =
+ # create table %%(part)s (like %%(parent)s);
+ # alter table only %%(part)s add primary key (%%(pkey)s);
+ #
+ ### Inherited partitions
+ #split_part_template =
+ # create table %%(part)s () inherits (%%(parent)s);
+ # alter table only %%(part)s add primary key (%%(pkey)s);
+
+
+ # row_mode - How to apply the events
+ #
+ # plain - each event creates SQL statement to run
+ # keep_latest - change updates to DELETE + INSERT
+ # keep_all - change updates to inserts, ignore deletes
+ # bulk - instead of statement-per-row, do bulk updates
+ #row_mode = plain
+
+
+ # bulk_mode - How to do the bulk update
+ #
+ # correct - inserts as COPY into table,
+ # update as COPY into temp table and single UPDATE from there
+ # delete as COPY into temp table and single DELETE from there
+ # delete - as 'correct', but do update as DELETE + COPY
+ # merged - as 'delete', but merge insert rows with update rows
+ #bulk_mode=correct
+
+ [table public.foo]
+ mode =
+ create_sql =
+"""
import sys, time, skytools
@@ -253,9 +330,8 @@ class BulkLoader(BasicLoader):
self.log.debug(del_sql)
curs.execute(del_sql)
self.log.debug("%s - %d" % (curs.statusmessage, curs.rowcount))
- self.log.debug(curs.statusmessage)
if len(del_list) != curs.rowcount:
- self.log.warning("Delete mismatch: expected=%s updated=%d"
+ self.log.warning("Delete mismatch: expected=%s deleted=%d"
% (len(del_list), curs.rowcount))
temp_used = True
@@ -274,7 +350,7 @@ class BulkLoader(BasicLoader):
# update main table
self.log.debug(upd_sql)
curs.execute(upd_sql)
- self.log.debug(curs.statusmessage)
+ self.log.debug("%s - %d" % (curs.statusmessage, curs.rowcount))
# check count
if len(upd_list) != curs.rowcount:
self.log.warning("Update mismatch: expected=%s updated=%d"